Java Code Examples for com.lmax.disruptor.WaitStrategy

The following are top voted examples for showing how to use com.lmax.disruptor.WaitStrategy. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: eds   File: PublishManager.java   Source Code and License 6 votes vote down vote up
private void initDisruptor(int processors, int ringBufferSize) {
  LOG.info("eds client init disruptor with processors="
      + processors + " and ringBufferSize=" + ringBufferSize);

  executor = Executors.newFixedThreadPool(
      processors,
      new ThreadFactoryBuilder().setNameFormat("disruptor-executor-%d").build());

  final WaitStrategy waitStrategy = createWaitStrategy();
  ringBufferSize = sizeFor(ringBufferSize); // power of 2
  disruptor = new Disruptor<>(EdsRingBufferEvent.FACTORY, ringBufferSize, executor,
      ProducerType.MULTI, waitStrategy);

  EdsEventWorkHandler[] handlers = new EdsEventWorkHandler[processors];
  for (int i = 0; i < handlers.length; i++) {
    handlers[i] = new EdsEventWorkHandler();
  }
  // handlers number = threads number
  disruptor.handleEventsWithWorkerPool(handlers); // "handleEventsWith" just like topics , with multiple consumers

  disruptor.start();
}
 
Example 2
Project: incubator-omid   File: DisruptorModule.java   Source Code and License 6 votes vote down vote up
@Override
protected void configure() {
    switch (config.getWaitStrategyEnum()) {
    // A low-cpu usage Disruptor configuration for using in local/test environments
    case LOW_CPU:
         bind(WaitStrategy.class).annotatedWith(Names.named("PersistenceStrategy")).to(BlockingWaitStrategy.class);
         bind(WaitStrategy.class).annotatedWith(Names.named("ReplyStrategy")).to(BlockingWaitStrategy.class);
         bind(WaitStrategy.class).annotatedWith(Names.named("RetryStrategy")).to(BlockingWaitStrategy.class);
         break;
    // The default high-cpu usage Disruptor configuration for getting high throughput on production environments
    case HIGH_THROUGHPUT:
    default:
         bind(WaitStrategy.class).annotatedWith(Names.named("PersistenceStrategy")).to(BusySpinWaitStrategy.class);
         bind(WaitStrategy.class).annotatedWith(Names.named("ReplyStrategy")).to(BusySpinWaitStrategy.class);
         bind(WaitStrategy.class).annotatedWith(Names.named("RetryStrategy")).to(YieldingWaitStrategy.class);
         break;
    }
    bind(RequestProcessor.class).to(RequestProcessorImpl.class).in(Singleton.class);
    bind(PersistenceProcessor.class).to(PersistenceProcessorImpl.class).in(Singleton.class);
    bind(ReplyProcessor.class).to(ReplyProcessorImpl.class).in(Singleton.class);
    bind(RetryProcessor.class).to(RetryProcessorImpl.class).in(Singleton.class);

}
 
Example 3
Project: camunda-bpm-reactor   File: RingBufferProcessor.java   Source Code and License 6 votes vote down vote up
private RingBufferProcessor(String name,
                            ExecutorService executor,
                            int bufferSize,
                            WaitStrategy waitStrategy,
                            boolean shared,
                            boolean autoCancel) {
  super(name, executor, autoCancel);

  this.ringBuffer = RingBuffer.create(
    shared ? ProducerType.MULTI : ProducerType.SINGLE,
    new EventFactory<MutableSignal<E>>() {
      @Override
      public MutableSignal<E> newInstance() {
        return new MutableSignal<E>();
      }
    },
    bufferSize,
    waitStrategy
  );

  this.recentSequence = new Sequence(Sequencer.INITIAL_CURSOR_VALUE);
  this.barrier = ringBuffer.newBarrier();
  //ringBuffer.addGatingSequences(recentSequence);
}
 
Example 4
Project: camunda-bpm-reactor   File: RingBufferWorkProcessor.java   Source Code and License 6 votes vote down vote up
private RingBufferWorkProcessor(String name,
                                ExecutorService executor,
                                int bufferSize,
                                WaitStrategy waitStrategy,
                                boolean share,
                                boolean autoCancel) {
  super(name, executor, autoCancel);

  this.ringBuffer = RingBuffer.create(
    share ? ProducerType.MULTI : ProducerType.SINGLE,
    new EventFactory<MutableSignal<E>>() {
      @Override
      public MutableSignal<E> newInstance() {
        return new MutableSignal<E>();
      }
    },
    bufferSize,
    waitStrategy
  );

  ringBuffer.addGatingSequences(workSequence);

}
 
Example 5
Project: jstorm-0.9.6.3-   File: DisruptorQueueImpl.java   Source Code and License 6 votes vote down vote up
public DisruptorQueueImpl(String queueName, ProducerType producerType,
		int bufferSize, WaitStrategy wait) {
	this._queueName = PREFIX + queueName;
	_buffer = RingBuffer.create(producerType, new ObjectEventFactory(),
			bufferSize, wait);
	_consumer = new Sequence();
	_barrier = _buffer.newBarrier();
	_buffer.addGatingSequences(_consumer);
	if (producerType == ProducerType.SINGLE) {
		consumerStartedFlag = true;
	} else {
		// make sure we flush the pending messages in cache first
		if (bufferSize < 2) {
			throw new RuntimeException("QueueSize must >= 2");
		}
		try {
			publishDirect(FLUSH_CACHE, true);
		} catch (InsufficientCapacityException e) {
			throw new RuntimeException("This code should be unreachable!",
					e);
		}
	}
}
 
Example 6
Project: jstorm-0.9.6.3-   File: RingBuffer.java   Source Code and License 6 votes vote down vote up
/**
 * Create a new Ring Buffer with the specified producer type (SINGLE or MULTI)
 *
 * @param producerType producer type to use {@link ProducerType}.
 * @param factory used to create events within the ring buffer.
 * @param bufferSize number of elements to create within the ring buffer.
 * @param waitStrategy used to determine how to wait for new elements to become available.
 * @throws IllegalArgumentException if bufferSize is less than 1 or not a power of 2
 */
public static <E> RingBuffer<E> create(ProducerType    producerType,
                                       EventFactory<E> factory,
                                       int             bufferSize,
                                       WaitStrategy    waitStrategy)
{
    switch (producerType)
    {
    case SINGLE:
        return createSingleProducer(factory, bufferSize, waitStrategy);
    case MULTI:
        return createMultiProducer(factory, bufferSize, waitStrategy);
    default:
        throw new IllegalStateException(producerType.toString());
    }
}
 
Example 7
Project: learn_jstorm   File: DisruptorQueueImpl.java   Source Code and License 6 votes vote down vote up
public DisruptorQueueImpl(String queueName, ProducerType producerType,
		int bufferSize, WaitStrategy wait) {
	this._queueName = PREFIX + queueName;
	_buffer = RingBuffer.create(producerType, new ObjectEventFactory(),
			bufferSize, wait);
	_consumer = new Sequence();
	_barrier = _buffer.newBarrier();
	_buffer.addGatingSequences(_consumer);
	if (producerType == ProducerType.SINGLE) {
		consumerStartedFlag = true;
	} else {
		// make sure we flush the pending messages in cache first
		if (bufferSize < 2) {
			throw new RuntimeException("QueueSize must >= 2");
		}
		try {
			publishDirect(FLUSH_CACHE, true);
		} catch (InsufficientCapacityException e) {
			throw new RuntimeException("This code should be unreachable!",
					e);
		}
	}
}
 
Example 8
Project: learn_jstorm   File: RingBuffer.java   Source Code and License 6 votes vote down vote up
/**
 * Create a new Ring Buffer with the specified producer type (SINGLE or MULTI)
 *
 * @param producerType producer type to use {@link ProducerType}.
 * @param factory used to create events within the ring buffer.
 * @param bufferSize number of elements to create within the ring buffer.
 * @param waitStrategy used to determine how to wait for new elements to become available.
 * @throws IllegalArgumentException if bufferSize is less than 1 or not a power of 2
 */
public static <E> RingBuffer<E> create(ProducerType    producerType,
                                       EventFactory<E> factory,
                                       int             bufferSize,
                                       WaitStrategy    waitStrategy)
{
    switch (producerType)
    {
    case SINGLE:
        return createSingleProducer(factory, bufferSize, waitStrategy);
    case MULTI:
        return createMultiProducer(factory, bufferSize, waitStrategy);
    default:
        throw new IllegalStateException(producerType.toString());
    }
}
 
Example 9
Project: jstrom   File: DisruptorQueueImpl.java   Source Code and License 6 votes vote down vote up
public DisruptorQueueImpl(String queueName, ProducerType producerType, int bufferSize, WaitStrategy wait) {
    this._queueName = PREFIX + queueName;
    _buffer = RingBuffer.create(producerType, new ObjectEventFactory(), bufferSize, wait);
    _consumer = new Sequence();
    _barrier = _buffer.newBarrier();
    _buffer.addGatingSequences(_consumer);
    if (producerType == ProducerType.SINGLE) {
        consumerStartedFlag = true;
    } else {
        // make sure we flush the pending messages in cache first
        if (bufferSize < 2) {
            throw new RuntimeException("QueueSize must >= 2");
        }
        try {
            publishDirect(FLUSH_CACHE, true);
        } catch (InsufficientCapacityException e) {
            throw new RuntimeException("This code should be unreachable!", e);
        }
    }
}
 
Example 10
Project: Tstream   File: DisruptorQueueImpl.java   Source Code and License 6 votes vote down vote up
public DisruptorQueueImpl(String queueName, ProducerType producerType,
		int bufferSize, WaitStrategy wait) {
	this._queueName = PREFIX + queueName;
	_buffer = RingBuffer.create(producerType, new ObjectEventFactory(),
			bufferSize, wait);
	_consumer = new Sequence();
	_barrier = _buffer.newBarrier();
	_buffer.addGatingSequences(_consumer);
	if (producerType == ProducerType.SINGLE) {
		consumerStartedFlag = true;
	} else {
		// make sure we flush the pending messages in cache first
		if (bufferSize < 2) {
			throw new RuntimeException("QueueSize must >= 2");
		}
		try {
			publishDirect(FLUSH_CACHE, true);
		} catch (InsufficientCapacityException e) {
			throw new RuntimeException("This code should be unreachable!",
					e);
		}
	}
}
 
Example 11
Project: Tstream   File: RingBuffer.java   Source Code and License 6 votes vote down vote up
/**
 * Create a new Ring Buffer with the specified producer type (SINGLE or MULTI)
 *
 * @param producerType producer type to use {@link ProducerType}.
 * @param factory used to create events within the ring buffer.
 * @param bufferSize number of elements to create within the ring buffer.
 * @param waitStrategy used to determine how to wait for new elements to become available.
 * @throws IllegalArgumentException if bufferSize is less than 1 or not a power of 2
 */
public static <E> RingBuffer<E> create(ProducerType    producerType,
                                       EventFactory<E> factory,
                                       int             bufferSize,
                                       WaitStrategy    waitStrategy)
{
    switch (producerType)
    {
    case SINGLE:
        return createSingleProducer(factory, bufferSize, waitStrategy);
    case MULTI:
        return createMultiProducer(factory, bufferSize, waitStrategy);
    default:
        throw new IllegalStateException(producerType.toString());
    }
}
 
Example 12
Project: smart-cqrs   File: EventBus.java   Source Code and License 6 votes vote down vote up
protected Disruptor<BaseEvent> createDisruptor(Class<?> eventClass, List<RegistedEventHandler> registedEventHandlers) {
    WaitStrategy waitStrategy = new BlockingWaitStrategy();
    // load the customized event bufferSize.
    int bufferSize = EventUtils.getEventBufferSize(eventClass);
    Disruptor<BaseEvent> disruptor =new Disruptor<BaseEvent>(new BaseEventFactory(), 
            bufferSize, Executors.newCachedThreadPool(), ProducerType.SINGLE, waitStrategy);
    List<BaseEventHandler> baseEventHandlers = Lists.newArrayList();
    EventType eventType = EventUtils.getEventType(eventClass);
    if(EventType.ORDER.equals(eventType)) {
        List<RegistedEventHandler> orderingHandlers = orderingRegistedEventHandlers(registedEventHandlers);
        baseEventHandlers.add(new BaseEventHandler(EventType.ORDER, orderingHandlers.toArray(new RegistedEventHandler[0])));
    } else if(EventType.CONCURRENCY.equals(eventType)) { 
        for(RegistedEventHandler registedEventHandler : registedEventHandlers) {
            baseEventHandlers.add(new BaseEventHandler(EventType.CONCURRENCY, registedEventHandler));
        }
    } else {
        throw new RuntimeException("The definition of event type is not correct.");
    }
    disruptor.handleEventsWith(baseEventHandlers.toArray(new BaseEventHandler[0]));
    disruptor.start();
    return disruptor;
}
 
Example 13
Project: log4j2   File: AsyncLogger.java   Source Code and License 6 votes vote down vote up
private static WaitStrategy createWaitStrategy() {
    final String strategy = System.getProperty("AsyncLogger.WaitStrategy");
    LOGGER.debug("property AsyncLogger.WaitStrategy={}", strategy);
    if ("Sleep".equals(strategy)) {
        LOGGER.debug("disruptor event handler uses SleepingWaitStrategy");
        return new SleepingWaitStrategy();
    } else if ("Yield".equals(strategy)) {
        LOGGER.debug("disruptor event handler uses YieldingWaitStrategy");
        return new YieldingWaitStrategy();
    } else if ("Block".equals(strategy)) {
        LOGGER.debug("disruptor event handler uses BlockingWaitStrategy");
        return new BlockingWaitStrategy();
    }
    LOGGER.debug("disruptor event handler uses SleepingWaitStrategy");
    return new SleepingWaitStrategy();
}
 
Example 14
Project: log4j2   File: AsyncLoggerConfigHelper.java   Source Code and License 6 votes vote down vote up
private static synchronized void initDisruptor() {
    if (disruptor != null) {
        LOGGER.trace("AsyncLoggerConfigHelper not starting new disruptor, using existing object. Ref count is {}.", count);
        return;
    }
    LOGGER.trace("AsyncLoggerConfigHelper creating new disruptor. Ref count is {}.", count);
    final int ringBufferSize = calculateRingBufferSize();
    final WaitStrategy waitStrategy = createWaitStrategy();
    executor = Executors.newSingleThreadExecutor(threadFactory);
    disruptor = new Disruptor<Log4jEventWrapper>(FACTORY, ringBufferSize,
            executor, ProducerType.MULTI, waitStrategy);
    final EventHandler<Log4jEventWrapper>[] handlers = new Log4jEventWrapperHandler[] {//
    new Log4jEventWrapperHandler() };
    final ExceptionHandler errorHandler = getExceptionHandler();
    disruptor.handleExceptionsWith(errorHandler);
    disruptor.handleEventsWith(handlers);

    LOGGER.debug(
            "Starting AsyncLoggerConfig disruptor with ringbuffer size={}, waitStrategy={}, exceptionHandler={}...",
            disruptor.getRingBuffer().getBufferSize(), waitStrategy.getClass().getSimpleName(), errorHandler);
    disruptor.start();
}
 
Example 15
Project: jstorm   File: DisruptorQueueImpl.java   Source Code and License 6 votes vote down vote up
public DisruptorQueueImpl(String queueName, ProducerType producerType, int bufferSize, WaitStrategy wait, boolean isBatch, int batchSize, long flushMs) {
    _queueName = PREFIX + queueName;
    _buffer = RingBuffer.create(producerType, new ObjectEventFactory(), bufferSize, wait);
    _consumer = new Sequence();
    _barrier = _buffer.newBarrier();
    _buffer.addGatingSequences(_consumer);
    _isBatch = isBatch;
    _cache = new ArrayList<>();
    _inputBatchSize = batchSize;
    if (_isBatch) {
        _batcher = new ThreadLocalBatch();
        _flusher = new DisruptorFlusher(Math.max(flushMs, 1));
        _flusher.start();
    } else {
        _batcher = null;
    }
}
 
Example 16
Project: jstorm   File: NettyUnitTest.java   Source Code and License 6 votes vote down vote up
private IConnection initNettyServer(int port) {
    ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    //ConcurrentHashMap<Integer, DisruptorQueue> deserializeCtrlQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();

    WaitStrategy wait = (WaitStrategy)Utils.newInstance("com.lmax.disruptor.TimeoutBlockingWaitStrategy", 5, TimeUnit.MILLISECONDS);
    DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI,
            256, wait, false, 0, 0);
    Set<Integer> taskSet = new HashSet<Integer>();
    taskSet.add(1);
    IConnection server = context.bind(null, port, deserializeQueues, recvControlQueue, true, taskSet);

    WaitStrategy waitStrategy = new BlockingWaitStrategy();
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy, false, 0, 0);
    server.registerQueue(task, recvQueue);

    return server;
}
 
Example 17
Project: hashsdn-controller   File: DOMNotificationRouter.java   Source Code and License 5 votes vote down vote up
@SuppressWarnings("unchecked")
private DOMNotificationRouter(final ExecutorService executor, final int queueDepth, final WaitStrategy strategy) {
    this.executor = Preconditions.checkNotNull(executor);

    disruptor = new Disruptor<>(DOMNotificationRouterEvent.FACTORY, queueDepth, executor, ProducerType.MULTI, strategy);
    disruptor.handleEventsWith(DISPATCH_NOTIFICATIONS);
    disruptor.after(DISPATCH_NOTIFICATIONS).handleEventsWith(NOTIFY_FUTURE);
    disruptor.start();
}
 
Example 18
Project: hashsdn-controller   File: DOMNotificationRouter.java   Source Code and License 5 votes vote down vote up
public static DOMNotificationRouter create(final int queueDepth, final long spinTime, final long parkTime, final TimeUnit unit) {
    Preconditions.checkArgument(Long.lowestOneBit(queueDepth) == Long.highestOneBit(queueDepth),
            "Queue depth %s is not power-of-two", queueDepth);
    final ExecutorService executor = Executors.newCachedThreadPool();
    final WaitStrategy strategy = PhasedBackoffWaitStrategy.withLock(spinTime, parkTime, unit);

    return new DOMNotificationRouter(executor, queueDepth, strategy);
}
 
Example 19
Project: incubator-omid   File: ReplyProcessorImpl.java   Source Code and License 5 votes vote down vote up
@Inject
ReplyProcessorImpl(@Named("ReplyStrategy") WaitStrategy strategy,
        MetricsRegistry metrics, Panicker panicker, ObjectPool<Batch> batchPool) {

    // ------------------------------------------------------------------------------------------------------------
    // Disruptor initialization
    // ------------------------------------------------------------------------------------------------------------

    ThreadFactoryBuilder threadFactory = new ThreadFactoryBuilder().setNameFormat("reply-%d");
    this.disruptorExec = Executors.newSingleThreadExecutor(threadFactory.build());

    this.disruptor = new Disruptor<>(EVENT_FACTORY, 1 << 12, disruptorExec, MULTI, strategy);
    disruptor.handleExceptionsWith(new FatalExceptionHandler(panicker));
    disruptor.handleEventsWith(this);
    this.replyRing = disruptor.start();

    // ------------------------------------------------------------------------------------------------------------
    // Attribute initialization
    // ------------------------------------------------------------------------------------------------------------

    this.batchPool = batchPool;
    this.nextIDToHandle.set(0);
    this.futureEvents = new PriorityQueue<>(10, new Comparator<ReplyBatchEvent>() {
        public int compare(ReplyBatchEvent replyBatchEvent1, ReplyBatchEvent replyBatchEvent2) {
            return Long.compare(replyBatchEvent1.getBatchSequence(), replyBatchEvent2.getBatchSequence());
        }
    });

    // Metrics config
    this.abortMeter = metrics.meter(name("tso", "aborts"));
    this.commitMeter = metrics.meter(name("tso", "commits"));
    this.timestampMeter = metrics.meter(name("tso", "timestampAllocation"));
    this.fenceMeter = metrics.meter(name("tso", "fences"));

    LOG.info("ReplyProcessor initialized");

}
 
Example 20
Project: incubator-omid   File: RetryProcessorImpl.java   Source Code and License 5 votes vote down vote up
@Inject
RetryProcessorImpl(@Named("RetryStrategy") WaitStrategy strategy,
                   MetricsRegistry metrics,
                   CommitTable commitTable,
                   ReplyProcessor replyProc,
                   Panicker panicker,
                   ObjectPool<Batch> batchPool)
        throws InterruptedException, ExecutionException, IOException {

    // ------------------------------------------------------------------------------------------------------------
    // Disruptor initialization
    // ------------------------------------------------------------------------------------------------------------

    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("retry-%d").build();
    this.disruptorExec = Executors.newSingleThreadExecutor(threadFactory);

    this.disruptor = new Disruptor<>(EVENT_FACTORY, 1 << 12, disruptorExec, SINGLE, strategy);
    disruptor.handleExceptionsWith(new FatalExceptionHandler(panicker)); // This must be before handleEventsWith()
    disruptor.handleEventsWith(this);
    this.retryRing = disruptor.start();

    // ------------------------------------------------------------------------------------------------------------
    // Attribute initialization
    // ------------------------------------------------------------------------------------------------------------

    this.commitTableClient = commitTable.getClient();
    this.replyProc = replyProc;
    this.batchPool = batchPool;

    // Metrics configuration
    this.txAlreadyCommittedMeter = metrics.meter(name("tso", "retries", "commits", "tx-already-committed"));
    this.invalidTxMeter = metrics.meter(name("tso", "retries", "aborts", "tx-invalid"));
    this.noCTFoundMeter = metrics.meter(name("tso", "retries", "aborts", "tx-without-commit-timestamp"));

    LOG.info("RetryProcessor initialized");

}
 
Example 21
Project: disruptor-code-analysis   File: Disruptor.java   Source Code and License 5 votes vote down vote up
/**
 * Create a new Disruptor.
 *
 * @param eventFactory   the factory to create events in the ring buffer.
 * @param ringBufferSize the size of the ring buffer, must be power of 2.
 * @param threadFactory  a {@link ThreadFactory} to create threads for processors.
 * @param producerType   the claim strategy to use for the ring buffer.
 * @param waitStrategy   the wait strategy to use for the ring buffer.
 */
public Disruptor(
        final EventFactory<T> eventFactory,
        final int ringBufferSize,
        final ThreadFactory threadFactory,
        final ProducerType producerType,
        final WaitStrategy waitStrategy) {
    this(RingBuffer.create(
            producerType, eventFactory, ringBufferSize, waitStrategy),
            new BasicExecutor(threadFactory));
}
 
Example 22
Project: vulcan   File: AvroWriterBuilder.java   Source Code and License 5 votes vote down vote up
@Override
public OptionalSteps withWaitStrategy(WaitStrategy waitStrategy) {
  if (waitStrategy != null) {
    this.waitStrategy = waitStrategy;
  } else {
    log.warn("Tried to configure the waiting strategy with a null value");
  }
  return this;
}
 
Example 23
Project: Camel   File: DisruptorWaitStrategyCreationTest.java   Source Code and License 5 votes vote down vote up
@Test
public void testCreateWaitStrategyInstance() throws Exception {
    for (final DisruptorWaitStrategy strategy : DisruptorWaitStrategy.values()) {
        final WaitStrategy waitStrategyInstance = strategy.createWaitStrategyInstance();

        assertNotNull(waitStrategyInstance);
        assertTrue(waitStrategyInstance instanceof WaitStrategy);
    }
}
 
Example 24
Project: camunda-bpm-reactor   File: Environment.java   Source Code and License 5 votes vote down vote up
public RoundRobinSupplier(int poolsize,
                          String name,
                          int bufferSize,
                          Consumer<Throwable> errorHandler,
                          ProducerType producerType,
                          WaitStrategy waitStrategy) {
  this.poolsize = poolsize;
  this.name = name;
  this.bufferSize = bufferSize;
  this.errorHandler = errorHandler;
  this.producerType = producerType;
  this.waitStrategy = waitStrategy;
  dispatchers = new Dispatcher[poolsize];
  terminated = false;
}
 
Example 25
Project: jstorm-0.9.6.3-   File: Worker.java   Source Code and License 5 votes vote down vote up
private AsyncLoopThread startDispatchThread() {
	Map stormConf = workerData.getStormConf();

	int queue_size = Utils.getInt(
			stormConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE), 1024);
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance("Dispatch", ProducerType.MULTI,
			queue_size, waitStrategy);
	// stop  consumerStarted
	//recvQueue.consumerStarted();

	IContext context = workerData.getContext();
	String topologyId = workerData.getTopologyId();

	IConnection recvConnection = context.bind(topologyId,
			workerData.getPort());
	recvConnection.registerQueue(recvQueue);

	RunnableCallback recvDispather = new VirtualPortDispatch(workerData,
			recvConnection, recvQueue);

	AsyncLoopThread vthread = new AsyncLoopThread(recvDispather, false,
			Thread.MAX_PRIORITY, false);

	return vthread;
}
 
Example 26
Project: jstorm-0.9.6.3-   File: Task.java   Source Code and License 5 votes vote down vote up
public DisruptorQueue registerDisruptorQueue() {
	int queueSize = JStormUtils.parseInt(
			stormConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE), 256);
	
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue queue = DisruptorQueue.mkInstance("TaskDeserialize", ProducerType.SINGLE,
			queueSize, waitStrategy);

	deserializeQueues.put(taskid, queue);

	return queue;
}
 
Example 27
Project: jstorm-0.9.6.3-   File: TaskTransfer.java   Source Code and License 5 votes vote down vote up
public TaskTransfer(String taskName, 
		KryoTupleSerializer serializer, TaskStatus taskStatus,
		WorkerData workerData) {
	this.taskName = taskName;
	this.serializer = serializer;
	this.taskStatus = taskStatus;
	this.storm_conf = workerData.getConf();
	this.transferQueue = workerData.getTransferQueue();
	this.innerTaskTransfer = workerData.getInnerTaskTransfer();

	int queue_size = Utils.getInt(storm_conf
			.get(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE));
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	this.serializeQueue = DisruptorQueue.mkInstance(taskName, ProducerType.MULTI, 
			queue_size, waitStrategy);
	this.serializeQueue.consumerStarted();
	
	String taskId = taskName.substring(taskName.indexOf(":") + 1);
	Metrics.registerQueue(taskName, MetricDef.SERIALIZE_QUEUE, serializeQueue, taskId, Metrics.MetricType.TASK);
	timer = Metrics.registerTimer(taskName, MetricDef.SERIALIZE_TIME, taskId, Metrics.MetricType.TASK); 

	serializeThread = new AsyncLoopThread(new TransferRunnable());
	LOG.info("Successfully start TaskTransfer thread");

}
 
Example 28
Project: jstorm-0.9.6.3-   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_small_message() {
	System.out.println("!!!!!!!!Start test_small_message !!!!!!!!!!!");
	String req_msg = "Aloha is the most Hawaiian word.";

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	System.out.println("!!!!!!!!!!!!!!!!!!Test one time!!!!!!!!!!!!!!!!!");

	server.close();
	client.close();
	

	System.out.println("!!!!!!!!!!!!End test_small_message!!!!!!!!!!!!!");
}
 
Example 29
Project: jstorm-0.9.6.3-   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_large_msg() {
	System.out.println("!!!!!!!!!!start larget message test!!!!!!!!");
	String req_msg = setupLargMsg();
	System.out.println("!!!!Finish batch data, size:" + req_msg.length()
			+ "!!!!");

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	client.close();
	server.close();
	System.out.println("!!!!!!!!!!End larget message test!!!!!!!!");
}
 
Example 30
Project: jstorm-0.9.6.3-   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_server_delay() throws InterruptedException {
	System.out.println("!!!!!!!!!!Start delay message test!!!!!!!!");
	String req_msg = setupLargMsg();

	
	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);
	Thread.sleep(1000);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	server.close();
	client.close();
	System.out.println("!!!!!!!!!!End delay message test!!!!!!!!");
}
 
Example 31
Project: jstorm-0.9.6.3-   File: MultiProducerSequencer.java   Source Code and License 5 votes vote down vote up
/**
 * Construct a Sequencer with the selected wait strategy and buffer size.
 *
 * @param bufferSize the size of the buffer that this will sequence over.
 * @param waitStrategy for those waiting on sequences.
 */
public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy)
{
    super(bufferSize, waitStrategy);
    availableBuffer = new int[bufferSize];
    indexMask = bufferSize - 1;
    indexShift = Util.log2(bufferSize);
    initialiseAvailableBuffer();
}
 
Example 32
Project: jstorm-0.9.6.3-   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public static DisruptorQueue mkInstance(String queueName,
		ProducerType producerType, int bufferSize, WaitStrategy wait) {
	if (CAPACITY_LIMITED == true) {
		return new DisruptorQueueImpl(queueName, producerType, bufferSize,
				wait);
	} else {
		return new DisruptorWrapBlockingQueue(queueName, producerType,
				bufferSize, wait);
	}
}
 
Example 33
Project: learn_jstorm   File: Worker.java   Source Code and License 5 votes vote down vote up
private AsyncLoopThread startDispatchThread() {
	Map stormConf = workerData.getStormConf();

	int queue_size = Utils.getInt(
			stormConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE), 1024);
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance("Dispatch", ProducerType.MULTI,
			queue_size, waitStrategy);
	// stop  consumerStarted
	//recvQueue.consumerStarted();

	IContext context = workerData.getContext();
	String topologyId = workerData.getTopologyId();

	IConnection recvConnection = context.bind(topologyId,
			workerData.getPort());
	recvConnection.registerQueue(recvQueue);

	RunnableCallback recvDispather = new VirtualPortDispatch(workerData,
			recvConnection, recvQueue);

	AsyncLoopThread vthread = new AsyncLoopThread(recvDispather, false,
			Thread.MAX_PRIORITY, false);

	return vthread;
}
 
Example 34
Project: learn_jstorm   File: Task.java   Source Code and License 5 votes vote down vote up
public DisruptorQueue registerDisruptorQueue() {
	int queueSize = JStormUtils.parseInt(
			stormConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE), 256);
	
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue queue = DisruptorQueue.mkInstance("TaskDeserialize", ProducerType.SINGLE,
			queueSize, waitStrategy);

	deserializeQueues.put(taskid, queue);

	return queue;
}
 
Example 35
Project: learn_jstorm   File: TaskTransfer.java   Source Code and License 5 votes vote down vote up
public TaskTransfer(String taskName, 
		KryoTupleSerializer serializer, TaskStatus taskStatus,
		WorkerData workerData) {
	this.taskName = taskName;
	this.serializer = serializer;
	this.taskStatus = taskStatus;
	this.storm_conf = workerData.getConf();
	this.transferQueue = workerData.getTransferQueue();
	this.innerTaskTransfer = workerData.getInnerTaskTransfer();

	int queue_size = Utils.getInt(storm_conf
			.get(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE));
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	this.serializeQueue = DisruptorQueue.mkInstance(taskName, ProducerType.MULTI, 
			queue_size, waitStrategy);
	this.serializeQueue.consumerStarted();
	
	String taskId = taskName.substring(taskName.indexOf(":") + 1);
	Metrics.registerQueue(taskName, MetricDef.SERIALIZE_QUEUE, serializeQueue, taskId, Metrics.MetricType.TASK);
	timer = Metrics.registerTimer(taskName, MetricDef.SERIALIZE_TIME, taskId, Metrics.MetricType.TASK); 

	serializeThread = new AsyncLoopThread(new TransferRunnable());
	LOG.info("Successfully start TaskTransfer thread");

}
 
Example 36
Project: learn_jstorm   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_small_message() {
	System.out.println("!!!!!!!!Start test_small_message !!!!!!!!!!!");
	String req_msg = "Aloha is the most Hawaiian word.";

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	System.out.println("!!!!!!!!!!!!!!!!!!Test one time!!!!!!!!!!!!!!!!!");

	server.close();
	client.close();
	

	System.out.println("!!!!!!!!!!!!End test_small_message!!!!!!!!!!!!!");
}
 
Example 37
Project: learn_jstorm   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_large_msg() {
	System.out.println("!!!!!!!!!!start larget message test!!!!!!!!");
	String req_msg = setupLargMsg();
	System.out.println("!!!!Finish batch data, size:" + req_msg.length()
			+ "!!!!");

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	client.close();
	server.close();
	System.out.println("!!!!!!!!!!End larget message test!!!!!!!!");
}
 
Example 38
Project: learn_jstorm   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_server_delay() throws InterruptedException {
	System.out.println("!!!!!!!!!!Start delay message test!!!!!!!!");
	String req_msg = setupLargMsg();

	
	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);
	Thread.sleep(1000);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	server.close();
	client.close();
	System.out.println("!!!!!!!!!!End delay message test!!!!!!!!");
}
 
Example 39
Project: learn_jstorm   File: MultiProducerSequencer.java   Source Code and License 5 votes vote down vote up
/**
 * Construct a Sequencer with the selected wait strategy and buffer size.
 *
 * @param bufferSize the size of the buffer that this will sequence over.
 * @param waitStrategy for those waiting on sequences.
 */
public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy)
{
    super(bufferSize, waitStrategy);
    availableBuffer = new int[bufferSize];
    indexMask = bufferSize - 1;
    indexShift = Util.log2(bufferSize);
    initialiseAvailableBuffer();
}
 
Example 40
Project: learn_jstorm   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public static DisruptorQueue mkInstance(String queueName,
		ProducerType producerType, int bufferSize, WaitStrategy wait) {
	if (CAPACITY_LIMITED == true) {
		return new DisruptorQueueImpl(queueName, producerType, bufferSize,
				wait);
	} else {
		return new DisruptorWrapBlockingQueue(queueName, producerType,
				bufferSize, wait);
	}
}
 
Example 41
Project: jstrom   File: NettyClientSync.java   Source Code and License 5 votes vote down vote up
@SuppressWarnings("rawtypes")
NettyClientSync(Map storm_conf, ChannelFactory factory, ScheduledExecutorService scheduler, String host, int port, ReconnectRunnable reconnector) {
    super(storm_conf, factory, scheduler, host, port, reconnector);

    batchQueue = new ConcurrentLinkedQueue<MessageBatch>();

    WaitStrategy waitStrategy = (WaitStrategy) Utils.newInstance((String) storm_conf.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));

    disruptorQueue = DisruptorQueue.mkInstance(name, ProducerType.MULTI, MAX_SEND_PENDING * 8, waitStrategy);
    disruptorQueue.consumerStarted();

    if (connectMyself == false) {
        registerSyncMetrics();
    }

    Runnable trigger = new Runnable() {
        @Override
        public void run() {
            trigger();
        }
    };

    scheduler.scheduleAtFixedRate(trigger, 10, 1, TimeUnit.SECONDS);

    /**
     * In sync mode, it can't directly use common factory, it will occur problem when client close and restart
     */
    ThreadFactory bossFactory = new NettyRenameThreadFactory(MetricDef.NETTY_CLI + JStormServerUtils.getName(host, port) + "-boss");
    bossExecutor = Executors.newCachedThreadPool(bossFactory);
    ThreadFactory workerFactory = new NettyRenameThreadFactory(MetricDef.NETTY_CLI + JStormServerUtils.getName(host, port) + "-worker");
    workerExecutor = Executors.newCachedThreadPool(workerFactory);

    clientChannelFactory = new NioClientSocketChannelFactory(bossExecutor, workerExecutor, 1);

    start();

    LOG.info(this.toString());
}
 
Example 42
Project: jstrom   File: Worker.java   Source Code and License 5 votes vote down vote up
@Deprecated
private DisruptorQueue startDispatchDisruptor() {
    Map stormConf = workerData.getStormConf();

    int queue_size = Utils.getInt(stormConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE), 1024);
    WaitStrategy waitStrategy = (WaitStrategy) JStormUtils.createDisruptorWaitStrategy(stormConf);
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("Dispatch", ProducerType.MULTI, queue_size, waitStrategy);
    // stop consumerStarted
    recvQueue.consumerStarted();

    return recvQueue;
}
 
Example 43
Project: jstrom   File: TaskReceiver.java   Source Code and License 5 votes vote down vote up
public TaskReceiver(Task task, int taskId, Map stormConf, TopologyContext topologyContext, Map<Integer, DisruptorQueue> innerTaskTransfer,
        TaskStatus taskStatus, String taskName) {
    this.task = task;
    this.taskId = taskId;
    this.idStr = taskName;

    this.topologyContext = topologyContext;
    this.innerTaskTransfer = innerTaskTransfer;

    this.taskStatus = taskStatus;

    this.isDebugRecv = ConfigExtension.isTopologyDebugRecvTuple(stormConf);

    int queueSize = JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE), 256);

    WaitStrategy waitStrategy = (WaitStrategy) JStormUtils.createDisruptorWaitStrategy(stormConf);
    this.deserializeQueue = DisruptorQueue.mkInstance("TaskDeserialize", ProducerType.MULTI, queueSize, waitStrategy);
    setDeserializeThread();
    this.deserializer = new KryoTupleDeserializer(stormConf, topologyContext);

    String topologyId = topologyContext.getTopologyId();
    String component = topologyContext.getThisComponentId();

    deserializeTimer =
            (AsmHistogram) JStormMetrics.registerTaskMetric(
                    MetricUtils.taskMetricName(topologyId, component, taskId, MetricDef.DESERIALIZE_TIME, MetricType.HISTOGRAM), new AsmHistogram());

    QueueGauge deserializeQueueGauge = new QueueGauge(deserializeQueue, idStr, MetricDef.DESERIALIZE_QUEUE);
    JStormMetrics.registerTaskMetric(MetricUtils.taskMetricName(topologyId, component, taskId, MetricDef.DESERIALIZE_QUEUE, MetricType.GAUGE),
            new AsmGauge(deserializeQueueGauge));
    JStormHealthCheck.registerTaskHealthCheck(taskId, MetricDef.DESERIALIZE_QUEUE, deserializeQueueGauge);
}
 
Example 44
Project: jstrom   File: RingBuffer.java   Source Code and License 5 votes vote down vote up
/**
 * Create a new Ring Buffer with the specified producer type (SINGLE or MULTI)
 * 
 * @param producerType producer type to use {@link ProducerType}.
 * @param factory used to create events within the ring buffer.
 * @param bufferSize number of elements to create within the ring buffer.
 * @param waitStrategy used to determine how to wait for new elements to become available.
 * @throws IllegalArgumentException if bufferSize is less than 1 or not a power of 2
 */
public static <E> RingBuffer<E> create(ProducerType producerType, EventFactory<E> factory, int bufferSize, WaitStrategy waitStrategy) {
    switch (producerType) {
    case SINGLE:
        return createSingleProducer(factory, bufferSize, waitStrategy);
    case MULTI:
        return createMultiProducer(factory, bufferSize, waitStrategy);
    default:
        throw new IllegalStateException(producerType.toString());
    }
}
 
Example 45
Project: jstrom   File: MultiProducerSequencer.java   Source Code and License 5 votes vote down vote up
/**
 * Construct a Sequencer with the selected wait strategy and buffer size.
 * 
 * @param bufferSize the size of the buffer that this will sequence over.
 * @param waitStrategy for those waiting on sequences.
 */
public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy) {
    super(bufferSize, waitStrategy);
    availableBuffer = new int[bufferSize];
    indexMask = bufferSize - 1;
    indexShift = Util.log2(bufferSize);
    initialiseAvailableBuffer();
}
 
Example 46
Project: jstrom   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public static DisruptorQueue mkInstance(String queueName, ProducerType producerType, int bufferSize, WaitStrategy wait) {
    if (CAPACITY_LIMITED == true) {
        return new DisruptorQueueImpl(queueName, producerType, bufferSize, wait);
    } else {
        return new DisruptorWrapBlockingQueue(queueName, producerType, bufferSize, wait);
    }
}
 
Example 47
Project: jstrom   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
private IConnection initNettyServer(int port) {
    ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    IConnection server = context.bind(null, port, deserializeQueues);

    WaitStrategy waitStrategy = (WaitStrategy) JStormUtils.createDisruptorWaitStrategy(storm_conf);
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
    server.registerQueue(task, recvQueue);

    return server;
}
 
Example 48
Project: Tstream   File: MultiProducerSequencer.java   Source Code and License 5 votes vote down vote up
/**
 * Construct a Sequencer with the selected wait strategy and buffer size.
 *
 * @param bufferSize the size of the buffer that this will sequence over.
 * @param waitStrategy for those waiting on sequences.
 */
public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy)
{
    super(bufferSize, waitStrategy);
    availableBuffer = new int[bufferSize];
    indexMask = bufferSize - 1;
    indexShift = Util.log2(bufferSize);
    initialiseAvailableBuffer();
}
 
Example 49
Project: Tstream   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public static DisruptorQueue mkInstance(String queueName,
		ProducerType producerType, int bufferSize, WaitStrategy wait) {
	if (CAPACITY_LIMITED == true) {
		return new DisruptorQueueImpl(queueName, producerType, bufferSize,
				wait);
	} else {
		return new DisruptorWrapBlockingQueue(queueName, producerType,
				bufferSize, wait);
	}
}
 
Example 50
Project: Tstream   File: Worker.java   Source Code and License 5 votes vote down vote up
private AsyncLoopThread startDispatchThread() {
	Map stormConf = workerData.getStormConf();

	int queue_size = Utils.getInt(
			stormConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE), 1024);
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance("Dispatch", ProducerType.MULTI,
			queue_size, waitStrategy);
	// stop  consumerStarted
	//recvQueue.consumerStarted();

	IContext context = workerData.getContext();
	String topologyId = workerData.getTopologyId();

	IConnection recvConnection = context.bind(topologyId,
			workerData.getPort());
	recvConnection.registerQueue(recvQueue);

	RunnableCallback recvDispather = new VirtualPortDispatch(workerData,
			recvConnection, recvQueue);

	AsyncLoopThread vthread = new AsyncLoopThread(recvDispather, false,
			Thread.MAX_PRIORITY, false);

	return vthread;
}
 
Example 51
Project: Tstream   File: Task.java   Source Code and License 5 votes vote down vote up
public DisruptorQueue registerDisruptorQueue() {
	int queueSize = JStormUtils.parseInt(
			stormConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE), 256);
	
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) stormConf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue queue = DisruptorQueue.mkInstance("TaskDeserialize", ProducerType.SINGLE,
			queueSize, waitStrategy);

	deserializeQueues.put(taskid, queue);

	return queue;
}
 
Example 52
Project: Tstream   File: TaskTransfer.java   Source Code and License 5 votes vote down vote up
public TaskTransfer(String taskName, 
		KryoTupleSerializer serializer, TaskStatus taskStatus,
		WorkerData workerData) {
	this.taskName = taskName;
	this.serializer = serializer;
	this.taskStatus = taskStatus;
	this.storm_conf = workerData.getConf();
	this.transferQueue = workerData.getTransferQueue();
	this.innerTaskTransfer = workerData.getInnerTaskTransfer();

	int queue_size = Utils.getInt(storm_conf
			.get(Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE));
	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	this.serializeQueue = DisruptorQueue.mkInstance(taskName, ProducerType.MULTI, 
			queue_size, waitStrategy);
	this.serializeQueue.consumerStarted();
	
	String taskId = taskName.substring(taskName.indexOf(":") + 1);
	Metrics.registerQueue(taskName, MetricDef.SERIALIZE_QUEUE, serializeQueue, taskId, Metrics.MetricType.TASK);
	timer = Metrics.registerTimer(taskName, MetricDef.SERIALIZE_TIME, taskId, Metrics.MetricType.TASK); 

	serializeThread = new AsyncLoopThread(new TransferRunnable());
	LOG.info("Successfully start TaskTransfer thread");

}
 
Example 53
Project: Tstream   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_small_message() {
	System.out.println("!!!!!!!!Start test_small_message !!!!!!!!!!!");
	String req_msg = "Aloha is the most Hawaiian word.";

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	System.out.println("!!!!!!!!!!!!!!!!!!Test one time!!!!!!!!!!!!!!!!!");

	server.close();
	client.close();
	

	System.out.println("!!!!!!!!!!!!End test_small_message!!!!!!!!!!!!!");
}
 
Example 54
Project: Tstream   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_large_msg() {
	System.out.println("!!!!!!!!!!start larget message test!!!!!!!!");
	String req_msg = setupLargMsg();
	System.out.println("!!!!Finish batch data, size:" + req_msg.length()
			+ "!!!!");

	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	client.close();
	server.close();
	System.out.println("!!!!!!!!!!End larget message test!!!!!!!!");
}
 
Example 55
Project: Tstream   File: NettyUnitTest.java   Source Code and License 5 votes vote down vote up
@Test
public void test_server_delay() throws InterruptedException {
	System.out.println("!!!!!!!!!!Start delay message test!!!!!!!!");
	String req_msg = setupLargMsg();

	
	IConnection server = null;
	IConnection client = null;

	server = context.bind(null, port);

	WaitStrategy waitStrategy = (WaitStrategy) Utils
			.newInstance((String) storm_conf
					.get(Config.TOPOLOGY_DISRUPTOR_WAIT_STRATEGY));
	DisruptorQueue recvQueue = DisruptorQueue.mkInstance(
			"NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy);
	server.registerQueue(recvQueue);

	client = context.connect(null, "localhost", port);

	List<TaskMessage> list = new ArrayList<TaskMessage>();
	TaskMessage message = new TaskMessage(task, req_msg.getBytes());
	list.add(message);

	LOG.info("Client send data");
	client.send(message);
	Thread.sleep(1000);

	TaskMessage recv = server.recv(0);
	Assert.assertEquals(req_msg, new String(recv.message()));

	server.close();
	client.close();
	System.out.println("!!!!!!!!!!End delay message test!!!!!!!!");
}
 
Example 56
Project: incubator-storm   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public DisruptorQueue(ClaimStrategy claim, WaitStrategy wait) {
    _buffer = new RingBuffer<MutableObject>(new ObjectEventFactory(), claim, wait);
    _consumer = new Sequence();
    _barrier = _buffer.newBarrier();
    _buffer.setGatingSequences(_consumer);
    if(claim instanceof SingleThreadedClaimStrategy) {
        consumerStartedFlag = true;
    }
}
 
Example 57
Project: akka-disruptor   File: DisruptorMessageQueue.java   Source Code and License 5 votes vote down vote up
DisruptorMessageQueue(int bufferSize, WaitStrategy waitStrategy) {
    ringBuffer = RingBuffer.createMultiProducer(EVENT_FACTORY, bufferSize, waitStrategy);
    ringBuffer.addGatingSequences(sequence);
    sequenceBarrier = ringBuffer.newBarrier();
    mask = bufferSize - 1;
    buffer = new Envelope[bufferSize];
}
 
Example 58
Project: storm-resa   File: TracedDisruptorQueue.java   Source Code and License 5 votes vote down vote up
public TracedDisruptorQueue(ClaimStrategy claim, WaitStrategy wait, Map<String, Object> conf) {
    super(claim, wait);
    float sampleRate = 0.05f;
    if (conf.containsKey("topology.queue.sample.rate")) {
        sampleRate = ((Number) conf.get("topology.queue.sample.rate")).floatValue();
    }
    this.sampleInterval = Math.max((int) (1 / sampleRate), 2);
}
 
Example 59
Project: storm-resa   File: DisruptorQueue.java   Source Code and License 5 votes vote down vote up
public DisruptorQueue(ClaimStrategy claim, WaitStrategy wait) {
    _buffer = new RingBuffer<MutableObject>(new ObjectEventFactory(), claim, wait);
    _consumer = new Sequence();
    _barrier = _buffer.newBarrier();
    _buffer.setGatingSequences(_consumer);
    if(claim instanceof SingleThreadedClaimStrategy) {
        consumerStartedFlag = true;
    }
}
 
Example 60
Project: logback-ext   File: WaitStrategyFactory.java   Source Code and License 5 votes vote down vote up
public static WaitStrategy createFromType(String name) {
    if ("BusySpin".equalsIgnoreCase(name)) {
        return new BusySpinWaitStrategy();
    } else if ("Blocking".equalsIgnoreCase(name)) {
        return new BlockingWaitStrategy();
    } else if ("Yielding".equalsIgnoreCase(name)) {
        return new YieldingWaitStrategy();
    } else if ("Sleeping".equalsIgnoreCase(name)) {
        return new SleepingWaitStrategy();
    } else {
        throw new IllegalArgumentException("Invalid or unsupported wait strategy type '" + name + "'");
    }
}