org.apache.reef.io.network.util.StringIdentifierFactory Java Examples

The following examples show how to use org.apache.reef.io.network.util.StringIdentifierFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataTransferTest.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
      .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
      .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
      .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #2
Source File: DataTransferTest.java    From nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
        .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
        .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
        .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #3
Source File: JobLauncher.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
/**
 * Get driver ncs configuration.
 *
 * @return driver ncs configuration.
 */
private static Configuration getDriverNcsConf() {
  return Configurations.merge(NameServerConfiguration.CONF.build(),
    LocalNameResolverConfiguration.CONF.build(),
    TANG.newConfigurationBuilder()
      .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
      .build());
}
 
Example #4
Source File: NemoDriver.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
private Configuration getExecutorNcsConfiguration() {
  return Tang.Factory.getTang().newConfigurationBuilder()
    .bindNamedParameter(NameResolverNameServerPort.class, Integer.toString(nameServer.getPort()))
    .bindNamedParameter(NameResolverNameServerAddr.class, localAddressProvider.getLocalAddress())
    .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
    .build();
}
 
Example #5
Source File: JobLauncher.java    From nemo with Apache License 2.0 5 votes vote down vote up
/**
 * Get driver ncs configuration.
 * @return driver ncs configuration.
 * @throws InjectionException exception while injection.
 */
private static Configuration getDriverNcsConf() throws InjectionException {
  return Configurations.merge(NameServerConfiguration.CONF.build(),
      LocalNameResolverConfiguration.CONF.build(),
      TANG.newConfigurationBuilder()
          .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
          .build());
}
 
Example #6
Source File: NemoDriver.java    From nemo with Apache License 2.0 5 votes vote down vote up
private Configuration getExecutorNcsConfiguration() {
  return Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(NameResolverNameServerPort.class, Integer.toString(nameServer.getPort()))
      .bindNamedParameter(NameResolverNameServerAddr.class, localAddressProvider.getLocalAddress())
      .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
      .build();
}
 
Example #7
Source File: TopologySerializerTest.java    From reef with Apache License 2.0 5 votes vote down vote up
@Test
public void testEncodeDecode() {
  final IdentifierFactory ifac = new StringIdentifierFactory();

  // create a topology: Task-0[Task-1[Task-3], Task-2]
  final TaskNode rootNode = new TaskNodeImpl(null, null, null, "Task-0", null, true);
  final TaskNode childNode1 = new TaskNodeImpl(null, null, null, "Task-1", null, false);
  final TaskNode childNode2 = new TaskNodeImpl(null, null, null, "Task-2", null, false);
  final TaskNode childNode3 = new TaskNodeImpl(null, null, null, "Task-3", null, false);
  rootNode.addChild(childNode1);
  rootNode.addChild(childNode2);
  childNode1.addChild(childNode3);

  final Pair<TopologySimpleNode, List<Identifier>> retPair =
      TopologySerializer.decode(TopologySerializer.encode(rootNode), ifac);

  // check topology is recovered
  assertEquals(retPair.getFirst().getTaskId(), "Task-0");
  for (final TopologySimpleNode child : retPair.getFirst().getChildren()) {
    if (child.getTaskId().equals("Task-1")) {
      for (final TopologySimpleNode childchild : child.getChildren()) {
        assertEquals(childchild.getTaskId(), "Task-3");
      }
    } else {
      assertTrue(child.getTaskId().equals("Task-2"));
    }
  }

  // check identifier list contains [Task-0, Task-1, Task-2, Task-3] and nothing else
  assertTrue(retPair.getSecond().contains(ifac.getNewInstance("Task-0")));
  assertTrue(retPair.getSecond().contains(ifac.getNewInstance("Task-1")));
  assertTrue(retPair.getSecond().contains(ifac.getNewInstance("Task-2")));
  assertTrue(retPair.getSecond().contains(ifac.getNewInstance("Task-3")));
  assertEquals(retPair.getSecond().size(), 4);
}
 
Example #8
Source File: NameClientTest.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Test method for {@link org.apache.reef.io.network.naming.NameClient#close()}.
 *
 * @throws Exception
 */
@Test
public final void testClose() throws Exception {
  final String localAddress = localAddressProvider.getLocalAddress();
  final IdentifierFactory factory = new StringIdentifierFactory();
  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int serverPort = server.getPort();
    final Configuration nameResolverConf = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, serverPort)
        .set(NameResolverConfiguration.CACHE_TIMEOUT, 10000)
        .set(NameResolverConfiguration.RETRY_TIMEOUT, RETRY_TIMEOUT)
        .set(NameResolverConfiguration.RETRY_COUNT, RETRY_COUNT)
        .build();

    try (NameResolver client =
             Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameClient.class)) {
      final Identifier id = factory.getNewInstance("Task1");
      client.register(id, new InetSocketAddress(localAddress, 7001));
      client.unregister(id);
      Thread.sleep(100);
    }
  }
}
 
Example #9
Source File: LocalNameResolverTest.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Test method for {@link org.apache.reef.io.network.naming.LocalNameResolverImpl#close()}.
 *
 * @throws Exception
 */
@Test
public final void testClose() throws Exception {
  final String localAddress = localAddressProvider.getLocalAddress();
  final IdentifierFactory factory = new StringIdentifierFactory();
  try (NameResolver resolver = Tang.Factory.getTang().newInjector(LocalNameResolverConfiguration.CONF
      .set(LocalNameResolverConfiguration.CACHE_TIMEOUT, 10000)
      .build()).getInstance(NameResolver.class)) {
    final Identifier id = factory.getNewInstance("Task1");
    resolver.register(id, new InetSocketAddress(localAddress, 7001));
    resolver.unregister(id);
    Thread.sleep(100);
  }
}
 
Example #10
Source File: LocalNameResolverTest.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Test method for {@link org.apache.reef.io.network.naming.LocalNameResolverImpl#lookup(Identifier id)}.
 * To check caching behavior with expireAfterAccess & expireAfterWrite
 * Changing NameCache's pattern to expireAfterAccess causes this test to fail
 *
 * @throws Exception
 */
@Test
public final void testLookup() throws Exception {
  final IdentifierFactory factory = new StringIdentifierFactory();
  final String localAddress = localAddressProvider.getLocalAddress();
  try (NameResolver resolver = Tang.Factory.getTang().newInjector(LocalNameResolverConfiguration.CONF
      .set(LocalNameResolverConfiguration.CACHE_TIMEOUT, 150)
      .build()).getInstance(NameResolver.class)) {
    final Identifier id = factory.getNewInstance("Task1");
    final InetSocketAddress socketAddr = new InetSocketAddress(localAddress, 7001);
    resolver.register(id, socketAddr);
    InetSocketAddress lookupAddr = resolver.lookup(id); // caches the entry
    Assert.assertTrue(socketAddr.equals(lookupAddr));
    resolver.unregister(id);
    Thread.sleep(100);
    try {
      lookupAddr = resolver.lookup(id);
      Thread.sleep(100);
      //With expireAfterAccess, the previous lookup would reset expiry to 150ms
      //more and 100ms wait will not expire the item and will return the cached value
      //With expireAfterWrite, the extra wait of 100 ms will expire the item
      //resulting in NamingException and the test passes
      lookupAddr = resolver.lookup(id);
      Assert.assertNull("resolver.lookup(id)", lookupAddr);
    } catch (final Exception e) {
      if (e instanceof ExecutionException) {
        Assert.assertTrue("Execution Exception cause is instanceof NamingException",
            e.getCause() instanceof NamingException);
      } else {
        throw e;
      }
    }
  }
}
 
Example #11
Source File: ScatterCodecTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * Test that {@code ScatterEncoder} and {@code ScatterDecoder} function correctly.
 * Create a small topology of 4 nodes and simulate a scatter operation.
 */
@Test
public void testEncodeDecode() {
  final IdentifierFactory ifac = new StringIdentifierFactory();
  final Codec<Integer> codec = new SerializableCodec<>();

  final List<Integer> elements = new LinkedList<>();
  for (int element = 0; element < 400; element++) {
    elements.add(element);
  }

  final List<Integer> counts = new LinkedList<>();
  final List<Identifier> taskOrder = new LinkedList<>();
  for (int index = 0; index < 4; index++) {
    counts.add(100);
    taskOrder.add(ifac.getNewInstance("Task-" + index));
  }

  final TopologySimpleNode rootNode = new TopologySimpleNode("Task-0");
  final TopologySimpleNode childNode1 = new TopologySimpleNode("Task-1");
  final TopologySimpleNode childNode2 = new TopologySimpleNode("Task-2");
  final TopologySimpleNode childNode3 = new TopologySimpleNode("Task-3");
  rootNode.addChild(childNode1);
  rootNode.addChild(childNode2);
  childNode1.addChild(childNode3);

  final CommunicationGroupServiceClient mockCommGroupClient = mock(CommunicationGroupServiceClient.class);
  when(mockCommGroupClient.getTopologySimpleNodeRoot()).thenReturn(rootNode);
  final ScatterEncoder scatterEncoder = new ScatterEncoder(mockCommGroupClient);
  final ScatterDecoder scatterDecoder = new ScatterDecoder();

  final Map<String, byte[]> encodedDataMap = scatterEncoder.encode(elements, counts, taskOrder, codec);

  // check msg correctness for childNode1 (Task-1)
  final ScatterData childNode1Data = scatterDecoder.decode(encodedDataMap.get(childNode1.getTaskId()));
  for (int index = 0; index < 100; index++) {
    assertTrue(index + 100 == codec.decode(childNode1Data.getMyData()[index]));
  }
  assertTrue(childNode1Data.getChildrenData().containsKey("Task-3"));
  assertEquals(childNode1Data.getChildrenData().size(), 1);

  // check msg correctness for childNode2 (Task-2)
  final ScatterData childNode2Data = scatterDecoder.decode(encodedDataMap.get(childNode2.getTaskId()));
  for (int index = 0; index < 100; index++) {
    assertTrue(index + 200 == codec.decode(childNode2Data.getMyData()[index]));
  }
  assertTrue(childNode2Data.getChildrenData().isEmpty());
}
 
Example #12
Source File: NameClientTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * Test method for {@link org.apache.reef.io.network.naming.NameClient#lookup()}.
 * To check caching behavior with expireAfterAccess & expireAfterWrite
 * Changing NameCache's pattern to expireAfterAccess causes this test to fail
 *
 * @throws Exception
 */
@Test
public final void testLookup() throws Exception {
  final String localAddress = localAddressProvider.getLocalAddress();
  final IdentifierFactory factory = new StringIdentifierFactory();
  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int serverPort = server.getPort();
    final Configuration nameResolverConf = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, serverPort)
        .set(NameResolverConfiguration.CACHE_TIMEOUT, 150)
        .set(NameResolverConfiguration.RETRY_TIMEOUT, RETRY_TIMEOUT)
        .set(NameResolverConfiguration.RETRY_COUNT, RETRY_COUNT)
        .build();

    try (NameResolver client =
             Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameClient.class)) {
      final Identifier id = factory.getNewInstance("Task1");
      client.register(id, new InetSocketAddress(localAddress, 7001));
      client.lookup(id); // caches the entry
      client.unregister(id);
      Thread.sleep(100);
      try {
        InetSocketAddress addr = client.lookup(id);
        Thread.sleep(100);
        //With expireAfterAccess, the previous lookup would reset expiry to 150ms
        //more and 100ms wait will not expire the item and will return the cached value
        //With expireAfterWrite, the extra wait of 100 ms will expire the item
        //resulting in NamingException and the test passes
        addr = client.lookup(id);
        Assert.assertNull("client.lookup(id)", addr);
      } catch (final Exception e) {
        if (e instanceof ExecutionException) {
          Assert.assertTrue("Execution Exception cause is instanceof NamingException",
              e.getCause() instanceof NamingException);
        } else {
          throw e;
        }
      }
    }
  }
}
 
Example #13
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging test.
 */
@Test
public void testMessagingNetworkService() throws Exception {
  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int numMessages = 10;
    final Monitor monitor = new Monitor();

    // network service
    final String name2 = "task2";
    final String name1 = "task1";
    final Configuration nameResolverConf =
        Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
        .build())
        .build();

    final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

    LOG.log(Level.FINEST, "=== Test network service receiver start");
    LOG.log(Level.FINEST, "=== Test network service sender start");
    try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
      injector2.bindVolatileInstance(NameResolver.class, nameResolver);
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
          injector.getInstance(MessagingTransportFactory.class));
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
          new ExceptionHandler());

      final Injector injectorNs2 = injector2.forkInjector();
      injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name2, monitor, numMessages));
      final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

      final Injector injectorNs1 = injector2.forkInjector();
      injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name1, null, 0));
      final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

      ns2.registerId(factory.getNewInstance(name2));
      final int port2 = ns2.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

      ns1.registerId(factory.getNewInstance(name1));
      final int port1 = ns1.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

      final Identifier destId = factory.getNewInstance(name2);

      try (Connection<String> conn = ns1.newConnection(destId)) {
        conn.open();
        for (int count = 0; count < numMessages; ++count) {
          conn.write("hello! " + count);
        }
        monitor.mwait();

      } catch (final NetworkException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
}
 
Example #14
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {1, 16, 32, 64, 512, 64 * 1024, 1024 * 1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', size);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 
Example #15
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultithreadedSharedConnMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {2000}; // {1,16,32,64,512,64*1024,1024*1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final int numThreads = 2;
      final int totalNumMessages = numMessages * numThreads;
      final Monitor monitor = new Monitor();


      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, totalNumMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);

        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();

          final String message = StringUtils.repeat('1', size);
          final ExecutorService e = Executors.newCachedThreadPool();

          final long start = System.currentTimeMillis();
          for (int i = 0; i < numThreads; i++) {
            e.submit(new Runnable() {

              @Override
              public void run() {
                for (int i = 0; i < numMessages; i++) {
                  conn.write(message);
                }
              }
            });
          }


          e.shutdown();
          e.awaitTermination(30, TimeUnit.SECONDS);
          monitor.mwait();

          final long end = System.currentTimeMillis();
          final double runtime = ((double) end - start) / 1000;

          LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + totalNumMessages / runtime + 
              " bandwidth(bytes/s): " + ((double) totalNumMessages * 2 * size) / runtime); // x2 for unicode chars
        }
      }
    }
  }
}
 
Example #16
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceBatchingRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int batchSize = 1024 * 1024;
    final int[] messageSizes = {32, 64, 512};

    for (final int size : messageSizes) {
      final int numMessages = 300 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', batchSize);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        final long numAppMessages = numMessages * batchSize / size;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numAppMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numAppMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}