org.apache.reef.wake.remote.address.LocalAddressProvider Java Examples

The following examples show how to use org.apache.reef.wake.remote.address.LocalAddressProvider. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DriverRPCServer.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
/**
 * Runs the RPC server.
 * Specifically, creates a {@link NettyMessagingTransport} and binds it to a listening port.
 */
public void run() {
  // Calling 'run' multiple times is considered invalid, since it will override state variables like
  // 'transport', and 'host'.
  ensureServerState(false);
  try {
    final Injector injector = Tang.Factory.getTang().newInjector();
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    host = localAddressProvider.getLocalAddress();
    injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, host);
    injector.bindVolatileParameter(RemoteConfiguration.Port.class, 0);
    injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class,
      new SyncStage<>(new ServerEventHandler()));
    transport = injector.getInstance(NettyMessagingTransport.class);
    LOG.info("DriverRPCServer running at {}", transport.getListeningPort());
    isRunning = true;
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #2
Source File: NameLookupClient.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
  * Constructs a naming lookup client.
  *
  * @param serverAddr a server address
  * @param serverPort a server port number
  * @param timeout    request timeout in ms
  * @param factory    an identifier factory
  * @param tpFactory  a transport factory
  */
@Inject
private NameLookupClient(
          @Parameter(NameResolverNameServerAddr.class) final String serverAddr,
          @Parameter(NameResolverNameServerPort.class) final int serverPort,
          @Parameter(NameResolverCacheTimeout.class) final long timeout,
          @Parameter(NameResolverIdentifierFactory.class) final IdentifierFactory factory,
          @Parameter(NameResolverRetryCount.class) final int retryCount,
          @Parameter(NameResolverRetryTimeout.class) final int retryTimeout,
          final LocalAddressProvider localAddressProvider,
          final TransportFactory tpFactory) {
  this.serverSocketAddr = new InetSocketAddress(serverAddr, serverPort);
  this.timeout = timeout;
  this.cache = new NameCache(timeout);
  this.codec = NamingCodecFactory.createLookupCodec(factory);
  this.replyQueue = new LinkedBlockingQueue<>();

  this.transport = tpFactory.newInstance(localAddressProvider.getLocalAddress(), 0,
          new SyncStage<>(new NamingLookupClientHandler(
                  new NamingLookupResponseHandler(this.replyQueue), this.codec)),
          null, retryCount, retryTimeout);

  this.retryCount = retryCount;
  this.retryTimeout = retryTimeout;
}
 
Example #3
Source File: NameRegistryClient.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs a naming registry client.
 *
 * @param serverAddr a name server address
 * @param serverPort a name server port
 * @param timeout    timeout in ms
 * @param factory    an identifier factory
 */
NameRegistryClient(final String serverAddr,
                          final int serverPort,
                          final long timeout,
                          final IdentifierFactory factory,
                          final LocalAddressProvider localAddressProvider) {

  this.serverSocketAddr = new InetSocketAddress(serverAddr, serverPort);
  this.timeout = timeout;
  this.codec = NamingCodecFactory.createRegistryCodec(factory);
  this.replyQueue = new LinkedBlockingQueue<>();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, localAddressProvider.getLocalAddress());
  injector.bindVolatileParameter(RemoteConfiguration.RemoteClientStage.class,
      new SyncStage<>(new NamingRegistryClientHandler(new NamingRegistryResponseHandler(replyQueue), codec)));

  try {
    this.transport = injector.getInstance(NettyMessagingTransport.class);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #4
Source File: DefaultRemoteManagerFactory.java    From reef with Apache License 2.0 6 votes vote down vote up
@Inject
private DefaultRemoteManagerFactory(
    @Parameter(RemoteConfiguration.MessageCodec.class) final Codec<?> codec,
    @Parameter(RemoteConfiguration.ErrorHandler.class) final EventHandler<Throwable> errorHandler,
    @Parameter(RemoteConfiguration.OrderingGuarantee.class) final boolean orderingGuarantee,
    @Parameter(RemoteConfiguration.NumberOfTries.class) final int numberOfTries,
    @Parameter(RemoteConfiguration.RetryTimeout.class) final int retryTimeout,
    final LocalAddressProvider localAddressProvider,
    final TransportFactory tpFactory,
    final TcpPortProvider tcpPortProvider) {

  this.codec = codec;
  this.errorHandler = errorHandler;
  this.orderingGuarantee = orderingGuarantee;
  this.numberOfTries = numberOfTries;
  this.retryTimeout = retryTimeout;
  this.localAddressProvider = localAddressProvider;
  this.transportFactory = tpFactory;
  this.tcpPortProvider = tcpPortProvider;
}
 
Example #5
Source File: DataTransferTest.java    From nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
        .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
        .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
        .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #6
Source File: GrpcMessageEnvironment.java    From nemo with Apache License 2.0 6 votes vote down vote up
@Inject
private GrpcMessageEnvironment(
    final LocalAddressProvider localAddressProvider,
    final NameResolver nameResolver,
    final IdentifierFactory idFactory,
    @Parameter(MessageParameters.SenderId.class) final String localSenderId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.grpcServer = new GrpcMessageServer(localAddressProvider, nameResolver, idFactory, localSenderId);

  try {
    this.grpcServer.start();
  } catch (final Exception e) {
    LOG.warn("Failed to start grpc server", e);
    throw new RuntimeException(e);
  }
}
 
Example #7
Source File: NemoDriver.java    From nemo with Apache License 2.0 6 votes vote down vote up
@Inject
private NemoDriver(final UserApplicationRunner userApplicationRunner,
                   final RuntimeMaster runtimeMaster,
                   final NameServer nameServer,
                   final LocalAddressProvider localAddressProvider,
                   final JobMessageObserver client,
                   @Parameter(JobConf.ExecutorJsonContents.class) final String resourceSpecificationString,
                   @Parameter(JobConf.JobId.class) final String jobId,
                   @Parameter(JobConf.FileDirectory.class) final String localDirectory,
                   @Parameter(JobConf.GlusterVolumeDirectory.class) final String glusterDirectory) {
  IdManager.setInDriver();
  this.userApplicationRunner = userApplicationRunner;
  this.runtimeMaster = runtimeMaster;
  this.nameServer = nameServer;
  this.localAddressProvider = localAddressProvider;
  this.resourceSpecificationString = resourceSpecificationString;
  this.jobId = jobId;
  this.localDirectory = localDirectory;
  this.glusterDirectory = glusterDirectory;
  this.client = client;
  this.handler = new RemoteClientMessageLoggingHandler(client);
}
 
Example #8
Source File: DataTransferTest.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
      .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
      .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
      .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #9
Source File: GrpcMessageEnvironment.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
@Inject
private GrpcMessageEnvironment(
  final LocalAddressProvider localAddressProvider,
  final NameResolver nameResolver,
  final IdentifierFactory idFactory,
  @Parameter(MessageParameters.SenderId.class) final String localSenderId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.grpcServer = new GrpcMessageServer(localAddressProvider, nameResolver, idFactory, localSenderId);
  this.localSenderId = localSenderId;

  try {
    this.grpcServer.start();
  } catch (final Exception e) {
    LOG.warn("Failed to start grpc server", e);
    throw new RuntimeException(e);
  }
}
 
Example #10
Source File: HttpServerImpl.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor of HttpServer that wraps Jetty Server.
 *
 * @param jettyHandler
 * @throws Exception
 */
@Inject
HttpServerImpl(@Parameter(RemoteConfiguration.HostAddress.class) final String hostAddress,
               final JettyHandler jettyHandler,
               final LocalAddressProvider addressProvider,
               final TcpPortProvider tcpPortProvider,
               final LoggingScopeFactory loggingScopeFactory) throws Exception {
  this.loggingScopeFactory = loggingScopeFactory;
  this.jettyHandler = jettyHandler;

  this.hostAddress = UNKNOWN_HOST_NAME.equals(hostAddress) ? addressProvider.getLocalAddress() : hostAddress;
  try (LoggingScope ls = this.loggingScopeFactory.httpServer()) {

    Server srv = null;
    int availablePort = -1;
    for (final int p : tcpPortProvider) {
      srv = tryPort(p);
      if (srv != null) {
        availablePort = p;
        break;
      }
    }

    if (srv  != null) {
      this.server = srv;
      this.port = availablePort;
      this.server.setHandler(jettyHandler);
      LOG.log(Level.INFO, "Jetty Server started with port: {0}", availablePort);
    } else {
      throw new RuntimeException("Could not find available port for http");
    }
  }
}
 
Example #11
Source File: JobDriver.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Job driver constructor.
 * All parameters are injected from TANG automatically.
 *
 * @param clock                      Wake clock to schedule and check up running jobs.
 * @param jobMessageObserver         is used to send messages back to the client.
 * @param evaluatorRequestor         is used to request Evaluators.
 * @param activeContextBridgeFactory
 */
@Inject
JobDriver(final Clock clock,
          final HttpServer httpServer,
          final NameServer nameServer,
          final JobMessageObserver jobMessageObserver,
          final EvaluatorRequestor evaluatorRequestor,
          final DriverStatusManager driverStatusManager,
          final LoggingScopeFactory loggingScopeFactory,
          final LocalAddressProvider localAddressProvider,
          final ActiveContextBridgeFactory activeContextBridgeFactory,
          final REEFFileNames reefFileNames,
          final AllocatedEvaluatorBridgeFactory allocatedEvaluatorBridgeFactory,
          final CLRProcessFactory clrProcessFactory,
          @Parameter(DefinedRuntimes.class) final Set<String> definedRuntimes) {
  this.clock = clock;
  this.httpServer = httpServer;
  this.jobMessageObserver = jobMessageObserver;
  this.evaluatorRequestor = evaluatorRequestor;
  this.nameServer = nameServer;
  this.driverStatusManager = driverStatusManager;
  this.activeContextBridgeFactory = activeContextBridgeFactory;
  this.allocatedEvaluatorBridgeFactory = allocatedEvaluatorBridgeFactory;
  this.nameServerInfo = localAddressProvider.getLocalAddress() + ":" + this.nameServer.getPort();
  this.loggingScopeFactory = loggingScopeFactory;
  this.reefFileNames = reefFileNames;
  this.localAddressProvider = localAddressProvider;
  this.clrProcessFactory = clrProcessFactory;
  this.definedRuntimes = definedRuntimes;
}
 
Example #12
Source File: DefaultRemoteManagerFactory.java    From reef with Apache License 2.0 5 votes vote down vote up
@Override
public <T> RemoteManager getInstance(final String newRmName,
                                     final String newHostAddress,
                                     final int newListeningPort,
                                     final Codec<T> newCodec,
                                     final EventHandler<Throwable> newErrorHandler,
                                     final boolean newOrderingGuarantee,
                                     final int newNumberOfTries,
                                     final int newRetryTimeout,
                                     final LocalAddressProvider newLocalAddressProvider,
                                     final TcpPortProvider newTcpPortProvider) {
  try {

    final Injector newInjector = injector.forkInjector();

    if (newHostAddress != null) {
      newInjector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, newHostAddress);
    }

    if (newListeningPort > 0) {
      newInjector.bindVolatileParameter(RemoteConfiguration.Port.class, newListeningPort);
    }

    newInjector.bindVolatileParameter(RemoteConfiguration.ManagerName.class, newRmName);
    newInjector.bindVolatileParameter(RemoteConfiguration.MessageCodec.class, newCodec);
    newInjector.bindVolatileParameter(RemoteConfiguration.ErrorHandler.class, newErrorHandler);
    newInjector.bindVolatileParameter(RemoteConfiguration.OrderingGuarantee.class, newOrderingGuarantee);
    newInjector.bindVolatileParameter(RemoteConfiguration.NumberOfTries.class, newNumberOfTries);
    newInjector.bindVolatileParameter(RemoteConfiguration.RetryTimeout.class, newRetryTimeout);
    newInjector.bindVolatileInstance(LocalAddressProvider.class, newLocalAddressProvider);
    newInjector.bindVolatileInstance(TransportFactory.class, this.transportFactory);
    newInjector.bindVolatileInstance(TcpPortProvider.class, newTcpPortProvider);

    return newInjector.getInstance(RemoteManager.class);

  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #13
Source File: NetworkTransport.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Sends and receives messages between the java bridge and C# bridge.
 * @param observer A multiobserver instance that will receive all incoming messages.
 */
@Inject
private NetworkTransport(
    final RemoteManagerFactory remoteManagerFactory,
    final LocalAddressProvider localAddressProvider,
    final ProtocolSerializer serializer,
    final InjectionFuture<MultiObserver> observer) {

  LOG.log(Level.FINE, "Java bridge network initializing");

  this.serializer = serializer;
  this.messageObserver = observer;

  this.remoteManager = remoteManagerFactory.getInstance(
      "JavaBridgeNetwork", localAddressProvider.getLocalAddress(), 0, new ByteCodec());

  // Get our address and port number.
  final RemoteIdentifier remoteIdentifier = this.remoteManager.getMyIdentifier();
  if (remoteIdentifier instanceof SocketRemoteIdentifier) {
    final SocketRemoteIdentifier socketIdentifier = (SocketRemoteIdentifier)remoteIdentifier;
    this.inetSocketAddress = socketIdentifier.getSocketAddress();
  } else {
    throw new RuntimeException("Identifier is not a SocketRemoteIdentifier: " + remoteIdentifier);
  }

  // Register as the message handler for any incoming messages.
  this.remoteManager.registerHandler(byte[].class, new LocalObserver());
}
 
Example #14
Source File: YarnContainerManager.java    From reef with Apache License 2.0 5 votes vote down vote up
@Inject
private YarnContainerManager(
    @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
    @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
    final YarnConfiguration yarnConf,
    final YarnProxyUser yarnProxyUser,
    final REEFEventHandlers reefEventHandlers,
    final Containers containers,
    final ApplicationMasterRegistration registration,
    final ContainerRequestCounter containerRequestCounter,
    final DriverStatusManager driverStatusManager,
    final REEFFileNames reefFileNames,
    final TrackingURLProvider trackingURLProvider,
    final LocalAddressProvider addressProvider,
    final RackNameFormatter rackNameFormatter,
    final InjectionFuture<ProgressProvider> progressProvider) throws IOException {

  this.reefEventHandlers = reefEventHandlers;
  this.driverStatusManager = driverStatusManager;

  this.containers = containers;
  this.registration = registration;
  this.containerRequestCounter = containerRequestCounter;
  this.yarnConf = yarnConf;
  this.yarnProxyUser = yarnProxyUser;
  this.rackNameFormatter = rackNameFormatter;

  this.trackingUrl = trackingURLProvider.getTrackingUrl();
  this.amRegistrationHost = addressProvider.getLocalAddress();

  this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this);
  this.nodeManager = new NMClientAsyncImpl(this);

  this.jobSubmissionDirectory = jobSubmissionDirectory;
  this.reefFileNames = reefFileNames;
  this.progressProvider = progressProvider;

  LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
      new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
 
Example #15
Source File: NemoDriver.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
@Inject
private NemoDriver(final UserApplicationRunner userApplicationRunner,
                   final RuntimeMaster runtimeMaster,
                   final NameServer nameServer,
                   final LocalAddressProvider localAddressProvider,
                   final JobMessageObserver client,
                   final ClientRPC clientRPC,
                   final DataPlaneConf dataPlaneConf,
                   @Parameter(JobConf.ExecutorJSONContents.class) final String resourceSpecificationString,
                   @Parameter(JobConf.BandwidthJSONContents.class) final String bandwidthString,
                   @Parameter(JobConf.JobId.class) final String jobId,
                   @Parameter(JobConf.FileDirectory.class) final String localDirectory,
                   @Parameter(JobConf.GlusterVolumeDirectory.class) final String glusterDirectory) {
  IdManager.setInDriver();
  this.userApplicationRunner = userApplicationRunner;
  this.runtimeMaster = runtimeMaster;
  this.nameServer = nameServer;
  this.localAddressProvider = localAddressProvider;
  this.resourceSpecificationString = resourceSpecificationString;
  this.jobId = jobId;
  this.localDirectory = localDirectory;
  this.glusterDirectory = glusterDirectory;
  this.handler = new RemoteClientMessageLoggingHandler(client);
  this.clientRPC = clientRPC;
  this.dataPlaneConf = dataPlaneConf;
  // TODO #69: Support job-wide execution property
  ResourceSitePass.setBandwidthSpecificationString(bandwidthString);
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.Notification, this::handleNotification);
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.LaunchDAG, message -> {
    startSchedulingUserDAG(message.getLaunchDAG().getDag());
    final Map<Serializable, Object> broadcastVars =
      SerializationUtils.deserialize(message.getLaunchDAG().getBroadcastVars().toByteArray());
    BroadcastManagerMaster.registerBroadcastVariablesFromClient(broadcastVars);
  });
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.DriverShutdown, message -> shutdown());
  // Send DriverStarted message to the client
  clientRPC.send(ControlMessage.DriverToClientMessage.newBuilder()
    .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
}
 
Example #16
Source File: AzureBatchDriverConfigurationProviderImpl.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles the Driver configuration.
 *
 * @param jobFolder the job folder.
 * @param clientRemoteId the client remote id.
 * @param jobId the job id.
 * @param applicationConfiguration the application configuration.
 * @return the Driver configuration.
 */
@Override
public Configuration getDriverConfiguration(final URI jobFolder,
                                            final String clientRemoteId,
                                            final String jobId,
                                            final Configuration applicationConfiguration) {
  ConfigurationModuleBuilder driverConfigurationBuilder = AzureBatchDriverConfiguration.CONF.getBuilder()
      .bindImplementation(CommandBuilder.class, this.commandBuilder.getClass());

  // If using docker containers, then use a different set of bindings
  if (this.containerRegistryProvider.isValid()) {
    driverConfigurationBuilder = driverConfigurationBuilder
        .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
        .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
  }

  final Configuration driverConfiguration = driverConfigurationBuilder.build()
      .set(AzureBatchDriverConfiguration.JOB_IDENTIFIER, jobId)
      .set(AzureBatchDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, clientRemoteId)
      .set(AzureBatchDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack)
      .set(AzureBatchDriverConfiguration.RUNTIME_NAME, RuntimeIdentifier.RUNTIME_NAME)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_URI, this.azureBatchAccountUri)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_ACCOUNT_NAME, this.azureBatchAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_BATCH_POOL_ID, this.azureBatchPoolId)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_ACCOUNT_NAME, this.azureStorageAccountName)
      .set(AzureBatchDriverConfiguration.AZURE_STORAGE_CONTAINER_NAME, this.azureStorageContainerName)
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_SERVER,
          this.containerRegistryProvider.getContainerRegistryServer())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_USERNAME,
          this.containerRegistryProvider.getContainerRegistryUsername())
      .set(AzureBatchDriverConfiguration.CONTAINER_REGISTRY_PASSWORD,
          this.containerRegistryProvider.getContainerRegistryPassword())
      .set(AzureBatchDriverConfiguration.CONTAINER_IMAGE_NAME,
          this.containerRegistryProvider.getContainerImageName())
      .setMultiple(AzureBatchDriverConfiguration.TCP_PORT_SET, this.tcpPortSet)
      .build();
  return Configurations.merge(driverConfiguration, applicationConfiguration);
}
 
Example #17
Source File: EvaluatorShimConfiguration.java    From reef with Apache License 2.0 5 votes vote down vote up
public static ConfigurationModule getConfigurationModule(final boolean includeContainerConfiguration) {
  ConfigurationModuleBuilder shimConfigurationBuilder = EvaluatorShimConfiguration.CONF.getBuilder();

  // If using docker containers, then use a different set of bindings
  if (includeContainerConfiguration) {
    shimConfigurationBuilder = shimConfigurationBuilder
        .bindImplementation(LocalAddressProvider.class, ContainerBasedLocalAddressProvider.class)
        .bindImplementation(TcpPortProvider.class, SetTcpPortProvider.class);
  }

  return shimConfigurationBuilder.build();
}
 
Example #18
Source File: NameClientTest.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Test method for {@link org.apache.reef.io.network.naming.NameClient#close()}.
 *
 * @throws Exception
 */
@Test
public final void testClose() throws Exception {
  final String localAddress = localAddressProvider.getLocalAddress();
  final IdentifierFactory factory = new StringIdentifierFactory();
  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int serverPort = server.getPort();
    final Configuration nameResolverConf = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, serverPort)
        .set(NameResolverConfiguration.CACHE_TIMEOUT, 10000)
        .set(NameResolverConfiguration.RETRY_TIMEOUT, RETRY_TIMEOUT)
        .set(NameResolverConfiguration.RETRY_COUNT, RETRY_COUNT)
        .build();

    try (NameResolver client =
             Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameClient.class)) {
      final Identifier id = factory.getNewInstance("Task1");
      client.register(id, new InetSocketAddress(localAddress, 7001));
      client.unregister(id);
      Thread.sleep(100);
    }
  }
}
 
Example #19
Source File: NamingTest.java    From reef with Apache License 2.0 5 votes vote down vote up
private static NameLookupClient getNewNameLookupClient(final String serverAddr,
                                                       final int serverPort,
                                                       final long timeout,
                                                       final int retryCount,
                                                       final int retryTimeout,
                                                       final Optional<LocalAddressProvider> localAddressProvider,
                                                       final Optional<IdentifierFactory> factory)
    throws InjectionException {


  final Configuration injectorConf = Tang.Factory.getTang().newConfigurationBuilder()
      .bindNamedParameter(NameResolverNameServerAddr.class, serverAddr)
      .bindNamedParameter(NameResolverNameServerPort.class, Integer.toString(serverPort))
      .bindNamedParameter(NameResolverCacheTimeout.class, Long.toString(timeout))
      .bindNamedParameter(NameResolverRetryCount.class, Integer.toString(retryCount))
      .bindNamedParameter(NameResolverRetryTimeout.class, Integer.toString(retryTimeout))
      .build();

  final Injector injector = Tang.Factory.getTang().newInjector(injectorConf);
  if (localAddressProvider.isPresent()) {
    injector.bindVolatileInstance(LocalAddressProvider.class, localAddressProvider.get());
  }

  if (factory.isPresent()) {
    injector.bindVolatileInstance(IdentifierFactory.class, factory.get());
  }

  return injector.getInstance(NameLookupClient.class);
}
 
Example #20
Source File: NameClient.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
   * Constructs a naming client.
   *
   * @param serverAddr a server address
   * @param serverPort a server port number
   * @param timeout timeout in ms
   * @param factory an identifier factory
   * @param retryCount the number of retries
   * @param retryTimeout retry timeout
   * @param localAddressProvider a local address provider
   * @param tpFactory transport factory
   */
@Inject
private NameClient(
    @Parameter(NameResolverNameServerAddr.class) final String serverAddr,
    @Parameter(NameResolverNameServerPort.class) final int serverPort,
    @Parameter(NameResolverCacheTimeout.class) final long timeout,
    @Parameter(NameResolverIdentifierFactory.class) final IdentifierFactory factory,
    @Parameter(NameResolverRetryCount.class) final int retryCount,
    @Parameter(NameResolverRetryTimeout.class) final int retryTimeout,
    final LocalAddressProvider localAddressProvider,
    final TransportFactory tpFactory) {

  final BlockingQueue<NamingLookupResponse> replyLookupQueue = new LinkedBlockingQueue<>();
  final BlockingQueue<NamingRegisterResponse> replyRegisterQueue = new LinkedBlockingQueue<>();
  final Codec<NamingMessage> codec = NamingCodecFactory.createFullCodec(factory);

  this.transport = tpFactory.newInstance(localAddressProvider.getLocalAddress(), 0,
      new SyncStage<>(new NamingClientEventHandler(
          new NamingResponseHandler(replyLookupQueue, replyRegisterQueue), codec)),
      null, retryCount, retryTimeout);

  this.lookupClient = new NameLookupClient(serverAddr, serverPort, timeout, factory,
      retryCount, retryTimeout, replyLookupQueue, this.transport);

  this.registryClient = new NameRegistryClient(serverAddr, serverPort, timeout,
      factory, replyRegisterQueue, this.transport);
}
 
Example #21
Source File: NameServerImpl.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * @param port    a listening port number
 * @param factory an identifier factory
 * @param localAddressProvider a local address provider
 * Constructs a name server
 */
@Inject
private NameServerImpl(
    @Parameter(RemoteConfiguration.HostAddress.class) final String hostAddress,
    @Parameter(NameServerParameters.NameServerPort.class) final int port,
    @Parameter(NameServerParameters.NameServerIdentifierFactory.class) final IdentifierFactory factory,
    final TcpPortProvider portProvider,
    final LocalAddressProvider localAddressProvider) {

  final Injector injector = Tang.Factory.getTang().newInjector();

  this.localAddressProvider = localAddressProvider;
  this.reefEventStateManager = null;
  final Codec<NamingMessage> codec = NamingCodecFactory.createFullCodec(factory);
  final EventHandler<NamingMessage> handler = createEventHandler(codec);

  String host = UNKNOWN_HOST_NAME.equals(hostAddress) ? localAddressProvider.getLocalAddress() : hostAddress;
  injector.bindVolatileParameter(RemoteConfiguration.HostAddress.class, host);
  injector.bindVolatileParameter(RemoteConfiguration.Port.class, port);
  injector.bindVolatileInstance(TcpPortProvider.class, portProvider);
  injector.bindVolatileParameter(RemoteConfiguration.RemoteServerStage.class,
      new SyncStage<>(new NamingServerHandler(handler, codec)));

  try {
    this.transport = injector.getInstance(NettyMessagingTransport.class);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }

  this.port = transport.getListeningPort();
  this.idToAddrMap = Collections.synchronizedMap(new HashMap<Identifier, InetSocketAddress>());

  LOG.log(Level.FINE, "NameServer starting, listening at port {0}", this.port);
}
 
Example #22
Source File: NetworkConnectionServiceTest.java    From reef with Apache License 2.0 5 votes vote down vote up
public NetworkConnectionServiceTest() throws InjectionException {
  localAddressProvider = Tang.Factory.getTang().newInjector().getInstance(LocalAddressProvider.class);
  localAddress = localAddressProvider.getLocalAddress();

  final IdentifierFactory idFac = new StringIdentifierFactory();
  this.groupCommClientId = idFac.getNewInstance("groupComm");
  this.shuffleClientId = idFac.getNewInstance("shuffle");
}
 
Example #23
Source File: GrpcMessageServer.java    From nemo with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor.
 * @param localAddressProvider local address provider.
 * @param nameResolver name resolver.
 * @param idFactory identifier factory.
 * @param localSenderId id of the local sender.
 */
GrpcMessageServer(final LocalAddressProvider localAddressProvider,
                  final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String localSenderId) {
  this.localAddressProvider = localAddressProvider;
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.localSenderId = localSenderId;
  this.listenerMap = new ConcurrentHashMap<>();
}
 
Example #24
Source File: ClientRPC.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
@Inject
private ClientRPC(final TransportFactory transportFactory,
                  final LocalAddressProvider localAddressProvider,
                  @Parameter(JobConf.ClientSideRPCServerHost.class) final String clientHost,
                  @Parameter(JobConf.ClientSideRPCServerPort.class) final int clientPort) {
  transport = transportFactory.newInstance(localAddressProvider.getLocalAddress(),
    0, new SyncStage<>(new RPCEventHandler()), null, RETRY_COUNT, RETRY_TIMEOUT);
  final SocketAddress clientAddress = new InetSocketAddress(clientHost, clientPort);
  try {
    link = transport.open(clientAddress, ENCODER, LINK_LISTENER);
  } catch (final IOException e) {
    throw new IllegalStateException("Failed to setup an RPC connection to the Client. "
      + "A failure at the client-side is suspected.");
  }
}
 
Example #25
Source File: GrpcMessageServer.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor.
 *
 * @param localAddressProvider local address provider.
 * @param nameResolver         name resolver.
 * @param idFactory            identifier factory.
 * @param localSenderId        id of the local sender.
 */
GrpcMessageServer(final LocalAddressProvider localAddressProvider,
                  final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String localSenderId) {
  this.localAddressProvider = localAddressProvider;
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.localSenderId = localSenderId;
  this.listenerMap = new ConcurrentHashMap<>();
}
 
Example #26
Source File: RemoteManagerTest.java    From reef with Apache License 2.0 4 votes vote down vote up
public RemoteManagerTest() throws InjectionException {
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.localAddressProvider = injector.getInstance(LocalAddressProvider.class);
  this.remoteManagerFactory = injector.getInstance(RemoteManagerFactory.class);
}
 
Example #27
Source File: TransportTest.java    From reef with Apache License 2.0 4 votes vote down vote up
public TransportTest() throws InjectionException {
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.localAddressProvider = injector.getInstance(LocalAddressProvider.class);
  this.tpFactory = injector.getInstance(TransportFactory.class);
}
 
Example #28
Source File: LargeMsgTest.java    From reef with Apache License 2.0 4 votes vote down vote up
public LargeMsgTest() throws InjectionException {
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.localAddressProvider = injector.getInstance(LocalAddressProvider.class);
  this.tpFactory = injector.getInstance(TransportFactory.class);
}
 
Example #29
Source File: TransportRaceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
public TransportRaceTest() throws InjectionException {
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.localAddressProvider = injector.getInstance(LocalAddressProvider.class);
  this.tpFactory = injector.getInstance(TransportFactory.class);
}
 
Example #30
Source File: RemoteTest.java    From reef with Apache License 2.0 4 votes vote down vote up
public RemoteTest() throws InjectionException {
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.localAddressProvider = injector.getInstance(LocalAddressProvider.class);
  this.tpFactory = injector.getInstance(TransportFactory.class);
}