Java Code Examples for org.apache.sqoop.core.SqoopConfiguration

The following examples show how to use org.apache.sqoop.core.SqoopConfiguration. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void doInitialize(AuthenticationProvider provider, String serverName) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
  MapContext mapContext = SqoopConfiguration.getInstance().getContext();
  String accessController = mapContext.getString(
          SecurityConstants.AUTHORIZATION_ACCESS_CONTROLLER,
          DEFAULT_AUTHORIZATION_ACCESS_CONTROLLER).trim();
  this.authorizationAccessController = SecurityFactory.getAuthorizationAccessController(accessController);

  String validator = mapContext.getString(
          SecurityConstants.AUTHORIZATION_VALIDATOR,
          DEFAULT_AUTHORIZATION_VALIDATOR).trim();
  this.authorizationValidator = SecurityFactory.getAuthorizationValidator(validator);

  this.authenticationProvider = provider;
  this.serverName = serverName;
}
 
Example 2
Source Project: sqoop-on-spark   Source File: AuthenticationManager.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized void initialize() throws ClassNotFoundException, IllegalAccessException, InstantiationException {
  if (LOG.isTraceEnabled()) {
    LOG.trace("Begin authentication manager initialization");
  }

  String handler = SqoopConfiguration.getInstance().getContext().getString(
      SecurityConstants.AUTHENTICATION_HANDLER,
      DEFAULT_AUTHENTICATION_HANDLER).trim();
  authenticationHandler = SecurityFactory.getAuthenticationHandler(handler);
  authenticationHandler.doInitialize();
  authenticationHandler.secureLogin();

  if (LOG.isInfoEnabled()) {
    LOG.info("Authentication loaded.");
  }
}
 
Example 3
Source Project: sqoop-on-spark   Source File: AuthorizationManager.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized void initialize() throws ClassNotFoundException, IllegalAccessException, InstantiationException {
  LOG.trace("Begin authorization manager initialization");

  String handler = SqoopConfiguration.getInstance().getContext().getString(
          SecurityConstants.AUTHORIZATION_HANDLER,
          DEFAULT_AUTHORIZATION_HANDLER).trim();
  authorizationHandler = SecurityFactory.getAuthorizationHandler(handler);

  String provider = SqoopConfiguration.getInstance().getContext().getString(
          SecurityConstants.AUTHENTICATION_PROVIDER,
          DEFAULT_AUTHENTICATION_PROVIDER).trim();

  String serverName = SqoopConfiguration.getInstance().getContext().getString(
          SecurityConstants.SERVER_NAME,
          DEFAULT_SERVER_NAME).trim();

  authorizationHandler.doInitialize(SecurityFactory.getAuthenticationProvider(provider), serverName);

  LOG.info("Authorization loaded.");
}
 
Example 4
Source Project: sqoop-on-spark   Source File: RepositoryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void configurationChanged() {
  LOG.info("Begin repository manager reconfiguring");
  MapContext newContext = SqoopConfiguration.getInstance().getContext();
  MapContext oldContext = SqoopConfiguration.getInstance().getOldContext();

  String newProviderClassName = newContext.getString(RepoConfigurationConstants.SYSCFG_REPO_PROVIDER);
  if (newProviderClassName == null
      || newProviderClassName.trim().length() == 0) {
    throw new SqoopException(RepositoryError.REPO_0001,
        RepoConfigurationConstants.SYSCFG_REPO_PROVIDER);
  }

  String oldProviderClassName = oldContext.getString(RepoConfigurationConstants.SYSCFG_REPO_PROVIDER);
  if (!newProviderClassName.equals(oldProviderClassName)) {
    LOG.warn("Repository provider cannot be replaced at the runtime. " +
             "You might need to restart the server.");
  }

  provider.configurationChanged();

  LOG.info("Repository manager reconfigured.");
}
 
Example 5
Source Project: sqoop-on-spark   Source File: TestRepositoryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSystemNotInitialized() throws Exception {
  // Unset any configuration dir if it is set by another test
  System.getProperties().remove(ConfigurationConstants.SYSPROP_CONFIG_DIR);
  Properties bootProps = new Properties();
  bootProps.setProperty(ConfigurationConstants.BOOTCFG_CONFIG_PROVIDER,
      PropertiesConfigurationProvider.class.getCanonicalName());
  Properties configProps = new Properties();
  SqoopTestUtils.setupTestConfigurationUsingProperties(bootProps, configProps);
  try {
    SqoopConfiguration.getInstance().initialize();
    RepositoryManager.getInstance().initialize();
  } catch (Exception ex) {
    Assert.assertTrue(ex instanceof SqoopException);
    Assert.assertSame(((SqoopException) ex).getErrorCode(),
        RepositoryError.REPO_0001);
  }
}
 
Example 6
private SqoopAuthBindingSingleton() {
  SqoopAuthBinding tmpBinding = null;
  try {
    String serverName = SqoopConfiguration.getInstance().getContext().getString(SecurityConstants.SERVER_NAME);
    if (Strings.isNullOrEmpty(serverName)) {
      throw new IllegalArgumentException(SecurityConstants.SERVER_NAME + " can't be null or empty");
    }
    SqoopAuthConf conf = loadAuthzConf();
    validateSentrySqoopConfig(conf);
    tmpBinding = new SqoopAuthBinding(conf, serverName.trim());
    log.info("SqoopAuthBinding created successfully");
  } catch (Exception ex) {
    log.error("Unable to create SqoopAuthBinding", ex);
    throw new RuntimeException("Unable to create SqoopAuthBinding: " + ex.getMessage(), ex);
  }
  binding = tmpBinding;
}
 
Example 7
private SqoopAuthConf loadAuthzConf() {
  String sentry_site = SqoopConfiguration.getInstance().getContext()
      .getString(SqoopAuthConf.SENTRY_SQOOP_SITE_URL);
  if (Strings.isNullOrEmpty(sentry_site)) {
    throw new IllegalArgumentException("Configuration key " + SqoopAuthConf.SENTRY_SQOOP_SITE_URL
        + " value '" + sentry_site + "' is invalid.");
  }

  SqoopAuthConf sqoopAuthConf = null;
  try {
    sqoopAuthConf = new SqoopAuthConf(new URL(sentry_site));
  } catch (MalformedURLException e) {
    throw new IllegalArgumentException("Configuration key " + SqoopAuthConf.SENTRY_SQOOP_SITE_URL
        + " specifies a malformed URL '" + sentry_site + "'", e);
  }
  return sqoopAuthConf;
}
 
Example 8
public void secureLogin() {
  MapContext mapContext = SqoopConfiguration.getInstance().getContext();
  String keytab = mapContext.getString(
          SecurityConstants.AUTHENTICATION_KERBEROS_KEYTAB).trim();
  if (keytab.length() == 0) {
    throw new SqoopException(SecurityError.AUTH_0001,
            SecurityConstants.AUTHENTICATION_KERBEROS_KEYTAB);
  }
  keytabFile = keytab;

  String principal = mapContext.getString(
          SecurityConstants.AUTHENTICATION_KERBEROS_PRINCIPAL).trim();
  if (principal.length() == 0) {
    throw new SqoopException(SecurityError.AUTH_0002,
            SecurityConstants.AUTHENTICATION_KERBEROS_PRINCIPAL);
  }
  keytabPrincipal = principal;

  Configuration conf = new Configuration();
  conf.set(get_hadoop_security_authentication(),
          SecurityConstants.TYPE.KERBEROS.name());
  UserGroupInformation.setConfiguration(conf);
  try {
    String hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
    UserGroupInformation.loginUserFromKeytab(hostPrincipal, keytab);
  } catch (IOException ex) {
    throw new SqoopException(SecurityError.AUTH_0003, ex);
  }
  LOG.info("Using Kerberos authentication, principal ["
          + principal + "] keytab [" + keytab + "]");
}
 
Example 9
Source Project: sqoop-on-spark   Source File: SqoopAuthenticationFilter.java    License: Apache License 2.0 5 votes vote down vote up
protected Configuration getProxyuserConfiguration(FilterConfig filterConfig) {
  MapContext mapContext = SqoopConfiguration.getInstance().getContext();
  Map<String, String> proxyuserConf = mapContext.getValByRegex("org\\.apache\\.sqoop\\.authentication\\.proxyuser");
  Configuration conf = new Configuration(false);
  for (Map.Entry<String, String> entry : proxyuserConf.entrySet()) {
    conf.set(entry.getKey().substring("org.apache.sqoop.authentication.proxyuser.".length()), entry.getValue());
  }
  return conf;
}
 
Example 10
Source Project: sqoop-on-spark   Source File: ConfiguredTool.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public final boolean runTool(String[] arguments) {
  try {
    SqoopConfiguration.getInstance().initialize();
    return runToolWithConfiguration(arguments);
  } finally {
    SqoopConfiguration.getInstance().destroy();
  }
}
 
Example 11
Source Project: sqoop-on-spark   Source File: AuditLoggerManager.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized void initialize() {
  LOG.info("Begin audit logger manager initialization");
  initializeLoggers();

  SqoopConfiguration.getInstance().getProvider()
      .registerListener(new CoreConfigurationListener(this));

  LOG.info("Audit logger manager initialized: OK");
}
 
Example 12
Source Project: sqoop-on-spark   Source File: JdbcRepositoryProvider.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void initialize(MapContext context) {
  repoContext = new JdbcRepositoryContext(SqoopConfiguration.getInstance().getContext());

  initializeRepositoryHandler();

  LOG.info("JdbcRepository initialized.");
}
 
Example 13
Source Project: ranger   Source File: RangerSqoopAuthorizerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Help function: init sqoop to enable ranger authentication
 */
private static void initSqoopAuth() throws IOException, ClassNotFoundException, IllegalAccessException,
		InstantiationException {
	// init sqoop configruation
	String basedir = System.getProperty("basedir");
	if (basedir == null) {
		basedir = new File(".").getCanonicalPath();
	}
	String sqoopConfigDirPath = basedir + "/src/test/resources/";
	System.setProperty(ConfigurationConstants.SYSPROP_CONFIG_DIR, sqoopConfigDirPath);
	SqoopConfiguration.getInstance().initialize();

	// init sqoop authorization
	AuthorizationManager.getInstance().initialize();

	// mock sqoop class for authentication
	RepositoryManager repositoryManager = mock(RepositoryManager.class);
	RepositoryManager.setInstance(repositoryManager);
	Repository repository = mock(Repository.class);
	when(repositoryManager.getRepository()).thenReturn(repository);

	MLink link = mock(MLink.class);
	when(repository.findLink(anyString())).thenReturn(link);
	MJob job = mock(MJob.class);
	when(repository.findJob(anyString())).thenReturn(job);

	// mock user "zhangqiang" as the creator of any link and any job
	when(link.getCreationUser()).thenReturn(ZHANGQIANG);
	when(job.getCreationUser()).thenReturn(ZHANGQIANG);
}
 
Example 14
private void validateSentrySqoopConfig(SqoopAuthConf conf) {
  boolean isTestingMode = Boolean.parseBoolean(conf.get(AuthzConfVars.AUTHZ_TESTING_MODE.getVar(),
                          AuthzConfVars.AUTHZ_TESTING_MODE.getDefault()));
  String authentication = SqoopConfiguration.getInstance().getContext()
                          .getString(SecurityConstants.AUTHENTICATION_TYPE, SecurityConstants.TYPE.SIMPLE.name());
  String kerberos = SecurityConstants.TYPE.KERBEROS.name();
  if(!isTestingMode && !kerberos.equalsIgnoreCase(authentication)) {
    throw new IllegalArgumentException(SecurityConstants.AUTHENTICATION_TYPE + "can't be set simple mode in non-testing mode");
  }
}
 
Example 15
Source Project: sqoop-on-spark   Source File: SqoopAuthenticationFilter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected Properties getConfiguration(String configPrefix,
                                      FilterConfig filterConfig) throws ServletException {
  Properties properties = new Properties();
  MapContext mapContext = SqoopConfiguration.getInstance().getContext();
  String type = mapContext.getString(
      SecurityConstants.AUTHENTICATION_TYPE,
      SecurityConstants.TYPE.SIMPLE.name()).trim();

  if (type.equalsIgnoreCase(SecurityConstants.TYPE.KERBEROS.name())) {
    properties.setProperty(AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName());

    String keytab = mapContext.getString(
            SecurityConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB).trim();
    if (keytab.length() == 0) {
      throw new SqoopException(SecurityError.AUTH_0005,
              SecurityConstants.AUTHENTICATION_KERBEROS_HTTP_KEYTAB);
    }

    String principal = mapContext.getString(
            SecurityConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL).trim();
    if (principal.length() == 0) {
      throw new SqoopException(SecurityError.AUTH_0006,
              SecurityConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
    }

    String hostPrincipal = "";
    try {
      hostPrincipal = SecurityUtil.getServerPrincipal(principal, "0.0.0.0");
    } catch (IOException e) {
      throw new SqoopException(SecurityError.AUTH_0006,
              SecurityConstants.AUTHENTICATION_KERBEROS_HTTP_PRINCIPAL);
    }

    properties.setProperty(KerberosAuthenticationHandler.PRINCIPAL, hostPrincipal);
    properties.setProperty(KerberosAuthenticationHandler.KEYTAB, keytab);
  } else if (type.equalsIgnoreCase(SecurityConstants.TYPE.SIMPLE.name())) {
    properties.setProperty(AUTH_TYPE, PseudoDelegationTokenAuthenticationHandler.class.getName());
    properties.setProperty(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED,
        mapContext.getString(SecurityConstants.AUTHENTICATION_ANONYMOUS, "true").trim());
  } else {
    throw new SqoopException(SecurityError.AUTH_0004, type);
  }

  properties.setProperty(DelegationTokenAuthenticationHandler.TOKEN_KIND,
          SecurityConstants.TOKEN_KIND);

  return properties;
}
 
Example 16
Source Project: sqoop-on-spark   Source File: ConnectorManager.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize() {
  initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.CONNECTOR_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
}
 
Example 17
Source Project: sqoop-on-spark   Source File: ConnectorManager.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize(boolean autoUpgrade) {
  if (LOG.isTraceEnabled()) {
    LOG.trace("Begin connector manager initialization");
  }
  
  System.out.println("Connector :" + autoUpgrade);


  // add external connectors into the class path
  // NOTE: class loading happens later in the ConnectorHandler
  ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext()
      .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY));

  List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();

  LOG.info("Connector config urls: " + connectorConfigs);

  System.out.println("Connector config urls: " + connectorConfigs);



  if (connectorConfigs.size() == 0) {
    throw new SqoopException(ConnectorError.CONN_0002);
  }

  for (URL url : connectorConfigs) {
    ConnectorHandler handler = new ConnectorHandler(url);
    
    System.out.println("Connector config URL: " + url);

    ConnectorHandler handlerOld =
        handlerMap.put(handler.getUniqueName(), handler);
    if (handlerOld != null) {
      throw new SqoopException(ConnectorError.CONN_0006,
          handler + ", " + handlerOld);
    }
  }

  registerConnectors(autoUpgrade);

  SqoopConfiguration.getInstance().getProvider()
      .registerListener(new CoreConfigurationListener(this));

  if (LOG.isInfoEnabled()) {
    LOG.info("Connectors loaded: " + handlerMap);
  }
}
 
Example 18
Source Project: sqoop-on-spark   Source File: AuditLogger.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Parse out all configurations for current logger
 * @return all configurations
 */
protected Map<String, String> getLoggerConfig() {
  String prefix = AuditLoggerConstants.PREFIX_AUDITLOGGER_CONFIG + getLoggerName() + ".";
  return SqoopConfiguration.getInstance().getContext().getNestedProperties(prefix);
}
 
Example 19
Source Project: sqoop-on-spark   Source File: AuditLoggerManager.java    License: Apache License 2.0 4 votes vote down vote up
private void initializeLoggers() {
  loggers.clear();

  MapContext context = SqoopConfiguration.getInstance().getContext();

  Map<String, String> auditLoggerProps = context.getNestedProperties(
      AuditLoggerConstants.PREFIX_AUDITLOGGER_CONFIG);

  // Initialize audit loggers
  for (String key : auditLoggerProps.keySet()) {
    if (key.endsWith(AuditLoggerConstants.SUFFIX_AUDITLOGGER_CLASS)) {
      String loggerName = key.substring(0, key.indexOf("."));
      String loggerClassName = auditLoggerProps.get(key);

      if (loggerClassName == null || loggerClassName.trim().length() == 0) {
        throw new SqoopException(AuditLoggerError.AUDIT_0001,
            "Logger name: " + loggerName);
      }

      Class<?> loggerClass =
          ClassUtils.loadClass(loggerClassName);

      if (loggerClass == null) {
        throw new SqoopException(AuditLoggerError.AUDIT_0001,
            "Logger Class: " + loggerClassName);
      }

      AuditLogger newLogger;
      try {
        newLogger = (AuditLogger) loggerClass.newInstance();
      } catch (Exception ex) {
        throw new SqoopException(AuditLoggerError.AUDIT_0001,
            "Logger Class: " + loggerClassName, ex);
      }

      newLogger.setLoggerName(loggerName);
      newLogger.initialize();
      loggers.add(newLogger);
      LOG.info("Audit Logger has been initialized: " + loggerName);
    }
  }
}
 
Example 20
Source Project: sqoop-on-spark   Source File: RepositoryManager.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize() {
  initialize(SqoopConfiguration.getInstance().getContext().getBoolean(RepoConfigurationConstants.SYSCFG_REPO_SCHEMA_IMMUTABLE, true));
}
 
Example 21
Source Project: sqoop-on-spark   Source File: RepositoryManager.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize(boolean immutableRepository) {
  MapContext context = SqoopConfiguration.getInstance().getContext();

  Map<String, String> repoSysProps = context.getNestedProperties(
      RepoConfigurationConstants.SYSCFG_REPO_SYSPROP_PREFIX);

  LOG.info("Setting system properties: " + repoSysProps);

  for (Map.Entry<String, String> entry : repoSysProps.entrySet()) {
    System.setProperty(entry.getKey(), entry.getValue());
  }

  String repoProviderClassName = context.getString(
      RepoConfigurationConstants.SYSCFG_REPO_PROVIDER);

  if (repoProviderClassName == null
      || repoProviderClassName.trim().length() == 0) {
    throw new SqoopException(RepositoryError.REPO_0001,
        RepoConfigurationConstants.SYSCFG_REPO_PROVIDER);
  }

  if (LOG.isTraceEnabled()) {
    LOG.trace("Repository provider: " + repoProviderClassName);
  }

  Class<?> repoProviderClass = ClassUtils.loadClass(repoProviderClassName);

  if (repoProviderClass == null) {
    throw new SqoopException(RepositoryError.REPO_0001,
        repoProviderClassName);
  }

  try {
    provider = (RepositoryProvider) repoProviderClass.newInstance();
  } catch (Exception ex) {
    throw new SqoopException(RepositoryError.REPO_0001,
        repoProviderClassName, ex);
  }

  provider.initialize(context);

  if(!immutableRepository) {
    LOG.info("Creating or updating respository at bootup");
    provider.getRepository().createOrUpgradeRepository();
  }

  // NOTE: There are scenarios where a repository upgrade/ changes may happen outside of the
  // server bootup lifecyle. Hence always check/ verify for the repository sanity before marking the repo manager ready
  if(!provider.getRepository().isRepositorySuitableForUse()) {
    throw new SqoopException(RepositoryError.REPO_0002);
  }

  SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));

  LOG.info("Repository Manager initialized: OK");
}
 
Example 22
Source Project: sqoop-on-spark   Source File: JdbcRepositoryProvider.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void configurationChanged() {
  LOG.info("Begin JdbcRepository reconfiguring.");
  JdbcRepositoryContext oldRepoContext = repoContext;
  repoContext = new JdbcRepositoryContext(SqoopConfiguration.getInstance().getContext());

  // reconfigure jdbc handler
  String newJdbcHandlerClassName = repoContext.getHandlerClassName();
  if (newJdbcHandlerClassName == null
      || newJdbcHandlerClassName.trim().length() == 0) {
    throw new SqoopException(RepositoryError.JDBCREPO_0001,
        newJdbcHandlerClassName);
  }

  String oldJdbcHandlerClassName = oldRepoContext.getHandlerClassName();
  if (!newJdbcHandlerClassName.equals(oldJdbcHandlerClassName)) {
    LOG.warn("Repository JDBC handler cannot be replaced at the runtime. " +
             "You might need to restart the server.");
  }

  // reconfigure jdbc driver
  String newJdbcDriverClassName = repoContext.getDriverClass();
  if (newJdbcDriverClassName == null
      || newJdbcDriverClassName.trim().length() == 0) {
    throw new SqoopException(RepositoryError.JDBCREPO_0003,
            newJdbcDriverClassName);
  }

  String oldJdbcDriverClassName = oldRepoContext.getDriverClass();
  if (!newJdbcDriverClassName.equals(oldJdbcDriverClassName)) {
    LOG.warn("Repository JDBC driver cannot be replaced at the runtime. " +
             "You might need to restart the server.");
  }

  // reconfigure max connection
  connectionPool.setMaxActive(repoContext.getMaximumConnections());

  // reconfigure the url of repository
  String connectUrl = repoContext.getConnectionUrl();
  String oldurl = oldRepoContext.getConnectionUrl();
  if (connectUrl != null && !connectUrl.equals(oldurl)) {
    LOG.warn("Repository URL cannot be replaced at the runtime. " +
             "You might need to restart the server.");
  }

  // if connection properties or transaction isolation option changes
  boolean connFactoryChanged = false;

  // compare connection properties
  if (!connFactoryChanged) {
    Properties oldProp = oldRepoContext.getConnectionProperties();
    Properties newProp = repoContext.getConnectionProperties();

    if (newProp.size() != oldProp.size()) {
      connFactoryChanged = true;
    } else {
      for (Object key : newProp.keySet()) {
        if (!newProp.getProperty((String) key).equals(oldProp.getProperty((String) key))) {
          connFactoryChanged = true;
          break;
        }
      }
    }
  }

  // compare the transaction isolation option
  if (!connFactoryChanged) {
    String oldTxOption = oldRepoContext.getTransactionIsolation().toString();
    String newTxOption = repoContext.getTransactionIsolation().toString();

    if (!newTxOption.equals(oldTxOption)) {
      connFactoryChanged = true;
    }
  }

  if (connFactoryChanged) {
    // try to reconfigure connection factory
    try {
      LOG.info("Reconfiguring Connection Factory.");
      Properties jdbcProps = repoContext.getConnectionProperties();

      ConnectionFactory connFactory =
          new DriverManagerConnectionFactory(connectUrl, jdbcProps);

      new PoolableConnectionFactory(connFactory, connectionPool, statementPool,
              handler.validationQuery(), false, false,
              repoContext.getTransactionIsolation().getCode());
    } catch (IllegalStateException ex) {
      // failed to reconfigure connection factory
      LOG.warn("Repository connection cannot be reconfigured currently. " +
               "You might need to restart the server.");
    }
  }

  // ignore the create schema option, because the repo url is not allowed to change

  LOG.info("JdbcRepository reconfigured.");
}
 
Example 23
Source Project: sqoop-on-spark   Source File: JobManager.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize() {
  LOG.trace("Begin submission engine manager initialization");
  MapContext context = SqoopConfiguration.getInstance().getContext();

  // Let's load configured submission engine
  String submissionEngineClassName =
    context.getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);

  submissionEngine = (SubmissionEngine) ClassUtils
    .instantiate(submissionEngineClassName);
  if (submissionEngine == null) {
    throw new SqoopException(DriverError.DRIVER_0001,
      submissionEngineClassName);
  }

  submissionEngine.initialize(context,
      DriverConstants.PREFIX_SUBMISSION_ENGINE_CONFIG);

  // Execution engine
  String executionEngineClassName =
    context.getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);

  executionEngine = (ExecutionEngine) ClassUtils
    .instantiate(executionEngineClassName);
  if (executionEngine == null) {
    throw new SqoopException(DriverError.DRIVER_0007,
      executionEngineClassName);
  }

  // We need to make sure that user has configured compatible combination of
  // submission engine and execution engine
  if (!submissionEngine
    .isExecutionEngineSupported(executionEngine.getClass())) {
    throw new SqoopException(DriverError.DRIVER_0008);
  }

  executionEngine.initialize(context,
      DriverConstants.PREFIX_EXECUTION_ENGINE_CONFIG);

  // Set up worker threads
  purgeThreshold = context.getLong(
    DriverConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
    DEFAULT_PURGE_THRESHOLD
    );
  purgeSleep = context.getLong(
    DriverConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
    DEFAULT_PURGE_SLEEP
    );

  purgeThread = new PurgeThread();
  purgeThread.start();

  updateSleep = context.getLong(
    DriverConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
    DEFAULT_UPDATE_SLEEP
    );

  updateThread = new UpdateThread();
  updateThread.start();

  SqoopConfiguration.getInstance().getProvider()
    .registerListener(new CoreConfigurationListener(this));

  LOG.info("Submission manager initialized: OK");
}
 
Example 24
Source Project: sqoop-on-spark   Source File: JobManager.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void configurationChanged() {
  LOG.info("Begin submission engine manager reconfiguring");
  MapContext newContext = SqoopConfiguration.getInstance().getContext();
  MapContext oldContext = SqoopConfiguration.getInstance().getOldContext();

  String newSubmissionEngineClassName = newContext
    .getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);
  if (newSubmissionEngineClassName == null
    || newSubmissionEngineClassName.trim().length() == 0) {
    throw new SqoopException(DriverError.DRIVER_0001,
      newSubmissionEngineClassName);
  }

  String oldSubmissionEngineClassName = oldContext
    .getString(DriverConstants.SYSCFG_SUBMISSION_ENGINE);
  if (!newSubmissionEngineClassName.equals(oldSubmissionEngineClassName)) {
    LOG.warn("Submission engine cannot be replaced at the runtime. " +
      "You might need to restart the server.");
  }

  String newExecutionEngineClassName = newContext
    .getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);
  if (newExecutionEngineClassName == null
    || newExecutionEngineClassName.trim().length() == 0) {
    throw new SqoopException(DriverError.DRIVER_0007,
      newExecutionEngineClassName);
  }

  String oldExecutionEngineClassName = oldContext
    .getString(DriverConstants.SYSCFG_EXECUTION_ENGINE);
  if (!newExecutionEngineClassName.equals(oldExecutionEngineClassName)) {
    LOG.warn("Execution engine cannot be replaced at the runtime. " +
      "You might need to restart the server.");
  }

  // Set up worker threads
  purgeThreshold = newContext.getLong(
    DriverConstants.SYSCFG_SUBMISSION_PURGE_THRESHOLD,
    DEFAULT_PURGE_THRESHOLD
    );
  purgeSleep = newContext.getLong(
    DriverConstants.SYSCFG_SUBMISSION_PURGE_SLEEP,
    DEFAULT_PURGE_SLEEP
    );
  purgeThread.interrupt();

  updateSleep = newContext.getLong(
    DriverConstants.SYSCFG_SUBMISSION_UPDATE_SLEEP,
    DEFAULT_UPDATE_SLEEP
    );
  updateThread.interrupt();

  LOG.info("Submission engine manager reconfigured.");
}
 
Example 25
Source Project: sqoop-on-spark   Source File: Driver.java    License: Apache License 2.0 4 votes vote down vote up
public synchronized void initialize() {
  initialize(SqoopConfiguration.getInstance().getContext()
      .getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE));
}
 
Example 26
Source Project: sqoop-on-spark   Source File: Driver.java    License: Apache License 2.0 3 votes vote down vote up
public synchronized void initialize(boolean autoUpgrade) {
  LOG.trace("Begin Driver initialization");

  // Register driver in repository
  mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade);

  SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this));

  LOG.info("Driver initialized: OK");
}