org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HistoryLogUtilsTest.java    From spydra with Apache License 2.0 6 votes vote down vote up
@Test
public void testGenerateHadoopConfig() throws Exception {
  Configuration cfg = HistoryLogUtils.generateHadoopConfig(DUMMY_CLIENT_ID,
      DUMMY_USER_NAME, DUMMY_BUCKET);

  // We are asserting that the properties involving substitution have been changed
  checkPropertySubstitution(this.configWithoutSubstitute, cfg,
      YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
      "gs://" + DUMMY_BUCKET + "/logs/such-client");

  checkPropertySubstitution(this.configWithoutSubstitute, cfg,
      JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "gs://" + DUMMY_BUCKET + "/history/such-client/done-intermediate");

  checkPropertySubstitution(this.configWithoutSubstitute, cfg,
      JHAdminConfig.MR_HISTORY_DONE_DIR,
      "gs://" + DUMMY_BUCKET + "/history/such-client/done");

  // Some additional guards to check whether we accidentally load additional config
  assertEquals("Sizes of configuration must not differ. Except for the user, client-id and bucket properties",
      cfg.size(), this.configWithoutSubstitute.size() + 3);
}
 
Example #2
Source File: TestJobHistoryEventHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example #3
Source File: ClientCache.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected MRClientProtocol instantiateHistoryProxy()
    throws IOException {
  final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
  if (StringUtils.isEmpty(serviceAddr)) {
    return null;
  }
  LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
  final YarnRPC rpc = YarnRPC.create(conf);
  LOG.debug("Connected to HistoryServer at: " + serviceAddr);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), conf);
    }
  });
}
 
Example #4
Source File: HistoryClientService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected void initializeWebApp(Configuration conf) {
  webApp = new HsWebApp(history);
  InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf);
  // NOTE: there should be a .at(InetSocketAddress)
  WebApps
      .$for("jobhistory", HistoryClientService.class, this, "ws")
      .with(conf)
      .withHttpSpnegoKeytabKey(
          JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
      .withHttpSpnegoPrincipalKey(
          JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY)
      .at(NetUtils.getHostPortString(bindAddress)).start(webApp);
  
  String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
  MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
      connectHost + ":" + webApp.getListenerAddress().getPort());
}
 
Example #5
Source File: HistoryServerStateStoreServiceFactory.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs an instance of the configured storage class
 * 
 * @param conf the configuration
 * @return the state storage instance
 */
public static HistoryServerStateStoreService getStore(Configuration conf) {
  Class<? extends HistoryServerStateStoreService> storeClass =
      HistoryServerNullStateStoreService.class;
  boolean recoveryEnabled = conf.getBoolean(
      JHAdminConfig.MR_HS_RECOVERY_ENABLE,
      JHAdminConfig.DEFAULT_MR_HS_RECOVERY_ENABLE);
  if (recoveryEnabled) {
    storeClass = conf.getClass(JHAdminConfig.MR_HS_STATE_STORE, null,
        HistoryServerStateStoreService.class);
    if (storeClass == null) {
      throw new RuntimeException("Unable to locate storage class, check "
          + JHAdminConfig.MR_HS_STATE_STORE);
    }
  }
  return ReflectionUtils.newInstance(storeClass, conf);
}
 
Example #6
Source File: MiniTajoYarnCluster.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void start() {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
          MiniYARNCluster.getHostname() + ":0");
      getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
          MiniYARNCluster.getHostname() + ":0");
    }
    super.start();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }

  LOG.info("MiniMRYARN ResourceManager address: " +
      getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
      getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer address: " +
      getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: " +
      getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
}
 
Example #7
Source File: HSAdmin.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int refreshUserToGroupsMappings() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();

  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  RefreshUserMappingsProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, RefreshUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  // Refresh the user-to-groups mappings
  refreshProtocol.refreshUserToGroupsMappings();

  return 0;
}
 
Example #8
Source File: HistoryServerStateStoreServiceFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs an instance of the configured storage class
 * 
 * @param conf the configuration
 * @return the state storage instance
 */
public static HistoryServerStateStoreService getStore(Configuration conf) {
  Class<? extends HistoryServerStateStoreService> storeClass =
      HistoryServerNullStateStoreService.class;
  boolean recoveryEnabled = conf.getBoolean(
      JHAdminConfig.MR_HS_RECOVERY_ENABLE,
      JHAdminConfig.DEFAULT_MR_HS_RECOVERY_ENABLE);
  if (recoveryEnabled) {
    storeClass = conf.getClass(JHAdminConfig.MR_HS_STATE_STORE, null,
        HistoryServerStateStoreService.class);
    if (storeClass == null) {
      throw new RuntimeException("Unable to locate storage class, check "
          + JHAdminConfig.MR_HS_STATE_STORE);
    }
  }
  return ReflectionUtils.newInstance(storeClass, conf);
}
 
Example #9
Source File: HSAdmin.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private int refreshUserToGroupsMappings() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();

  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  RefreshUserMappingsProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, RefreshUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  // Refresh the user-to-groups mappings
  refreshProtocol.refreshUserToGroupsMappings();

  return 0;
}
 
Example #10
Source File: HSAdmin.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private int refreshSuperUserGroupsConfiguration() throws IOException {
  // Refresh the super-user groups
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  RefreshUserMappingsProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, RefreshUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  // Refresh the super-user group mappings
  refreshProtocol.refreshSuperUserGroupsConfiguration();

  return 0;
}
 
Example #11
Source File: ClientHSSecurityInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
  if (!protocol
      .equals(HSClientProtocolPB.class)) {
    return null;
  }
  return new KerberosInfo() {

    @Override
    public Class<? extends Annotation> annotationType() {
      return null;
    }

    @Override
    public String serverPrincipal() {
      return JHAdminConfig.MR_HISTORY_PRINCIPAL;
    }

    @Override
    public String clientPrincipal() {
      return null;
    }
  };
}
 
Example #12
Source File: HistoryClientService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected void initializeWebApp(Configuration conf) {
  webApp = new HsWebApp(history);
  InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf);
  // NOTE: there should be a .at(InetSocketAddress)
  WebApps
      .$for("jobhistory", HistoryClientService.class, this, "ws")
      .with(conf)
      .withHttpSpnegoKeytabKey(
          JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
      .withHttpSpnegoPrincipalKey(
          JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY)
      .at(NetUtils.getHostPortString(bindAddress)).start(webApp);
  
  String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
  MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
      connectHost + ":" + webApp.getListenerAddress().getPort());
}
 
Example #13
Source File: TestJobCleanup.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws IOException {
  JobConf conf = new JobConf();
  fileSys = FileSystem.get(conf);
  fileSys.delete(new Path(TEST_ROOT_DIR), true);
  conf.set("mapred.job.tracker.handler.count", "1");
  conf.set("mapred.job.tracker", "127.0.0.1:0");
  conf.set("mapred.job.tracker.http.address", "127.0.0.1:0");
  conf.set("mapred.task.tracker.http.address", "127.0.0.1:0");
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, TEST_ROOT_DIR +
    "/intermediate");
  conf.set(org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
    .SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, "true");

  mr = new MiniMRCluster(1, "file:///", 1, null, null, conf);
  inDir = new Path(TEST_ROOT_DIR, "test-input");
  String input = "The quick brown fox\n" + "has many silly\n"
      + "red fox sox\n";
  DataOutputStream file = fileSys.create(new Path(inDir, "part-" + 0));
  file.writeBytes(input);
  file.close();
  emptyInDir = new Path(TEST_ROOT_DIR, "empty-input");
  fileSys.mkdirs(emptyInDir);
}
 
Example #14
Source File: TestJobHistoryEventHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example #15
Source File: HSAdmin.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int refreshSuperUserGroupsConfiguration() throws IOException {
  // Refresh the super-user groups
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  RefreshUserMappingsProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, RefreshUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  // Refresh the super-user group mappings
  refreshProtocol.refreshSuperUserGroupsConfiguration();

  return 0;
}
 
Example #16
Source File: ClientCache.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected MRClientProtocol instantiateHistoryProxy()
    throws IOException {
  final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
  if (StringUtils.isEmpty(serviceAddr)) {
    return null;
  }
  LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
  final YarnRPC rpc = YarnRPC.create(conf);
  LOG.debug("Connected to HistoryServer at: " + serviceAddr);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), conf);
    }
  });
}
 
Example #17
Source File: TestHistoryServerFileSystemStateStoreService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  FileUtil.fullyDelete(testDir);
  testDir.mkdirs();
  conf = new Configuration();
  conf.setBoolean(JHAdminConfig.MR_HS_RECOVERY_ENABLE, true);
  conf.setClass(JHAdminConfig.MR_HS_STATE_STORE,
      HistoryServerFileSystemStateStoreService.class,
      HistoryServerStateStoreService.class);
  conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI,
      testDir.getAbsoluteFile().toURI().toString());
}
 
Example #18
Source File: TestMapReduceTrackingUriPlugin.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testProducesHistoryServerUriForAppId() throws URISyntaxException {
  final String historyAddress = "example.net:424242";
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress);
  MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
  plugin.setConf(conf);
  ApplicationId id = ApplicationId.newInstance(6384623l, 5);
  String jobSuffix = id.toString().replaceFirst("^application_", "job_");
  URI expected =
      new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix);
  URI actual = plugin.getTrackingUri(id);
  assertEquals(expected, actual);
}
 
Example #19
Source File: JobHistory.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  LOG.info("JobHistory Init");
  this.conf = conf;
  this.appID = ApplicationId.newInstance(0, 0);
  this.appAttemptID = RecordFactoryProvider.getRecordFactory(conf)
      .newRecordInstance(ApplicationAttemptId.class);

  moveThreadInterval = conf.getLong(
      JHAdminConfig.MR_HISTORY_MOVE_INTERVAL_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_INTERVAL_MS);

  hsManager = createHistoryFileManager();
  hsManager.init(conf);
  try {
    hsManager.initExisting();
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to intialize existing directories", e);
  }

  storage = createHistoryStorage();
  
  if (storage instanceof Service) {
    ((Service) storage).init(conf);
  }
  storage.setHistoryFileManager(hsManager);

  super.serviceInit(conf);
}
 
Example #20
Source File: HistoryClientService.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  initializeWebApp(conf);
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.MR_HISTORY_BIND_HOST,
      JHAdminConfig.MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

  server =
      rpc.getServer(HSClientProtocol.class, protocolHandler, address,
          conf, jhsDTSecretManager,
          conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
              JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    server.refreshServiceAcl(conf, new ClientHSPolicyProvider());
  }
  
  server.start();
  this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_BIND_HOST,
                                            JHAdminConfig.MR_HISTORY_ADDRESS,
                                            JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
                                            server.getListenerAddress());
  LOG.info("Instantiated HistoryClientService at " + this.bindAddress);

  super.serviceStart();
}
 
Example #21
Source File: CachedHistoryStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("serial")
private void createLoadedJobCache(Configuration conf) {
  loadedJobCacheSize = conf.getInt(
      JHAdminConfig.MR_HISTORY_LOADED_JOB_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_LOADED_JOB_CACHE_SIZE);

  loadedJobCache = Collections.synchronizedMap(new LinkedHashMap<JobId, Job>(
      loadedJobCacheSize + 1, 0.75f, true) {
    @Override
    public boolean removeEldestEntry(final Map.Entry<JobId, Job> eldest) {
      return super.size() > loadedJobCacheSize;
    }
  });
}
 
Example #22
Source File: HistoryLogUtils.java    From spydra with Apache License 2.0 5 votes vote down vote up
/**
 * Starts a minimal JobHistoryServer.
 */
public static void startJhs(Configuration cfg) {
  try {
    JobHistoryServer jobHistoryServer = new JobHistoryServer();
    jobHistoryServer.init(cfg);
    logger.info(String.format(
        "Starting JobHistoryServer on: http://%s",
        cfg.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)));
    jobHistoryServer.start();
  } catch (Exception e) {
    logger.error("Error starting JobHistoryServer", e);
    System.exit(1);
  }
}
 
Example #23
Source File: HSAdminServer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void refreshAdminAcls() throws IOException {
  UserGroupInformation user = checkAcls("refreshAdminAcls");

  Configuration conf = createConf();
  adminAcl = new AccessControlList(conf.get(JHAdminConfig.JHS_ADMIN_ACL,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ACL));
  HSAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
      HISTORY_ADMIN_SERVER);
}
 
Example #24
Source File: HSAdminServer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void refreshAdminAcls() throws IOException {
  UserGroupInformation user = checkAcls("refreshAdminAcls");

  Configuration conf = createConf();
  adminAcl = new AccessControlList(conf.get(JHAdminConfig.JHS_ADMIN_ACL,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ACL));
  HSAuditLogger.logSuccess(user.getShortUserName(), "refreshAdminAcls",
      HISTORY_ADMIN_SERVER);
}
 
Example #25
Source File: TestHistoryServerLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  FileUtil.fullyDelete(testDir);
  testDir.mkdirs();
  conf = new Configuration();
  conf.setBoolean(JHAdminConfig.MR_HS_RECOVERY_ENABLE, true);
  conf.setClass(JHAdminConfig.MR_HS_STATE_STORE,
      HistoryServerLeveldbStateStoreService.class,
      HistoryServerStateStoreService.class);
  conf.set(JHAdminConfig.MR_HS_LEVELDB_STATE_STORE_PATH,
      testDir.getAbsoluteFile().toString());
}
 
Example #26
Source File: TestHSAdminServer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testRefreshAdminAcls() throws Exception {
  // Setting current user to admin acl
  conf.set(JHAdminConfig.JHS_ADMIN_ACL, UserGroupInformation.getCurrentUser()
      .getUserName());
  String[] args = new String[1];
  args[0] = "-refreshAdminAcls";
  hsAdminClient.run(args);

  // Now I should be able to run any hsadmin command without any exception
  // being thrown
  args[0] = "-refreshSuperUserGroupsConfiguration";
  hsAdminClient.run(args);

  // Lets remove current user from admin acl
  conf.set(JHAdminConfig.JHS_ADMIN_ACL, "notCurrentUser");
  args[0] = "-refreshAdminAcls";
  hsAdminClient.run(args);

  // Now I should get an exception if i run any hsadmin command
  Throwable th = null;
  args[0] = "-refreshSuperUserGroupsConfiguration";
  try {
    hsAdminClient.run(args);
  } catch (Exception e) {
    th = e;
  }
  assertTrue(th instanceof RemoteException);
}
 
Example #27
Source File: TestHistoryFileManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void testCreateHistoryDirs(Configuration conf, Clock clock)
    throws Exception {
  conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID());
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID());
  HistoryFileManager hfm = new HistoryFileManager();
  hfm.conf = conf;
  hfm.createHistoryDirs(clock, 500, 2000);
}
 
Example #28
Source File: TestHSAdminServer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void init() throws HadoopIllegalArgumentException, IOException {
  conf = new JobConf();
  conf.set(JHAdminConfig.JHS_ADMIN_ADDRESS, "0.0.0.0:0");
  conf.setClass("hadoop.security.group.mapping", MockUnixGroupsMapping.class,
      GroupMappingServiceProvider.class);
  conf.setLong("hadoop.security.groups.cache.secs", groupRefreshTimeoutSec);
  conf.setBoolean(
        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
        securityEnabled);
  Groups.getUserToGroupsMappingService(conf);
  jobHistoryService = mock(JobHistory.class);
  alds = mock(AggregatedLogDeletionService.class);

  hsAdminServer = new HSAdminServer(alds, jobHistoryService) {

    @Override
    protected Configuration createConf() {
      return conf;
    }
  };
  hsAdminServer.init(conf);
  hsAdminServer.start();
  conf.setSocketAddr(JHAdminConfig.JHS_ADMIN_ADDRESS,
      hsAdminServer.clientRpcServer.getListenerAddress());
  hsAdminClient = new HSAdmin(conf);
}
 
Example #29
Source File: TestHistoryFileManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void testTryCreateHistoryDirs(Configuration conf, boolean expected)
    throws Exception {
  conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, getDoneDirNameForTest());
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, getIntermediateDoneDirNameForTest());
  HistoryFileManager hfm = new HistoryFileManager();
  hfm.conf = conf;
  Assert.assertEquals(expected, hfm.tryCreatingHistoryDirs(false));
}
 
Example #30
Source File: TestHistoryServerFileSystemStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  FileUtil.fullyDelete(testDir);
  testDir.mkdirs();
  conf = new Configuration();
  conf.setBoolean(JHAdminConfig.MR_HS_RECOVERY_ENABLE, true);
  conf.setClass(JHAdminConfig.MR_HS_STATE_STORE,
      HistoryServerFileSystemStateStoreService.class,
      HistoryServerStateStoreService.class);
  conf.set(JHAdminConfig.MR_HS_FS_STATE_STORE_URI,
      testDir.getAbsoluteFile().toURI().toString());
}