org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFetcher.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example #2
Source File: TestJobImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
    String user, int numSplits, AppContext appContext) {
  super(jobId, applicationAttemptId, conf, eventHandler,
      null, new JobTokenSecretManager(), new Credentials(),
      new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
      MRAppMetrics.create(), null, newApiCommitter, user,
      System.currentTimeMillis(), null, appContext, null, null);

  initTransition = getInitTransition(numSplits);
  localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
        EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
        JobEventType.JOB_INIT,
        // This is abusive.
        initTransition);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example #3
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
    String user, int numSplits, AppContext appContext) {
  super(jobId, applicationAttemptId, conf, eventHandler,
      null, new JobTokenSecretManager(), new Credentials(),
      new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
      MRAppMetrics.create(), null, newApiCommitter, user,
      System.currentTimeMillis(), null, appContext, null, null);

  initTransition = getInitTransition(numSplits);
  localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
        EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
        JobEventType.JOB_INIT,
        // This is abusive.
        initTransition);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example #4
Source File: MRApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public TestJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    OutputCommitter committer, boolean newApiCommitter,
    String user, AppContext appContext,
    JobStateInternal forcedState, String diagnostic) {
  super(jobId, getApplicationAttemptId(applicationId, getStartCount()),
      conf, eventHandler, taskAttemptListener,
      new JobTokenSecretManager(), new Credentials(), clock,
      getCompletedTaskFromPreviousRun(), metrics, committer,
      newApiCommitter, user, System.currentTimeMillis(), getAllAMInfos(),
      appContext, forcedState, diagnostic);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example #5
Source File: TestShuffleHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static int getShuffleResponseCode(ShuffleHandler shuffle,
    Token<JobTokenIdentifier> jt) throws IOException {
  URL url = new URL("http://127.0.0.1:"
      + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
      + "/mapOutput?job=job_12345_0001&reduce=0&map=attempt_12345_1_m_1_0");
  HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  String encHash = SecureShuffleUtils.hashFromString(
      SecureShuffleUtils.buildMsgFrom(url),
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  conn.addRequestProperty(
      SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
      ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
      ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
  conn.connect();
  int rc = conn.getResponseCode();
  conn.disconnect();
  return rc;
}
 
Example #6
Source File: TestFetcher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
@SuppressWarnings("unchecked") // mocked generics
public void setup() {
  LOG.info(">>>> " + name.getMethodName());
  job = new JobConf();
  job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
  jobWithRetry = new JobConf();
  jobWithRetry.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, true);
  id = TaskAttemptID.forName("attempt_0_1_r_1_1");
  ss = mock(ShuffleSchedulerImpl.class);
  mm = mock(MergeManagerImpl.class);
  r = mock(Reporter.class);
  metrics = mock(ShuffleClientMetrics.class);
  except = mock(ExceptionReporter.class);
  key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
  connection = mock(HttpURLConnection.class);

  allErrs = mock(Counters.Counter.class);
  when(r.getCounter(anyString(), anyString())).thenReturn(allErrs);

  ArrayList<TaskAttemptID> maps = new ArrayList<TaskAttemptID>(1);
  maps.add(map1ID);
  maps.add(map2ID);
  when(ss.getMapsForHost(host)).thenReturn(maps);
}
 
Example #7
Source File: TestShuffleHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static int getShuffleResponseCode(ShuffleHandler shuffle,
    Token<JobTokenIdentifier> jt) throws IOException {
  URL url = new URL("http://127.0.0.1:"
      + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY)
      + "/mapOutput?job=job_12345_0001&reduce=0&map=attempt_12345_1_m_1_0");
  HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  String encHash = SecureShuffleUtils.hashFromString(
      SecureShuffleUtils.buildMsgFrom(url),
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  conn.addRequestProperty(
      SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME,
      ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
  conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
      ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
  conn.connect();
  int rc = conn.getResponseCode();
  conn.disconnect();
  return rc;
}
 
Example #8
Source File: MRApp.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public TestJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    OutputCommitter committer, boolean newApiCommitter,
    String user, AppContext appContext,
    JobStateInternal forcedState, String diagnostic) {
  super(jobId, getApplicationAttemptId(applicationId, getStartCount()),
      conf, eventHandler, taskAttemptListener,
      new JobTokenSecretManager(), new Credentials(), clock,
      getCompletedTaskFromPreviousRun(), metrics, committer,
      newApiCommitter, user, System.currentTimeMillis(), getAllAMInfos(),
      appContext, forcedState, diagnostic);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  localStateMachine = localFactory.make(this);
}
 
Example #9
Source File: ShuffleHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  userRsrc = new ConcurrentHashMap<String,String>();
  secretManager = new JobTokenSecretManager();
  recoverState(conf);
  ServerBootstrap bootstrap = new ServerBootstrap(selector);
  try {
    pipelineFact = new HttpPipelineFactory(conf);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
  bootstrap.setPipelineFactory(pipelineFact);
  port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
  Channel ch = bootstrap.bind(new InetSocketAddress(port));
  accepted.add(ch);
  port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
  conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
  pipelineFact.SHUFFLE.setPort(port);
  LOG.info(getName() + " listening on port " + port);
  super.serviceStart();

  sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
                                  DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
  connectionKeepAliveEnabled =
      conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
  connectionKeepAliveTimeOut =
      Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
  mapOutputMetaInfoCacheSize =
      Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
        DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
}
 
Example #10
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testReportDiagnostics() throws Exception {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  final String diagMsg = "some diagnostic message";
  final JobDiagnosticsUpdateEvent diagUpdateEvent =
      new JobDiagnosticsUpdateEvent(jobId, diagMsg);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  JobImpl job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(diagUpdateEvent);
  String diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));

  job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  job.handle(diagUpdateEvent);
  diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));
}
 
Example #11
Source File: TestTaskAttemptListenerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    TaskHeartbeatHandler hbHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
  this.taskHeartbeatHandler = hbHandler;
}
 
Example #12
Source File: TestJobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private boolean testUberDecision(Configuration conf) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  JobImpl job =
      new JobImpl(jobId, ApplicationAttemptId.newInstance(
        ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
        null, new JobTokenSecretManager(), new Credentials(), null, null,
        mrAppMetrics, null, true, null, 0, null, null, null, null);
  InitTransition initTransition = getInitTransition(2);
  JobEvent mockJobEvent = mock(JobEvent.class);
  initTransition.transition(job, mockJobEvent);
  boolean isUber = job.isUber();
  return isUber;
}
 
Example #13
Source File: TaskAttemptListenerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public TaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    byte[] secretShuffleKey) {
  super(TaskAttemptListenerImpl.class.getName());
  this.context = context;
  this.jobTokenSecretManager = jobTokenSecretManager;
  this.rmHeartbeatHandler = rmHeartbeatHandler;
  this.encryptedSpillKey = secretShuffleKey;
}
 
Example #14
Source File: TaskAttemptListenerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public TaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    byte[] secretShuffleKey) {
  super(TaskAttemptListenerImpl.class.getName());
  this.context = context;
  this.jobTokenSecretManager = jobTokenSecretManager;
  this.rmHeartbeatHandler = rmHeartbeatHandler;
  this.encryptedSpillKey = secretShuffleKey;
}
 
Example #15
Source File: ShuffleHandler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  userRsrc = new ConcurrentHashMap<String,String>();
  secretManager = new JobTokenSecretManager();
  recoverState(conf);
  ServerBootstrap bootstrap = new ServerBootstrap(selector);
  try {
    pipelineFact = new HttpPipelineFactory(conf);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
  bootstrap.setPipelineFactory(pipelineFact);
  port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
  Channel ch = bootstrap.bind(new InetSocketAddress(port));
  accepted.add(ch);
  port = ((InetSocketAddress)ch.getLocalAddress()).getPort();
  conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port));
  pipelineFact.SHUFFLE.setPort(port);
  LOG.info(getName() + " listening on port " + port);
  super.serviceStart();

  sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY,
                                  DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE);
  connectionKeepAliveEnabled =
      conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED);
  connectionKeepAliveTimeOut =
      Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT,
        DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT));
  mapOutputMetaInfoCacheSize =
      Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE,
        DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE));
}
 
Example #16
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private boolean testUberDecision(Configuration conf) {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  JobImpl job =
      new JobImpl(jobId, ApplicationAttemptId.newInstance(
        ApplicationId.newInstance(0, 0), 0), conf, mock(EventHandler.class),
        null, new JobTokenSecretManager(), new Credentials(), null, null,
        mrAppMetrics, null, true, null, 0, null, null, null, null);
  InitTransition initTransition = getInitTransition(2);
  JobEvent mockJobEvent = mock(JobEvent.class);
  initTransition.transition(job, mockJobEvent);
  boolean isUber = job.isUber();
  return isUber;
}
 
Example #17
Source File: TestJobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testReportDiagnostics() throws Exception {
  JobID jobID = JobID.forName("job_1234567890000_0001");
  JobId jobId = TypeConverter.toYarn(jobID);
  final String diagMsg = "some diagnostic message";
  final JobDiagnosticsUpdateEvent diagUpdateEvent =
      new JobDiagnosticsUpdateEvent(jobId, diagMsg);
  MRAppMetrics mrAppMetrics = MRAppMetrics.create();
  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  JobImpl job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(diagUpdateEvent);
  String diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));

  job = new JobImpl(jobId, Records
      .newRecord(ApplicationAttemptId.class), new Configuration(),
      mock(EventHandler.class),
      null, mock(JobTokenSecretManager.class), null,
      new SystemClock(), null,
      mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  job.handle(diagUpdateEvent);
  diagnostics = job.getReport().getDiagnostics();
  Assert.assertNotNull(diagnostics);
  Assert.assertTrue(diagnostics.contains(diagMsg));
}
 
Example #18
Source File: TestTaskAttemptListenerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler,
    TaskHeartbeatHandler hbHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
  this.taskHeartbeatHandler = hbHandler;
}
 
Example #19
Source File: MRTask.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private void configureMRTask()
    throws IOException, InterruptedException {

  Credentials credentials = UserGroupInformation.getCurrentUser()
      .getCredentials();
  jobConf.setCredentials(credentials);
  // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
  // the Tez parameter.
  // TODO This could be fetched from the env if YARN is setting it for all
  // Containers.
  // Set it in conf, so as to be able to be used the the OutputCommitter.

  // Not needed. This is probably being set via the source/consumer meta
  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  if (jobToken != null) {
    // Will MR ever run without a job token.
    SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
        .getPassword());
    this.jobTokenSecret = sk;
  } else {
    LOG.warn("No job token set");
  }

  configureLocalDirs();

  // Set up the DistributedCache related configs
  setupDistributedCacheConfig(jobConf);
}
 
Example #20
Source File: MRTask.java    From tez with Apache License 2.0 5 votes vote down vote up
private void configureMRTask()
    throws IOException, InterruptedException {

  Credentials credentials = UserGroupInformation.getCurrentUser()
      .getCredentials();
  jobConf.setCredentials(credentials);
  // TODO Can this be avoided all together. Have the MRTezOutputCommitter use
  // the Tez parameter.
  // TODO This could be fetched from the env if YARN is setting it for all
  // Containers.
  // Set it in conf, so as to be able to be used the the OutputCommitter.

  // Not needed. This is probably being set via the source/consumer meta
  Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
  if (jobToken != null) {
    // Will MR ever run without a job token.
    SecretKey sk = JobTokenSecretManager.createSecretKey(jobToken
        .getPassword());
    this.jobTokenSecret = sk;
  } else {
    LOG.warn("No job token set");
  }

  configureLocalDirs();

  // Set up the DistributedCache related configs
  setupDistributedCacheConfig(jobConf);
}
 
Example #21
Source File: YarnChild.java    From big-c with Apache License 2.0 4 votes vote down vote up
private static void configureTask(JobConf job, Task task,
    Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
  job.setCredentials(credentials);
  
  ApplicationAttemptId appAttemptId =
      ConverterUtils.toContainerId(
          System.getenv(Environment.CONTAINER_ID.name()))
          .getApplicationAttemptId();
  LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
  // Set it in conf, so as to be able to be used the the OutputCommitter.
  job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      appAttemptId.getAttemptId());

  // set tcp nodelay
  job.setBoolean("ipc.client.tcpnodelay", true);
  job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
      YarnOutputFiles.class, MapOutputFile.class);
  // set the jobToken and shuffle secrets into task
  task.setJobTokenSecret(
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
  if (shuffleSecret == null) {
    LOG.warn("Shuffle secret missing from task credentials."
        + " Using job token secret as shuffle secret.");
    shuffleSecret = jt.getPassword();
  }
  task.setShuffleSecret(
      JobTokenSecretManager.createSecretKey(shuffleSecret));

  // setup the child's MRConfig.LOCAL_DIR.
  configureLocalDirs(task, job);

  // setup the child's attempt directories
  // Do the task-type specific localization
  task.localizeConfiguration(job);

  // Set up the DistributedCache related configs
  MRApps.setupDistributedCacheLocal(job);

  // Overwrite the localized task jobconf which is linked to in the current
  // work-dir.
  Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
  writeLocalJobFile(localTaskFile, job);
  task.setJobFile(localTaskFile.toString());
  task.setConf(job);
}
 
Example #22
Source File: Application.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static String createDigest(byte[] password, String data)
    throws IOException {
  SecretKey key = JobTokenSecretManager.createSecretKey(password);
  return SecureShuffleUtils.hashFromString(data, key);
}
 
Example #23
Source File: TestTaskAttemptListenerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test  (timeout=5000)
public void testGetTask() throws IOException {
  AppContext appCtx = mock(AppContext.class);
  JobTokenSecretManager secret = mock(JobTokenSecretManager.class); 
  RMHeartbeatHandler rmHeartbeatHandler =
      mock(RMHeartbeatHandler.class);
  TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
  MockTaskAttemptListenerImpl listener = 
    new MockTaskAttemptListenerImpl(appCtx, secret,
        rmHeartbeatHandler, hbHandler);
  Configuration conf = new Configuration();
  listener.init(conf);
  listener.start();
  JVMId id = new JVMId("foo",1, true, 1);
  WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());

  // Verify ask before registration.
  //The JVM ID has not been registered yet so we should kill it.
  JvmContext context = new JvmContext();
  context.jvmId = id; 
  JvmTask result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  // Verify ask after registration but before launch. 
  // Don't kill, should be null.
  TaskAttemptId attemptID = mock(TaskAttemptId.class);
  Task task = mock(Task.class);
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  result = listener.getTask(context);
  assertNull(result);
  // Unregister for more testing.
  listener.unregister(attemptID, wid);

  // Verify ask after registration and launch
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  listener.registerLaunchedTask(attemptID, wid);
  verify(hbHandler).register(attemptID);
  result = listener.getTask(context);
  assertNotNull(result);
  assertFalse(result.shouldDie);
  // Don't unregister yet for more testing.

  //Verify that if we call it again a second time we are told to die.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.unregister(attemptID, wid);

  // Verify after unregistration.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.stop();

  // test JVMID
  JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
  assertNotNull(jvmid);
  try {
    JVMId.forName("jvm_001_002_m_004_006");
    Assert.fail();
  } catch (IllegalArgumentException e) {
    assertEquals(e.getMessage(),
        "TaskId string : jvm_001_002_m_004_006 is not properly formed");
  }

}
 
Example #24
Source File: TestTaskAttemptListenerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
}
 
Example #25
Source File: JobImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener,
    JobTokenSecretManager jobTokenSecretManager,
    Credentials jobCredentials, Clock clock,
    Map<TaskId, TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics,
    OutputCommitter committer, boolean newApiCommitter, String userName,
    long appSubmitTime, List<AMInfo> amInfos, AppContext appContext,
    JobStateInternal forcedState, String forcedDiagnostic) {
  this.applicationAttemptId = applicationAttemptId;
  this.jobId = jobId;
  this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
  this.conf = new JobConf(conf);
  this.metrics = metrics;
  this.clock = clock;
  this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
  this.amInfos = amInfos;
  this.appContext = appContext;
  this.userName = userName;
  this.queueName = conf.get(MRJobConfig.QUEUE_NAME, "default");
  this.appSubmitTime = appSubmitTime;
  this.oldJobId = TypeConverter.fromYarn(jobId);
  this.committer = committer;
  this.newApiCommitter = newApiCommitter;

  this.taskAttemptListener = taskAttemptListener;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  this.jobCredentials = jobCredentials;
  this.jobTokenSecretManager = jobTokenSecretManager;

  this.aclsManager = new JobACLsManager(conf);
  this.username = System.getProperty("user.name");
  this.jobACLs = aclsManager.constructJobACLs(conf);

  ThreadFactory threadFactory = new ThreadFactoryBuilder()
    .setNameFormat("Job Fail Wait Timeout Monitor #%d")
    .setDaemon(true)
    .build();
  this.executor = new ScheduledThreadPoolExecutor(1, threadFactory);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
  this.forcedState  = forcedState;
  if(forcedDiagnostic != null) {
    this.diagnostics.add(forcedDiagnostic);
  }
  
  this.maxAllowedFetchFailuresFraction = conf.getFloat(
      MRJobConfig.MAX_ALLOWED_FETCH_FAILURES_FRACTION,
      MRJobConfig.DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION);
  this.maxFetchFailuresNotifications = conf.getInt(
      MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS,
      MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS);
}
 
Example #26
Source File: Application.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static String createDigest(byte[] password, String data)
    throws IOException {
  SecretKey key = JobTokenSecretManager.createSecretKey(password);
  return SecureShuffleUtils.hashFromString(data, key);
}
 
Example #27
Source File: TestTaskAttemptListenerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test  (timeout=5000)
public void testGetTask() throws IOException {
  AppContext appCtx = mock(AppContext.class);
  JobTokenSecretManager secret = mock(JobTokenSecretManager.class); 
  RMHeartbeatHandler rmHeartbeatHandler =
      mock(RMHeartbeatHandler.class);
  TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
  MockTaskAttemptListenerImpl listener = 
    new MockTaskAttemptListenerImpl(appCtx, secret,
        rmHeartbeatHandler, hbHandler);
  Configuration conf = new Configuration();
  listener.init(conf);
  listener.start();
  JVMId id = new JVMId("foo",1, true, 1);
  WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());

  // Verify ask before registration.
  //The JVM ID has not been registered yet so we should kill it.
  JvmContext context = new JvmContext();
  context.jvmId = id; 
  JvmTask result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  // Verify ask after registration but before launch. 
  // Don't kill, should be null.
  TaskAttemptId attemptID = mock(TaskAttemptId.class);
  Task task = mock(Task.class);
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  result = listener.getTask(context);
  assertNull(result);
  // Unregister for more testing.
  listener.unregister(attemptID, wid);

  // Verify ask after registration and launch
  //Now put a task with the ID
  listener.registerPendingTask(task, wid);
  listener.registerLaunchedTask(attemptID, wid);
  verify(hbHandler).register(attemptID);
  result = listener.getTask(context);
  assertNotNull(result);
  assertFalse(result.shouldDie);
  // Don't unregister yet for more testing.

  //Verify that if we call it again a second time we are told to die.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.unregister(attemptID, wid);

  // Verify after unregistration.
  result = listener.getTask(context);
  assertNotNull(result);
  assertTrue(result.shouldDie);

  listener.stop();

  // test JVMID
  JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
  assertNotNull(jvmid);
  try {
    JVMId.forName("jvm_001_002_m_004_006");
    Assert.fail();
  } catch (IllegalArgumentException e) {
    assertEquals(e.getMessage(),
        "TaskId string : jvm_001_002_m_004_006 is not properly formed");
  }

}
 
Example #28
Source File: TestTaskAttemptListenerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public MockTaskAttemptListenerImpl(AppContext context,
    JobTokenSecretManager jobTokenSecretManager,
    RMHeartbeatHandler rmHeartbeatHandler) {
  super(context, jobTokenSecretManager, rmHeartbeatHandler, null);
}
 
Example #29
Source File: JobImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
    Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener,
    JobTokenSecretManager jobTokenSecretManager,
    Credentials jobCredentials, Clock clock,
    Map<TaskId, TaskInfo> completedTasksFromPreviousRun, MRAppMetrics metrics,
    OutputCommitter committer, boolean newApiCommitter, String userName,
    long appSubmitTime, List<AMInfo> amInfos, AppContext appContext,
    JobStateInternal forcedState, String forcedDiagnostic) {
  this.applicationAttemptId = applicationAttemptId;
  this.jobId = jobId;
  this.jobName = conf.get(JobContext.JOB_NAME, "<missing job name>");
  this.conf = new JobConf(conf);
  this.metrics = metrics;
  this.clock = clock;
  this.completedTasksFromPreviousRun = completedTasksFromPreviousRun;
  this.amInfos = amInfos;
  this.appContext = appContext;
  this.userName = userName;
  this.queueName = conf.get(MRJobConfig.QUEUE_NAME, "default");
  this.appSubmitTime = appSubmitTime;
  this.oldJobId = TypeConverter.fromYarn(jobId);
  this.committer = committer;
  this.newApiCommitter = newApiCommitter;

  this.taskAttemptListener = taskAttemptListener;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  this.jobCredentials = jobCredentials;
  this.jobTokenSecretManager = jobTokenSecretManager;

  this.aclsManager = new JobACLsManager(conf);
  this.username = System.getProperty("user.name");
  this.jobACLs = aclsManager.constructJobACLs(conf);

  ThreadFactory threadFactory = new ThreadFactoryBuilder()
    .setNameFormat("Job Fail Wait Timeout Monitor #%d")
    .setDaemon(true)
    .build();
  this.executor = new ScheduledThreadPoolExecutor(1, threadFactory);

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
  this.forcedState  = forcedState;
  if(forcedDiagnostic != null) {
    this.diagnostics.add(forcedDiagnostic);
  }
  
  this.maxAllowedFetchFailuresFraction = conf.getFloat(
      MRJobConfig.MAX_ALLOWED_FETCH_FAILURES_FRACTION,
      MRJobConfig.DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION);
  this.maxFetchFailuresNotifications = conf.getInt(
      MRJobConfig.MAX_FETCH_FAILURES_NOTIFICATIONS,
      MRJobConfig.DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS);
}
 
Example #30
Source File: YarnChild.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private static void configureTask(JobConf job, Task task,
    Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
  job.setCredentials(credentials);
  
  ApplicationAttemptId appAttemptId =
      ConverterUtils.toContainerId(
          System.getenv(Environment.CONTAINER_ID.name()))
          .getApplicationAttemptId();
  LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
  // Set it in conf, so as to be able to be used the the OutputCommitter.
  job.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
      appAttemptId.getAttemptId());

  // set tcp nodelay
  job.setBoolean("ipc.client.tcpnodelay", true);
  job.setClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
      YarnOutputFiles.class, MapOutputFile.class);
  // set the jobToken and shuffle secrets into task
  task.setJobTokenSecret(
      JobTokenSecretManager.createSecretKey(jt.getPassword()));
  byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
  if (shuffleSecret == null) {
    LOG.warn("Shuffle secret missing from task credentials."
        + " Using job token secret as shuffle secret.");
    shuffleSecret = jt.getPassword();
  }
  task.setShuffleSecret(
      JobTokenSecretManager.createSecretKey(shuffleSecret));

  // setup the child's MRConfig.LOCAL_DIR.
  configureLocalDirs(task, job);

  // setup the child's attempt directories
  // Do the task-type specific localization
  task.localizeConfiguration(job);

  // Set up the DistributedCache related configs
  MRApps.setupDistributedCacheLocal(job);

  // Overwrite the localized task jobconf which is linked to in the current
  // work-dir.
  Path localTaskFile = new Path(MRJobConfig.JOB_CONF_FILE);
  writeLocalJobFile(localTaskFile, job);
  task.setJobFile(localTaskFile.toString());
  task.setConf(job);
}