Java Code Examples for org.apache.hadoop.fs.FileUtil#setExecutable()

The following examples show how to use org.apache.hadoop.fs.FileUtil#setExecutable() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private void setupMockExecutor(String executorPath, Configuration conf)
    throws IOException {
  //we'll always use the tmpMockExecutor - since
  // PrivilegedOperationExecutor can only be initialized once.

  Files.copy(Paths.get(executorPath), Paths.get(tmpMockExecutor),
      REPLACE_EXISTING);

  File executor = new File(tmpMockExecutor);

  if (!FileUtil.canExecute(executor)) {
    FileUtil.setExecutable(executor, true);
  }
    String executorAbsolutePath = executor.getAbsolutePath();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH,
      executorAbsolutePath);
}
 
Example 2
Source Project: hadoop   File: TestShell.java    License: Apache License 2.0 6 votes vote down vote up
public void testShellCommandTimeout() throws Throwable {
  if(Shell.WINDOWS) {
    // setExecutable does not work on Windows
    return;
  }
  String rootDir = new File(System.getProperty(
      "test.build.data", "/tmp")).getAbsolutePath();
  File shellFile = new File(rootDir, "timeout.sh");
  String timeoutCommand = "sleep 4; echo \"hello\"";
  PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
  writer.println(timeoutCommand);
  writer.close();
  FileUtil.setExecutable(shellFile, true);
  Shell.ShellCommandExecutor shexc 
  = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
                                    null, null, 100);
  try {
    shexc.execute();
  } catch (Exception e) {
    //When timing out exception is thrown.
  }
  shellFile.delete();
  assertTrue("Script didnt not timeout" , shexc.isTimedOut());
}
 
Example 3
@Test (timeout = 5000)
public void testContainerLaunchWithPriority() throws IOException {

  // set the scheduler priority to make sure still works with nice -n prio
  File f = new File("./src/test/resources/mock-container-executor");
  if (!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  conf.setInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 2);

  mockExec.setConf(conf);
  List<String> command = new ArrayList<String>();
  mockExec.addSchedPriorityCommand(command);
  assertEquals("first should be nice", "nice", command.get(0));
  assertEquals("second should be -n", "-n", command.get(1));
  assertEquals("third should be the priority", Integer.toString(2),
               command.get(2));

  testContainerLaunch();
}
 
Example 4
Source Project: big-c   File: TestShell.java    License: Apache License 2.0 6 votes vote down vote up
public void testShellCommandTimeout() throws Throwable {
  if(Shell.WINDOWS) {
    // setExecutable does not work on Windows
    return;
  }
  String rootDir = new File(System.getProperty(
      "test.build.data", "/tmp")).getAbsolutePath();
  File shellFile = new File(rootDir, "timeout.sh");
  String timeoutCommand = "sleep 4; echo \"hello\"";
  PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
  writer.println(timeoutCommand);
  writer.close();
  FileUtil.setExecutable(shellFile, true);
  Shell.ShellCommandExecutor shexc 
  = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
                                    null, null, 100);
  try {
    shexc.execute();
  } catch (Exception e) {
    //When timing out exception is thrown.
  }
  shellFile.delete();
  assertTrue("Script didnt not timeout" , shexc.isTimedOut());
}
 
Example 5
private Configuration getConfigurationWithMockContainerExecutor() {
  File f = new File("./src/test/resources/mock-container-executor");
  if(!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  return conf;
}
 
Example 6
Source Project: hadoop   File: TestContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that script exists with non-zero exit code when command fails.
 * @throws IOException
 */
@Test (timeout = 10000)
public void testShellScriptBuilderNonZeroExitCode() throws IOException {
  ShellScriptBuilder builder = ShellScriptBuilder.create();
  builder.command(Arrays.asList(new String[] {"unknownCommand"}));
  File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError");
  PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
  builder.write(writer);
  writer.close();
  try {
    FileUtil.setExecutable(shellFile, true);

    Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
        new String[]{shellFile.getAbsolutePath()}, tmpDir);
    try {
      shexc.execute();
      fail("builder shell command was expected to throw");
    }
    catch(IOException e) {
      // expected
      System.out.println("Received an expected exception: " + e.getMessage());
    }
  }
  finally {
    FileUtil.fullyDelete(shellFile);
  }
}
 
Example 7
@Before
public void setup() {
  assumeTrue(Shell.LINUX);
  File f = new File("./src/test/resources/mock-container-executor");
  if(!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
  Configuration conf = new Configuration();
  yarnImage = "yarnImage";
  long time = System.currentTimeMillis();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, "/tmp/nm-local-dir" + time);
  conf.set(YarnConfiguration.NM_LOG_DIRS, "/tmp/userlogs" + time);
  conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage);
  conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME , DOCKER_LAUNCH_COMMAND);
  dockerContainerExecutor = new DockerContainerExecutor();
  dirsHandler = new LocalDirsHandlerService();
  dirsHandler.init(conf);
  dockerContainerExecutor.setConf(conf);
  lfs = null;
  try {
    lfs = FileContext.getLocalFSFileContext();
    workDir = new Path("/tmp/temp-"+ System.currentTimeMillis());
    lfs.mkdir(workDir, FsPermission.getDirDefault(), true);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }

}
 
Example 8
Source Project: big-c   File: TestContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that script exists with non-zero exit code when command fails.
 * @throws IOException
 */
@Test (timeout = 10000)
public void testShellScriptBuilderNonZeroExitCode() throws IOException {
  ShellScriptBuilder builder = ShellScriptBuilder.create();
  builder.command(Arrays.asList(new String[] {"unknownCommand"}));
  File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError");
  PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
  builder.write(writer);
  writer.close();
  try {
    FileUtil.setExecutable(shellFile, true);

    Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
        new String[]{shellFile.getAbsolutePath()}, tmpDir);
    try {
      shexc.execute();
      fail("builder shell command was expected to throw");
    }
    catch(IOException e) {
      // expected
      System.out.println("Received an expected exception: " + e.getMessage());
    }
  }
  finally {
    FileUtil.fullyDelete(shellFile);
  }
}
 
Example 9
@Before
public void setup() {
  assumeTrue(!Path.WINDOWS);
  File f = new File("./src/test/resources/mock-container-executor");
  if(!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  mockExec = new LinuxContainerExecutor();
  dirsHandler = new LocalDirsHandlerService();
  dirsHandler.init(conf);
  mockExec.setConf(conf);
}
 
Example 10
@Before
public void setup() {
  assumeTrue(Shell.LINUX);
  File f = new File("./src/test/resources/mock-container-executor");
  if(!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
  Configuration conf = new Configuration();
  yarnImage = "yarnImage";
  long time = System.currentTimeMillis();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, "/tmp/nm-local-dir" + time);
  conf.set(YarnConfiguration.NM_LOG_DIRS, "/tmp/userlogs" + time);
  conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage);
  conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME , DOCKER_LAUNCH_COMMAND);
  dockerContainerExecutor = new DockerContainerExecutor();
  dirsHandler = new LocalDirsHandlerService();
  dirsHandler.init(conf);
  dockerContainerExecutor.setConf(conf);
  lfs = null;
  try {
    lfs = FileContext.getLocalFSFileContext();
    workDir = new Path("/tmp/temp-"+ System.currentTimeMillis());
    lfs.mkdir(workDir, FsPermission.getDirDefault(), true);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }

}
 
Example 11
Source Project: hadoop   File: TestContainerLaunch.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException  {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile);
    FileUtil.setExecutable(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    // invalid env
    env.put(
        "APPLICATION_WORKFLOW_CONTEXT", "{\"workflowId\":\"609f91c5cd83\"," +
        "\"workflowName\":\"\n\ninsert table " +
        "\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
    List<String> commands = new ArrayList<String>();
    new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    // It is supposed that LANG is set as C.
    Map<String, String> cmdEnv = new HashMap<String, String>();
    cmdEnv.put("LANG", "C");
    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
      tmpDir, cmdEnv);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
        "is not recognized as an internal or external command" :
        "command not found"));
    Assert.assertTrue(shexc.getExitCode() != 0);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 
Example 12
Source Project: hadoop   File: TestContainerLaunch.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 20000)
public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    // echo "hello" to stdout and "error" to stderr and exit code with 2;
    String command = Shell.WINDOWS ?
        "@echo \"hello\" & @echo \"error\" 1>&2 & exit /b 2" :
        "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
    PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
    FileUtil.setExecutable(shellFile, true);
    writer.println(command);
    writer.close();
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    List<String> commands = new ArrayList<String>();
    commands.add(command);
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, tmpDir);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    // test stderr
    Assert.assertTrue(diagnostics.contains("error"));
    // test stdout
    Assert.assertTrue(shexc.getOutput().contains("hello"));
    Assert.assertTrue(shexc.getExitCode() == 2);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 
Example 13
Source Project: hadoop   File: TestCheckpoint.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test that, if a storage directory is failed when a checkpoint occurs,
 * the non-failed storage directory receives the checkpoint.
 */
@Test
public void testCheckpointWithFailedStorageDir() throws Exception {
  MiniDFSCluster cluster = null;
  SecondaryNameNode secondary = null;
  File currentDir = null;
  
  Configuration conf = new HdfsConfiguration();

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
        .format(true).build();

    secondary = startSecondaryNameNode(conf);

    // Checkpoint once
    secondary.doCheckpoint();

    // Now primary NN experiences failure of a volume -- fake by
    // setting its current dir to a-x permissions
    NamenodeProtocols nn = cluster.getNameNodeRpc();
    NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
    StorageDirectory sd0 = storage.getStorageDir(0);
    StorageDirectory sd1 = storage.getStorageDir(1);
    
    currentDir = sd0.getCurrentDir();
    FileUtil.setExecutable(currentDir, false);

    // Upload checkpoint when NN has a bad storage dir. This should
    // succeed and create the checkpoint in the good dir.
    secondary.doCheckpoint();
    
    GenericTestUtils.assertExists(
        new File(sd1.getCurrentDir(), NNStorage.getImageFileName(2)));
    
    // Restore the good dir
    FileUtil.setExecutable(currentDir, true);
    nn.restoreFailedStorage("true");
    nn.rollEditLog();

    // Checkpoint again -- this should upload to both dirs
    secondary.doCheckpoint();
    
    assertNNHasCheckpoints(cluster, ImmutableList.of(8));
    assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
  } finally {
    if (currentDir != null) {
      FileUtil.setExecutable(currentDir, true);
    }
    cleanup(secondary);
    secondary = null;
    cleanup(cluster);
    cluster = null;
  }
}
 
Example 14
Source Project: big-c   File: TestContainerLaunch.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException  {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile);
    FileUtil.setExecutable(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    // invalid env
    env.put(
        "APPLICATION_WORKFLOW_CONTEXT", "{\"workflowId\":\"609f91c5cd83\"," +
        "\"workflowName\":\"\n\ninsert table " +
        "\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
    List<String> commands = new ArrayList<String>();
    new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    // It is supposed that LANG is set as C.
    Map<String, String> cmdEnv = new HashMap<String, String>();
    cmdEnv.put("LANG", "C");
    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
      tmpDir, cmdEnv);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
        "is not recognized as an internal or external command" :
        "command not found"));
    Assert.assertTrue(shexc.getExitCode() != 0);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 
Example 15
Source Project: big-c   File: TestContainerLaunch.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 20000)
public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    // echo "hello" to stdout and "error" to stderr and exit code with 2;
    String command = Shell.WINDOWS ?
        "@echo \"hello\" & @echo \"error\" 1>&2 & exit /b 2" :
        "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
    PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
    FileUtil.setExecutable(shellFile, true);
    writer.println(command);
    writer.close();
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    List<String> commands = new ArrayList<String>();
    commands.add(command);
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, tmpDir);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    // test stderr
    Assert.assertTrue(diagnostics.contains("error"));
    // test stdout
    Assert.assertTrue(shexc.getOutput().contains("hello"));
    Assert.assertTrue(shexc.getExitCode() == 2);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 
Example 16
@Test
public void testDeleteAsUser() throws IOException {
  String appSubmitter = "nobody";
  String cmd = String.valueOf(
      LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue());
  Path dir = new Path("/tmp/testdir");
  Path testFile = new Path("testfile");
  Path baseDir0 = new Path("/grid/0/BaseDir");
  Path baseDir1 = new Path("/grid/1/BaseDir");

  mockExec.deleteAsUser(appSubmitter, dir);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, "/tmp/testdir"),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, null);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, ""),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
      readMockParams());

  File f = new File("./src/test/resources/mock-container-executer-with-error");
  if (!FileUtil.canExecute(f)) {
    FileUtil.setExecutable(f, true);
  }
  String executorPath = f.getAbsolutePath();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
  mockExec.setConf(conf);

  mockExec.deleteAsUser(appSubmitter, dir);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, "/tmp/testdir"),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, null);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, ""),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
      readMockParams());

  mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
  assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
      appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
      readMockParams());
}
 
Example 17
Source Project: big-c   File: TestCheckpoint.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test that, if a storage directory is failed when a checkpoint occurs,
 * the non-failed storage directory receives the checkpoint.
 */
@Test
public void testCheckpointWithFailedStorageDir() throws Exception {
  MiniDFSCluster cluster = null;
  SecondaryNameNode secondary = null;
  File currentDir = null;
  
  Configuration conf = new HdfsConfiguration();

  try {
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
        .format(true).build();

    secondary = startSecondaryNameNode(conf);

    // Checkpoint once
    secondary.doCheckpoint();

    // Now primary NN experiences failure of a volume -- fake by
    // setting its current dir to a-x permissions
    NamenodeProtocols nn = cluster.getNameNodeRpc();
    NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
    StorageDirectory sd0 = storage.getStorageDir(0);
    StorageDirectory sd1 = storage.getStorageDir(1);
    
    currentDir = sd0.getCurrentDir();
    FileUtil.setExecutable(currentDir, false);

    // Upload checkpoint when NN has a bad storage dir. This should
    // succeed and create the checkpoint in the good dir.
    secondary.doCheckpoint();
    
    GenericTestUtils.assertExists(
        new File(sd1.getCurrentDir(), NNStorage.getImageFileName(2)));
    
    // Restore the good dir
    FileUtil.setExecutable(currentDir, true);
    nn.restoreFailedStorage("true");
    nn.rollEditLog();

    // Checkpoint again -- this should upload to both dirs
    secondary.doCheckpoint();
    
    assertNNHasCheckpoints(cluster, ImmutableList.of(8));
    assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
  } finally {
    if (currentDir != null) {
      FileUtil.setExecutable(currentDir, true);
    }
    cleanup(secondary);
    secondary = null;
    cleanup(cluster);
    cluster = null;
  }
}