org.mortbay.log.Log Java Examples

The following examples show how to use org.mortbay.log.Log. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobEndNotifier.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Notify the URL just once. Use best effort.
 */
protected boolean notifyURLOnce() {
  boolean success = false;
  try {
    Log.info("Job end notification trying " + urlToNotify);
    HttpURLConnection conn =
      (HttpURLConnection) urlToNotify.openConnection(proxyToUse);
    conn.setConnectTimeout(timeout);
    conn.setReadTimeout(timeout);
    conn.setAllowUserInteraction(false);
    if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
      Log.warn("Job end notification to " + urlToNotify +" failed with code: "
      + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
      +"\"");
    }
    else {
      success = true;
      Log.info("Job end notification to " + urlToNotify + " succeeded");
    }
  } catch(IOException ioe) {
    Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
  }
  return success;
}
 
Example #2
Source File: FileUtil.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Change the permissions on a file / directory, recursively, if
 * needed.
 * @param filename name of the file whose permissions are to change
 * @param perm permission string
 * @param recursive true, if permissions should be changed recursively
 * @return the exit code from the command.
 * @throws IOException
 * @throws InterruptedException
 */
public static int chmod(String filename, String perm, boolean recursive)
                          throws IOException, InterruptedException {
  StringBuffer cmdBuf = new StringBuffer();
  cmdBuf.append("chmod ");
  if (recursive) {
    cmdBuf.append("-R ");
  }
  cmdBuf.append(perm).append(" ");
  cmdBuf.append(filename);
  String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()};
  ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd);
  try {
    shExec.execute();
  }catch(IOException e) {
    if(Log.isDebugEnabled()) {
      Log.debug("Error while changing permission : " + filename 
          +" Exception: " + StringUtils.stringifyException(e));
    }
  }
  return shExec.getExitCode();
}
 
Example #3
Source File: IcebergStorage.java    From iceberg with Apache License 2.0 6 votes vote down vote up
private Table load(String location, Job job) throws IOException {
  if(iceberg == null) {
    Class<?> tablesImpl = job.getConfiguration().getClass(PIG_ICEBERG_TABLES_IMPL, HadoopTables.class);
    Log.info("Initializing iceberg tables implementation: " + tablesImpl);
    iceberg = (Tables) ReflectionUtils.newInstance(tablesImpl, job.getConfiguration());
  }

  Table result = tables.get(location);

  if (result == null) {
    try {
      LOG.info(format("[%s]: Loading table for location: %s", signature, location));
      result = iceberg.load(location);
      tables.put(location, result);
    } catch (Exception e) {
      throw new FrontendException("Failed to instantiate tables implementation", e);
    }
  }

  return result;
}
 
Example #4
Source File: JettyConfiguration.java    From pushfish-android with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Set up the classloader for the webapp, using the various parts of the Maven project
 *
 * @see org.mortbay.jetty.webapp.Configuration#configureClassLoader()
 */
public void configureClassLoader() throws Exception {
    if (classPathFiles != null) {
        Log.debug("Setting up classpath ...");

        //put the classes dir and all dependencies into the classpath
        for (File classPathFile : classPathFiles) {
            ((WebAppClassLoader) getWebAppContext().getClassLoader()).addClassPath(
                    classPathFile.getCanonicalPath());
        }

        if (Log.isDebugEnabled()) {
            Log.debug("Classpath = " + LazyList.array2List(
                    ((URLClassLoader) getWebAppContext().getClassLoader()).getURLs()));
        }
    } else {
        super.configureClassLoader();
    }
}
 
Example #5
Source File: JettyConfiguration.java    From pushfish-android with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Set up the classloader for the webapp, using the various parts of the Maven project
 *
 * @see org.mortbay.jetty.webapp.Configuration#configureClassLoader()
 */
public void configureClassLoader() throws Exception {
    if (classPathFiles != null) {
        Log.debug("Setting up classpath ...");

        //put the classes dir and all dependencies into the classpath
        for (File classPathFile : classPathFiles) {
            ((WebAppClassLoader) getWebAppContext().getClassLoader()).addClassPath(
                    classPathFile.getCanonicalPath());
        }

        if (Log.isDebugEnabled()) {
            Log.debug("Classpath = " + LazyList.array2List(
                    ((URLClassLoader) getWebAppContext().getClassLoader()).getURLs()));
        }
    } else {
        super.configureClassLoader();
    }
}
 
Example #6
Source File: PcapHelper.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
/**
 * Builds the reverseKey to fetch the pcaps in the reverse traffic
 * (destination to source).
 * 
 * @param key
 *          indicates hbase rowKey (partial or full) in the format
 *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
 * @return String indicates the key in the format
 *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
 */
public static String reverseKey(String key) {
  Assert.hasText(key, "key must not be null or empty");
  String delimeter = HBaseConfigConstants.PCAP_KEY_DELIMETER;
  String regex = "\\" + delimeter;
  StringBuffer sb = new StringBuffer();
  try {
    String[] tokens = key.split(regex);
    Assert
        .isTrue(
            (tokens.length == 5 || tokens.length == 6 || tokens.length == 7),
            "key is not in the format : 'srcAddr-dstAddr-protocol-srcPort-dstPort-{ipId-fragment identifier}'");
    sb.append(tokens[1]).append(delimeter).append(tokens[0])
        .append(delimeter).append(tokens[2]).append(delimeter)
        .append(tokens[4]).append(delimeter).append(tokens[3]);
  } catch (Exception e) {
    Log.warn("Failed to reverse the key. Reverse scan won't be performed.", e);
  }
  return sb.toString();
}
 
Example #7
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example #8
Source File: JobEndNotifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Notify the URL just once. Use best effort.
 */
protected boolean notifyURLOnce() {
  boolean success = false;
  try {
    Log.info("Job end notification trying " + urlToNotify);
    HttpURLConnection conn =
      (HttpURLConnection) urlToNotify.openConnection(proxyToUse);
    conn.setConnectTimeout(timeout);
    conn.setReadTimeout(timeout);
    conn.setAllowUserInteraction(false);
    if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
      Log.warn("Job end notification to " + urlToNotify +" failed with code: "
      + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
      +"\"");
    }
    else {
      success = true;
      Log.info("Job end notification to " + urlToNotify + " succeeded");
    }
  } catch(IOException ioe) {
    Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
  }
  return success;
}
 
Example #9
Source File: PcapHelper.java    From opensoc-streaming with Apache License 2.0 6 votes vote down vote up
/**
 * Builds the reverseKey to fetch the pcaps in the reverse traffic
 * (destination to source).
 * 
 * @param key
 *          indicates hbase rowKey (partial or full) in the format
 *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
 * @return String indicates the key in the format
 *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
 */
public static String reverseKey(String key) {
  Assert.hasText(key, "key must not be null or empty");
  String delimeter = HBaseConfigConstants.PCAP_KEY_DELIMETER;
  String regex = "\\" + delimeter;
  StringBuffer sb = new StringBuffer();
  try {
    String[] tokens = key.split(regex);
    Assert
        .isTrue(
            (tokens.length == 5 || tokens.length == 6 || tokens.length == 7),
            "key is not in the format : 'srcAddr-dstAddr-protocol-srcPort-dstPort-{ipId-fragment identifier}'");
    sb.append(tokens[1]).append(delimeter).append(tokens[0])
        .append(delimeter).append(tokens[2]).append(delimeter)
        .append(tokens[4]).append(delimeter).append(tokens[3]);
  } catch (Exception e) {
    Log.warn("Failed to reverse the key. Reverse scan won't be performed.", e);
  }
  return sb.toString();
}
 
Example #10
Source File: EmbeddedGithubJsonToParquet.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void downloadFile(String fileUrl, Path destination) {
  if (destination.toFile().exists()) {
    Log.info(String.format("Skipping download for %s at %s because destination already exists", fileUrl,
        destination.toString()));
    return;
  }

  try {
    URL archiveUrl = new URL(fileUrl);
    ReadableByteChannel rbc = Channels.newChannel(archiveUrl.openStream());
    FileOutputStream fos = new FileOutputStream(String.valueOf(destination));
    Log.info(String.format("Downloading %s at %s", fileUrl, destination.toString()));
    fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
    Log.info(String.format("Download complete for %s at %s", fileUrl, destination.toString()));
  } catch (IOException e) {
    e.printStackTrace();
  }
}
 
Example #11
Source File: MRCompactorJobRunner.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void moveTmpPathToOutputPath() throws IOException {
  Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig);

  LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath()));

  this.fs.delete(this.dataset.outputPath(), true);

  if (this.isRetryEnabled) {
    try {
      retryer.call(() -> {
        if (fs.exists(this.dataset.outputPath())) {
          throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more.");
        }
        return null;
      });
    } catch (Exception e) {
      throw new IOException(e);
    }
  }

  WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig);

  Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath());
  HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ;
}
 
Example #12
Source File: JettyConfiguration.java    From Pushjet-Android with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Set up the classloader for the webapp, using the various parts of the Maven project
 *
 * @see org.mortbay.jetty.webapp.Configuration#configureClassLoader()
 */
public void configureClassLoader() throws Exception {
    if (classPathFiles != null) {
        Log.debug("Setting up classpath ...");

        //put the classes dir and all dependencies into the classpath
        for (File classPathFile : classPathFiles) {
            ((WebAppClassLoader) getWebAppContext().getClassLoader()).addClassPath(
                    classPathFile.getCanonicalPath());
        }

        if (Log.isDebugEnabled()) {
            Log.debug("Classpath = " + LazyList.array2List(
                    ((URLClassLoader) getWebAppContext().getClassLoader()).getURLs()));
        }
    } else {
        super.configureClassLoader();
    }
}
 
Example #13
Source File: JettyConfiguration.java    From Pushjet-Android with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Set up the classloader for the webapp, using the various parts of the Maven project
 *
 * @see org.mortbay.jetty.webapp.Configuration#configureClassLoader()
 */
public void configureClassLoader() throws Exception {
    if (classPathFiles != null) {
        Log.debug("Setting up classpath ...");

        //put the classes dir and all dependencies into the classpath
        for (File classPathFile : classPathFiles) {
            ((WebAppClassLoader) getWebAppContext().getClassLoader()).addClassPath(
                    classPathFile.getCanonicalPath());
        }

        if (Log.isDebugEnabled()) {
            Log.debug("Classpath = " + LazyList.array2List(
                    ((URLClassLoader) getWebAppContext().getClassLoader()).getURLs()));
        }
    } else {
        super.configureClassLoader();
    }
}
 
Example #14
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example #15
Source File: MasterTask.java    From reef with Apache License 2.0 5 votes vote down vote up
@Override
public byte[] call(final byte[] memento) throws Exception {

  final Vector model = new DenseVector(dimensions);
  final long time1 = System.currentTimeMillis();
  final int numIters = 10;

  for (int i = 0; i < numIters; i++) {

    controlMessageBroadcaster.send(ControlMessages.ReceiveModel);
    modelBroadcaster.send(model);
    modelReceiveAckReducer.reduce();

    final GroupChanges changes = communicationGroupClient.getTopologyChanges();
    if (changes.exist()) {
      Log.info("There exist topology changes. Asking to update Topology");
      communicationGroupClient.updateTopology();
    } else {
      Log.info("No changes in topology exist. So not updating topology");
    }
  }

  final long time2 = System.currentTimeMillis();
  LOG.log(Level.FINE, "Broadcasting vector of dimensions {0} took {1} secs",
      new Object[]{dimensions, (time2 - time1) / (numIters * 1000.0)});

  controlMessageBroadcaster.send(ControlMessages.Stop);

  return null;
}
 
Example #16
Source File: HBaseConfigurationUtil.java    From opensoc-streaming with Apache License 2.0 5 votes vote down vote up
/**
 * Adds the shutdown hook.
 */
private static void addShutdownHook() {
  Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
    public void run() {
      System.out
          .println("Executing ShutdownHook HBaseConfigurationUtil : Closing HConnection");
      try {
        clusterConnection.close();
      } catch (IOException e) {
        Log.debug("Caught ignorable exception ", e);
      }
    }
  }, "HBaseConfigurationUtilShutDown"));
}
 
Example #17
Source File: TestAllLoader.java    From spork with Apache License 2.0 5 votes vote down vote up
/**
 * Validates that the loadAlias can read the correct amount of records
 * 
 * @param server
 * @param loadAlias
 * @throws IOException
 */
private void readRecordsFromLoader(PigServer server, String loadAlias,
        int totalRowCount) throws IOException {

    Iterator<Tuple> result = server.openIterator(loadAlias);
    int count = 0;

    while ((result.next()) != null) {
        count++;
    }

    Log.info("Validating expected: " + totalRowCount + " against " + count);
    assertEquals(totalRowCount, count);
}
 
Example #18
Source File: HBaseConfigurationUtil.java    From opensoc-streaming with Apache License 2.0 5 votes vote down vote up
/**
 * Closes the underlying connection to cluster; ignores if any exception is
 * thrown.
 */
public static void closeConnection() {
  if (clusterConnection != null) {
    try {
      clusterConnection.close();
    } catch (IOException e) {
      Log.debug("Caught ignorable exception ", e);
    }
  }
}
 
Example #19
Source File: WebServer.java    From tracing-framework with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    response.setContentType("text/json");
    response.setStatus(HttpServletResponse.SC_OK);
    String uri = request.getRequestURI();
    int pathLen = request.getServletPath().length() + 1;
    String taskIdString = uri.length() > pathLen ? uri.substring(pathLen) : null;
    String[] taskIds = taskIdString.split(",");

    Writer out = response.getWriter();
    out.write("[");
    boolean firstTaskDone = false;
    int count = 0;
    for (String taskId : taskIds) {
        Log.info("Writing task " + count++ + ": " + taskId);

        if (firstTaskDone)
            out.write("\n,");
        firstTaskDone = true;

        out.append("{\"id\":\"");
        out.append(taskId);
        out.append("\",\"reports\":[");

        boolean firstReportDone = false;
        Iterator<Report> iter = data.getReports(taskId);
        while (iter.hasNext()) {
            if (firstReportDone)
                out.append(",\n");
            out.append(iter.next().jsonRepr().toJSONString());
            firstReportDone = true;
        }

        out.append("]}");
        Log.info("... done");
    }
    out.write("]");
}
 
Example #20
Source File: SchemaTuple.java    From spork with Apache License 2.0 5 votes vote down vote up
protected static Schema staticSchemaGen(String s) {
    try {
        if (s.equals("")) {
            Log.warn("No Schema present in SchemaTuple generated class");
            return new Schema();
        }
        return (Schema) ObjectSerializer.deserialize(s);
    } catch (IOException e) {
        throw new RuntimeException("Unable to deserialize serialized Schema: " + s, e);
    }
}
 
Example #21
Source File: ViewFsTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void linkUpFirstComponents(Configuration conf, String path,
    FileContext fsTarget, String info) {
  int indexOfEnd = path.indexOf('/', 1);
  if (Shell.WINDOWS) {
    indexOfEnd = path.indexOf('/', indexOfEnd + 1);
  }
  String firstComponent = path.substring(0, indexOfEnd);
  URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri();
  ConfigUtil.addLink(conf, firstComponent, linkTarget);
  Log.info("Added link for " + info + " " 
      + firstComponent + "->" + linkTarget);    
}
 
Example #22
Source File: ViewFsTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void setUpHomeDir(Configuration conf, FileContext fsTarget) {
  String homeDir = fsTarget.getHomeDirectory().toUri().getPath();
  int indexOf2ndSlash = homeDir.indexOf('/', 1);
  if (indexOf2ndSlash >0) {
    linkUpFirstComponents(conf, homeDir, fsTarget, "home dir");
  } else { // home dir is at root. Just link the home dir itse
    URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri();
    ConfigUtil.addLink(conf, homeDir, linkTarget);
    Log.info("Added link for home dir " + homeDir + "->" + linkTarget);
  }
  // Now set the root of the home dir for viewfs
  String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath();
  ConfigUtil.setHomeDirConf(conf, homeDirRoot);
  Log.info("Home dir base for viewfs" + homeDirRoot);  
}
 
Example #23
Source File: ViewFsTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
static public FileContext setupForViewFsLocalFs(FileContextTestHelper helper) throws Exception {
  /**
   * create the test root on local_fs - the  mount table will point here
   */
  FileContext fsTarget = FileContext.getLocalFSFileContext();
  Path targetOfTests = helper.getTestRootPath(fsTarget);
  // In case previous test was killed before cleanup
  fsTarget.delete(targetOfTests, true);
  
  fsTarget.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
  Configuration conf = new Configuration();
  
  // Set up viewfs link for test dir as described above
  String testDir = helper.getTestRootPath(fsTarget).toUri()
      .getPath();
  linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
  
  
  // Set up viewfs link for home dir as described above
  setUpHomeDir(conf, fsTarget);
    
  // the test path may be relative to working dir - we need to make that work:
  // Set up viewfs link for wd as described above
  String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
  linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
  
  FileContext fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
  fc.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
  Log.info("Working dir is: " + fc.getWorkingDirectory());
  //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
  //System.out.println("TargetOfTests = "+ targetOfTests.toUri());
  return fc;
}
 
Example #24
Source File: ViewFileSystemTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void linkUpFirstComponents(Configuration conf, String path, FileSystem fsTarget, String info) {
  int indexOfEnd = path.indexOf('/', 1);
  if (Shell.WINDOWS) {
    indexOfEnd = path.indexOf('/', indexOfEnd + 1);
  }
  String firstComponent = path.substring(0, indexOfEnd);
  URI linkTarget = fsTarget.makeQualified(new Path(firstComponent)).toUri();
  ConfigUtil.addLink(conf, firstComponent, linkTarget);
  Log.info("Added link for " + info + " " 
      + firstComponent + "->" + linkTarget);    
}
 
Example #25
Source File: SentryPolicyProviderForDb.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
public void write(File file) throws Exception {
  super.write(file);
  if (!usingSentryService()) {
    return;
  }

  // remove existing metadata
  for (TSentryRole tRole : sentryClient.listRoles(StaticUserGroup.ADMIN1)) {
    sentryClient.dropRole(StaticUserGroup.ADMIN1, tRole.getRoleName());
  }

  // create roles and add privileges
  for (Entry<String, Collection<String>> roleEntry : rolesToPermissions
      .asMap().entrySet()) {
    sentryClient.createRole(StaticUserGroup.ADMIN1, roleEntry.getKey());
    for (String privilege : roleEntry.getValue()) {
      addPrivilege(roleEntry.getKey(), privilege);
    }
  }

  // grant roles to groups
  for (Entry<String, Collection<String>> groupEntry : groupsToRoles.asMap()
      .entrySet()) {
    for (String roleNames : groupEntry.getValue()) {
      for (String roleName : roleNames.split(",")) {
        try {
          sentryClient
              .grantRoleToGroup(StaticUserGroup.ADMIN1, groupEntry.getKey(), roleName);
        } catch (SentryUserException e) {
          Log.warn("Error granting role " + roleName + " to group "
              + groupEntry.getKey());
        }
      }
    }
  }
}
 
Example #26
Source File: ViewFileSystemTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
static void setUpHomeDir(Configuration conf, FileSystem fsTarget) {
  String homeDir = fsTarget.getHomeDirectory().toUri().getPath();
  int indexOf2ndSlash = homeDir.indexOf('/', 1);
  if (indexOf2ndSlash >0) {
    linkUpFirstComponents(conf, homeDir, fsTarget, "home dir");
  } else { // home dir is at root. Just link the home dir itse
    URI linkTarget = fsTarget.makeQualified(new Path(homeDir)).toUri();
    ConfigUtil.addLink(conf, homeDir, linkTarget);
    Log.info("Added link for home dir " + homeDir + "->" + linkTarget);
  }
  // Now set the root of the home dir for viewfs
  String homeDirRoot = fsTarget.getHomeDirectory().getParent().toUri().getPath();
  ConfigUtil.setHomeDirConf(conf, homeDirRoot);
  Log.info("Home dir base for viewfs" + homeDirRoot);  
}
 
Example #27
Source File: ViewFileSystemTestSetup.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * 
 * @param fsTarget - the target fs of the view fs.
 * @return return the ViewFS File context to be used for tests
 * @throws Exception
 */
static public FileSystem setupForViewFileSystem(Configuration conf, FileSystemTestHelper fileSystemTestHelper, FileSystem fsTarget) throws Exception {
  /**
   * create the test root on local_fs - the  mount table will point here
   */
  Path targetOfTests = fileSystemTestHelper.getTestRootPath(fsTarget);
  // In case previous test was killed before cleanup
  fsTarget.delete(targetOfTests, true);
  fsTarget.mkdirs(targetOfTests);


  // Set up viewfs link for test dir as described above
  String testDir = fileSystemTestHelper.getTestRootPath(fsTarget).toUri()
      .getPath();
  linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
  
  
  // Set up viewfs link for home dir as described above
  setUpHomeDir(conf, fsTarget);
  
  
  // the test path may be relative to working dir - we need to make that work:
  // Set up viewfs link for wd as described above
  String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
  linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");


  FileSystem fsView = FileSystem.get(FsConstants.VIEWFS_URI, conf);
  fsView.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
  Log.info("Working dir is: " + fsView.getWorkingDirectory());
  return fsView;
}
 
Example #28
Source File: AbstractApplicationMaster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 
Example #29
Source File: INodeReference.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
    final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  // since WithName node resides in deleted list acting as a snapshot copy,
  // the parameter snapshot must be non-null
  Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
  // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
  // previous WithName instance
  if (prior == Snapshot.NO_SNAPSHOT_ID) {
    prior = getPriorSnapshot(this);
  }
  
  if (prior != Snapshot.NO_SNAPSHOT_ID
      && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
    return new QuotaCounts.Builder().build();
  }

  QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior,
      collectedBlocks, removedINodes);
  INodeReference ref = getReferredINode().getParentReference();
  if (ref != null) {
    try {
      ref.addSpaceConsumed(counts.negation(), true);
    } catch (QuotaExceededException e) {
      Log.warn("Should not have QuotaExceededException");
    }
  }
  
  if (snapshot < lastSnapshotId) {
    // for a WithName node, when we compute its quota usage, we only count
    // in all the nodes existing at the time of the corresponding rename op.
    // Thus if we are deleting a snapshot before/at the snapshot associated 
    // with lastSnapshotId, we do not need to update the quota upwards.
    counts = new QuotaCounts.Builder().build();
  }
  return counts;
}
 
Example #30
Source File: JettyConfiguration.java    From Pushjet-Android with BSD 2-Clause "Simplified" License 5 votes vote down vote up
protected URL findWebXml() throws IOException {
    //if an explicit web.xml file has been set (eg for jetty:run) then use it
    if (webXmlFile != null && webXmlFile.exists()) {
        return webXmlFile.toURI().toURL();
    }

    //if we haven't overridden location of web.xml file, use the
    //standard way of finding it
    Log.debug("Looking for web.xml file in WEB-INF");
    return super.findWebXml();
}