Java Code Examples for org.apache.hadoop.util.StringUtils

The following are top voted examples for showing how to use org.apache.hadoop.util.StringUtils. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: aliyun-maxcompute-data-collectors   File: MSSQLTestUtils.java   Source Code and License 6 votes vote down vote up
public void createTableFromSQL(String sql) throws SQLException {
  Connection dbcon = this.getConnection();

  System.out.println("SQL : " + sql);
  this.dropTableIfExists("TPCH1M_LINEITEM");

  try {
    Statement st = dbcon.createStatement();
    int res = st.executeUpdate(sql);
    System.out.println("Result : " + res);

  } catch (SQLException e) {
    LOG.error("Got SQLException during creating table: " + StringUtils.stringifyException(e));
  }

}
 
Example 2
Project: ditb   File: MultiThreadedUpdaterWithACL.java   Source Code and License 6 votes vote down vote up
private void recordFailure(final Mutation m, final long keyBase,
    final long start, IOException e) {
  failedKeySet.add(keyBase);
  String exceptionInfo;
  if (e instanceof RetriesExhaustedWithDetailsException) {
    RetriesExhaustedWithDetailsException aggEx = (RetriesExhaustedWithDetailsException) e;
    exceptionInfo = aggEx.getExhaustiveDescription();
  } else {
    StringWriter stackWriter = new StringWriter();
    PrintWriter pw = new PrintWriter(stackWriter);
    e.printStackTrace(pw);
    pw.flush();
    exceptionInfo = StringUtils.stringifyException(e);
  }
  LOG.error("Failed to mutate: " + keyBase + " after " + (System.currentTimeMillis() - start)
      + "ms; region information: " + getRegionDebugInfoSafe(table, m.getRow()) + "; errors: "
      + exceptionInfo);
}
 
Example 3
Project: hadoop-oss   File: AccessControlList.java   Source Code and License 6 votes vote down vote up
/**
 * Build ACL from the given two Strings.
 * The Strings contain comma separated values.
 *
 * @param aclString build ACL from array of Strings
 */
private void buildACL(String[] userGroupStrings) {
  users = new HashSet<String>();
  groups = new HashSet<String>();
  for (String aclPart : userGroupStrings) {
    if (aclPart != null && isWildCardACLValue(aclPart)) {
      allAllowed = true;
      break;
    }
  }
  if (!allAllowed) {      
    if (userGroupStrings.length >= 1 && userGroupStrings[0] != null) {
      users = StringUtils.getTrimmedStringCollection(userGroupStrings[0]);
    } 
    
    if (userGroupStrings.length == 2 && userGroupStrings[1] != null) {
      groups = StringUtils.getTrimmedStringCollection(userGroupStrings[1]);
      groupsMapping.cacheGroupsAdd(new LinkedList<String>(groups));
    }
  }
}
 
Example 4
Project: aliyun-maxcompute-data-collectors   File: CreateHiveTableTool.java   Source Code and License 6 votes vote down vote up
@Override
/** {@inheritDoc} */
public int run(SqoopOptions options) {
  if (!init(options)) {
    return 1;
  }

  try {
    HiveImport hiveImport = new HiveImport(options, manager,
        options.getConf(), false);
    hiveImport.importTable(options.getTableName(),
        options.getHiveTableName(), true);
  } catch (IOException ioe) {
    LOG.error("Encountered IOException running create table job: "
        + StringUtils.stringifyException(ioe));
    if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
      throw new RuntimeException(ioe);
    } else {
      return 1;
    }
  } finally {
    destroy(options);
  }

  return 0;
}
 
Example 5
Project: aliyun-maxcompute-data-collectors   File: HsqldbMetaStore.java   Source Code and License 6 votes vote down vote up
/**
 * Blocks the current thread until the server is shut down.
 */
public void waitForServer() {
  while (true) {
    int curState = server.getState();
    if (curState == ServerConstants.SERVER_STATE_SHUTDOWN) {
      LOG.info("Got shutdown notification");
      break;
    }

    try {
      Thread.sleep(100);
    } catch (InterruptedException ie) {
      LOG.info("Interrupted while blocking for server:"
          + StringUtils.stringifyException(ie));
    }
  }
}
 
Example 6
Project: hadoop   File: CacheAdmin.java   Source Code and License 6 votes vote down vote up
@Override
public int run(Configuration conf, List<String> args) throws IOException {
  String name = StringUtils.popFirstNonOption(args);
  if (name == null) {
    System.err.println("You must specify a name when deleting a " +
        "cache pool.");
    return 1;
  }
  if (!args.isEmpty()) {
    System.err.print("Can't understand arguments: " +
      Joiner.on(" ").join(args) + "\n");
    System.err.println("Usage is " + getShortUsage());
    return 1;
  }
  DistributedFileSystem dfs = AdminHelper.getDFS(conf);
  try {
    dfs.removeCachePool(name);
  } catch (IOException e) {
    System.err.println(AdminHelper.prettifyException(e));
    return 2;
  }
  System.out.println("Successfully removed cache pool " + name + ".");
  return 0;
}
 
Example 7
Project: hadoop   File: SharedCacheManager.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) {
  Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(SharedCacheManager.class, args, LOG);
  try {
    Configuration conf = new YarnConfiguration();
    SharedCacheManager sharedCacheManager = new SharedCacheManager();
    ShutdownHookManager.get().addShutdownHook(
        new CompositeServiceShutdownHook(sharedCacheManager),
        SHUTDOWN_HOOK_PRIORITY);
    sharedCacheManager.init(conf);
    sharedCacheManager.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting SharedCacheManager", t);
    System.exit(-1);
  }
}
 
Example 8
Project: hadoop   File: TestFileSystemAccessService.java   Source Code and License 6 votes vote down vote up
@Test
@TestDir
public void serviceHadoopConf() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);

  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  Assert.assertEquals(fsAccess.serviceHadoopConf.get("foo"), "FOO");
  server.destroy();
}
 
Example 9
Project: hadoop   File: TestFileSystemAccessService.java   Source Code and License 6 votes vote down vote up
@Test
@TestException(exception = ServiceException.class, msgRegExp = "H02.*")
@TestDir
public void kerberosInitializationFailure() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "kerberos");
  conf.set("server.hadoop.authentication.kerberos.keytab", "/tmp/foo");
  conf.set("server.hadoop.authentication.kerberos.principal", "[email protected]");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 
Example 10
Project: hadoop   File: AccessControlList.java   Source Code and License 6 votes vote down vote up
/**
 * Build ACL from the given two Strings.
 * The Strings contain comma separated values.
 *
 * @param aclString build ACL from array of Strings
 */
private void buildACL(String[] userGroupStrings) {
  users = new HashSet<String>();
  groups = new HashSet<String>();
  for (String aclPart : userGroupStrings) {
    if (aclPart != null && isWildCardACLValue(aclPart)) {
      allAllowed = true;
      break;
    }
  }
  if (!allAllowed) {      
    if (userGroupStrings.length >= 1 && userGroupStrings[0] != null) {
      users = StringUtils.getTrimmedStringCollection(userGroupStrings[0]);
    } 
    
    if (userGroupStrings.length == 2 && userGroupStrings[1] != null) {
      groups = StringUtils.getTrimmedStringCollection(userGroupStrings[1]);
      groupsMapping.cacheGroupsAdd(new LinkedList<String>(groups));
    }
  }
}
 
Example 11
Project: hadoop   File: Application.java   Source Code and License 6 votes vote down vote up
/**
 * Abort the application and wait for it to finish.
 * @param t the exception that signalled the problem
 * @throws IOException A wrapper around the exception that was passed in
 */
void abort(Throwable t) throws IOException {
  LOG.info("Aborting because of " + StringUtils.stringifyException(t));
  try {
    downlink.abort();
    downlink.flush();
  } catch (IOException e) {
    // IGNORE cleanup problems
  }
  try {
    handler.waitForFinish();
  } catch (Throwable ignored) {
    process.destroy();
  }
  IOException wrapper = new IOException("pipe child exception");
  wrapper.initCause(t);
  throw wrapper;      
}
 
Example 12
Project: aliyun-maxcompute-data-collectors   File: MSSQLTestUtils.java   Source Code and License 6 votes vote down vote up
private Connection getConnection() {

    if (conn == null) {

      try {
        Connection con = DriverManager.getConnection(HOST_URL,
            DATABASE_USER, DATABASE_PASSWORD);
        conn = con;
        return con;
      } catch (SQLException e) {
        LOG.error("Get SQLException during setting up connection: " + StringUtils.stringifyException(e));
        return null;
      }
    }

    return conn;
  }
 
Example 13
Project: hadoop   File: TestMRApps.java   Source Code and License 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example 14
Project: hadoop   File: HamletGen.java   Source Code and License 6 votes vote down vote up
void genNewElementMethod(String className, Method method, int indent) {
  String methodName = method.getName();
  String retName = method.getReturnType().getSimpleName();
  Class<?>[] params = method.getParameterTypes();
  echo(indent, "\n",
       "@Override\n",
       "public ", retName, "<", className, topMode ? "> " : "<T>> ",
       methodName, "(");
  if (params.length == 0) {
    puts(0, ") {");
    puts(indent,
         topMode ? "" : "  closeAttrs();\n",
         "  return ", StringUtils.toLowerCase(retName), "_" + "(this, ",
         isInline(className, retName), ");\n", "}");
  } else if (params.length == 1) {
    puts(0, "String selector) {");
    puts(indent,
         "  return setSelector(", methodName, "(), selector);\n", "}");
  } else {
    throwUnhandled(className, method);
  }
}
 
Example 15
Project: hadoop   File: HistoryViewer.java   Source Code and License 6 votes vote down vote up
private void printJobDetails() {
  StringBuffer jobDetails = new StringBuffer();
  jobDetails.append("\nHadoop job: " ).append(job.getJobId());
  jobDetails.append("\n=====================================");
  jobDetails.append("\nUser: ").append(job.getUsername()); 
  jobDetails.append("\nJobName: ").append(job.getJobname()); 
  jobDetails.append("\nJobConf: ").append(job.getJobConfPath()); 
  jobDetails.append("\nSubmitted At: ").append(StringUtils.
                      getFormattedTimeWithDiff(dateFormat,
                      job.getSubmitTime(), 0)); 
  jobDetails.append("\nLaunched At: ").append(StringUtils.
                      getFormattedTimeWithDiff(dateFormat,
                      job.getLaunchTime(),
                      job.getSubmitTime()));
  jobDetails.append("\nFinished At: ").append(StringUtils.
                      getFormattedTimeWithDiff(dateFormat,
                      job.getFinishTime(),
                      job.getLaunchTime()));
  jobDetails.append("\nStatus: ").append(((job.getJobStatus() == null) ? 
                    "Incomplete" :job.getJobStatus()));
  printCounters(jobDetails, job.getTotalCounters(), job.getMapCounters(),
      job.getReduceCounters());
  jobDetails.append("\n");
  jobDetails.append("\n=====================================");
  System.out.println(jobDetails.toString());
}
 
Example 16
Project: ditb   File: LruBlockCache.java   Source Code and License 6 votes vote down vote up
public void logStats() {
  // Log size
  long totalSize = heapSize();
  long freeSize = maxSize - totalSize;
  LruBlockCache.LOG.info("totalSize=" + StringUtils.byteDesc(totalSize) + ", " +
      "freeSize=" + StringUtils.byteDesc(freeSize) + ", " +
      "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
      "blockCount=" + getBlockCount() + ", " +
      "accesses=" + stats.getRequestCount() + ", " +
      "hits=" + stats.getHitCount() + ", " +
      "hitRatio=" + (stats.getHitCount() == 0 ?
        "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
      "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
      "cachingHits=" + stats.getHitCachingCount() + ", " +
      "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
        "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
      "evictions=" + stats.getEvictionCount() + ", " +
      "evicted=" + stats.getEvictedCount() + ", " +
      "evictedPerRun=" + stats.evictedPerEviction());
}
 
Example 17
Project: hadoop   File: ShortCircuitCache.java   Source Code and License 6 votes vote down vote up
/**
 * Trim the eviction lists.
 */
private void trimEvictionMaps() {
  long now = Time.monotonicNow();
  demoteOldEvictableMmaped(now);

  while (true) {
    long evictableSize = evictable.size();
    long evictableMmappedSize = evictableMmapped.size();
    if (evictableSize + evictableMmappedSize <= maxTotalSize) {
      return;
    }
    ShortCircuitReplica replica;
    if (evictableSize == 0) {
     replica = evictableMmapped.firstEntry().getValue();
    } else {
     replica = evictable.firstEntry().getValue();
    }
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": trimEvictionMaps is purging " + replica +
        StringUtils.getStackTrace(Thread.currentThread()));
    }
    purge(replica);
  }
}
 
Example 18
Project: ditb   File: HStore.java   Source Code and License 6 votes vote down vote up
/**
 * Similar to commit, but called in secondary region replicas for replaying the flush cache from
 * primary region. Adds the new files to the store, and drops the snapshot depending on
 * dropMemstoreSnapshot argument.
 *
 * @param fileNames            names of the flushed files
 * @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
 * @throws IOException
 */
@Override public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
    throws IOException {
  List<StoreFile> storeFiles = new ArrayList<StoreFile>(fileNames.size());
  for (String file : fileNames) {
    // open the file as a store file (hfile link, etc)
    StoreFileInfo storeFileInfo = fs.getStoreFileInfo(getColumnFamilyName(), file);
    StoreFile storeFile = createStoreFileAndReader(storeFileInfo);
    storeFiles.add(storeFile);
    HStore.this.storeSize += storeFile.getReader().length();
    HStore.this.totalUncompressedBytes += storeFile.getReader().getTotalUncompressedBytes();
    if (LOG.isInfoEnabled()) {
      LOG.info("Region: " + HStore.this.getRegionInfo().getEncodedName() + " added " + storeFile
          + ", entries=" + storeFile.getReader().getEntries() + ", sequenceid=" + +storeFile
          .getReader().getSequenceID() + ", filesize=" + StringUtils
          .humanReadableInt(storeFile.getReader().length()));
    }
  }

  long snapshotId = -1; // -1 means do not drop
  if (dropMemstoreSnapshot && snapshot != null) {
    snapshotId = snapshot.getId();
  }
  HStore.this.updateStorefiles(storeFiles, snapshotId);
}
 
Example 19
Project: hadoop   File: Command.java   Source Code and License 6 votes vote down vote up
/**
 * Display an exception prefaced with the command name.  Also increments
 * the error count for the command which will result in a non-zero exit
 * code.
 * @param e exception to display
 */
public void displayError(Exception e) {
  // build up a list of exceptions that occurred
  exceptions.add(e);
  
  String errorMessage = e.getLocalizedMessage();
  if (errorMessage == null) {
    // this is an unexpected condition, so dump the whole exception since
    // it's probably a nasty internal error where the backtrace would be
    // useful
    errorMessage = StringUtils.stringifyException(e);
    LOG.debug(errorMessage);
  } else {
    errorMessage = errorMessage.split("\n", 2)[0];
  }
  displayError(errorMessage);
}
 
Example 20
Project: aliyun-maxcompute-data-collectors   File: OracleExportTest.java   Source Code and License 6 votes vote down vote up
@Before
public void setUp() {
  super.setUp();

  SqoopOptions options = new SqoopOptions(OracleUtils.CONNECT_STRING,
      getTableName());
  OracleUtils.setOracleAuth(options);
  this.manager = new OracleManager(options);
  try {
    this.conn = manager.getConnection();
    this.conn.setAutoCommit(false);
  } catch (SQLException sqlE) {
    LOG.error(StringUtils.stringifyException(sqlE));
    fail("Failed with sql exception in setup: " + sqlE);
  }
}
 
Example 21
Project: hadoop   File: QuorumJournalManager.java   Source Code and License 6 votes vote down vote up
private static List<InetSocketAddress> getLoggerAddresses(URI uri)
    throws IOException {
  String authority = uri.getAuthority();
  Preconditions.checkArgument(authority != null && !authority.isEmpty(),
      "URI has no authority: " + uri);
  
  String[] parts = StringUtils.split(authority, ';');
  for (int i = 0; i < parts.length; i++) {
    parts[i] = parts[i].trim();
  }

  if (parts.length % 2 == 0) {
    LOG.warn("Quorum journal URI '" + uri + "' has an even number " +
        "of Journal Nodes specified. This is not recommended!");
  }
  
  List<InetSocketAddress> addrs = Lists.newArrayList();
  for (String addr : parts) {
    addrs.add(NetUtils.createSocketAddr(
        addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT));
  }
  return addrs;
}
 
Example 22
Project: hadoop-oss   File: TimedOutTestsListener.java   Source Code and License 6 votes vote down vote up
static String buildThreadDump() {
  StringBuilder dump = new StringBuilder();
  Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
  for (Map.Entry<Thread, StackTraceElement[]> e : stackTraces.entrySet()) {
    Thread thread = e.getKey();
    dump.append(String.format(
        "\"%s\" %s prio=%d tid=%d %s\njava.lang.Thread.State: %s",
        thread.getName(),
        (thread.isDaemon() ? "daemon" : ""),
        thread.getPriority(),
        thread.getId(),
        Thread.State.WAITING.equals(thread.getState()) ? 
            "in Object.wait()" :
            StringUtils.toLowerCase(thread.getState().name()),
        Thread.State.WAITING.equals(thread.getState()) ?
            "WAITING (on object monitor)" : thread.getState()));
    for (StackTraceElement stackTraceElement : e.getValue()) {
      dump.append("\n        at ");
      dump.append(stackTraceElement);
    }
    dump.append("\n");
  }
  return dump.toString();
}
 
Example 23
Project: hadoop   File: TestFileSystemAccessService.java   Source Code and License 6 votes vote down vote up
@Test
@TestException(exception = FileSystemAccessException.class, msgRegExp = "H05.*")
@TestDir
public void NameNodeNotinWhitelists() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.name.node.whitelist", "NN");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  fsAccess.validateNamenode("NNx");
}
 
Example 24
Project: hadoop-oss   File: TestUTF8.java   Source Code and License 6 votes vote down vote up
/**
 * Test encoding and decoding of UTF8 outside the basic multilingual plane.
 *
 * This is a regression test for HADOOP-9103.
 */
@Test
public void testNonBasicMultilingualPlane() throws Exception {
  // Test using the "CAT FACE" character (U+1F431)
  // See http://www.fileformat.info/info/unicode/char/1f431/index.htm
  String catFace = "\uD83D\uDC31";

  // This encodes to 4 bytes in UTF-8:
  byte[] encoded = catFace.getBytes("UTF-8");
  assertEquals(4, encoded.length);
  assertEquals("f09f90b1", StringUtils.byteToHexString(encoded));

  // Decode back to String using our own decoder
  String roundTrip = UTF8.fromBytes(encoded);
  assertEquals(catFace, roundTrip);
}
 
Example 25
Project: hadoop   File: LeaseRenewer.java   Source Code and License 5 votes vote down vote up
private LeaseRenewer(Factory.Key factorykey) {
  this.factorykey = factorykey;
  unsyncSetGraceSleepPeriod(LEASE_RENEWER_GRACE_DEFAULT);
  
  if (LOG.isTraceEnabled()) {
    instantiationTrace = StringUtils.stringifyException(
      new Throwable("TRACE"));
  } else {
    instantiationTrace = null;
  }
}
 
Example 26
Project: hadoop   File: TestEditLog.java   Source Code and License 5 votes vote down vote up
/**
 * Construct FSEditLog with default configuration, taking editDirs from NNStorage
 * 
 * @param storage Storage object used by namenode
 */
private static FSEditLog getFSEditLog(NNStorage storage) throws IOException {
  Configuration conf = new Configuration();
  // Make sure the edits dirs are set in the provided configuration object.
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      StringUtils.join(",", storage.getEditsDirectories()));
  FSEditLog log = new FSEditLog(conf, storage, FSNamesystem.getNamespaceEditsDirs(conf));
  return log;
}
 
Example 27
Project: hadoop   File: TestParam.java   Source Code and License 5 votes vote down vote up
@Test
public void testConcatSourcesParam() {
  final String[] strings = {"/", "/foo", "/bar"};
  for(int n = 0; n < strings.length; n++) {
    final String[] sub = new String[n]; 
    final Path[] paths = new Path[n];
    for(int i = 0; i < paths.length; i++) {
      paths[i] = new Path(sub[i] = strings[i]);
    }

    final String expected = StringUtils.join(",", Arrays.asList(sub));
    final ConcatSourcesParam computed = new ConcatSourcesParam(paths);
    Assert.assertEquals(expected, computed.getValue());
  }
}
 
Example 28
Project: hadoop   File: Dispatcher.java   Source Code and License 5 votes vote down vote up
/**
 * Read set of host names from a file
 * 
 * @return set of host names
 */
static Set<String> getHostListFromFile(String fileName, String type) {
  Set<String> nodes = new HashSet<String>();
  try {
    HostsFileReader.readFileToSet(type, fileName, nodes);
    return StringUtils.getTrimmedStrings(nodes);
  } catch (IOException e) {
    throw new IllegalArgumentException(
        "Failed to read host list from file: " + fileName);
  }
}
 
Example 29
Project: hadoop   File: XAttrHelper.java   Source Code and License 5 votes vote down vote up
/**
 * Get name with prefix from <code>XAttr</code>
 */
public static String getPrefixName(XAttr xAttr) {
  if (xAttr == null) {
    return null;
  }
  
  String namespace = xAttr.getNameSpace().toString();
  return StringUtils.toLowerCase(namespace) + "." + xAttr.getName();
}
 
Example 30
Project: hadoop   File: TestShortCircuitLocalRead.java   Source Code and License 5 votes vote down vote up
static private void checkData(byte[] actual, int from, byte[] expected, int len,
    String message) {
  for (int idx = 0; idx < len; idx++) {
    if (expected[from + idx] != actual[idx]) {
      Assert.fail(message + " byte " + (from + idx) + " differs. expected "
          + expected[from + idx] + " actual " + actual[idx] +
          "\nexpected: " + StringUtils.byteToHexString(expected, from, from + len) +
          "\nactual:   " + StringUtils.byteToHexString(actual, 0, len));
    }
  }
}
 
Example 31
Project: hadoop-oss   File: SecurityUtil.java   Source Code and License 5 votes vote down vote up
/**
 * Construct the service key for a token
 * @param addr InetSocketAddress of remote connection with a token
 * @return "ip:port" or "host:port" depending on the value of
 *          hadoop.security.token.service.use_ip
 */
public static Text buildTokenService(InetSocketAddress addr) {
  String host = null;
  if (useIpForTokenService) {
    if (addr.isUnresolved()) { // host has no ip address
      throw new IllegalArgumentException(
          new UnknownHostException(addr.getHostName())
      );
    }
    host = addr.getAddress().getHostAddress();
  } else {
    host = StringUtils.toLowerCase(addr.getHostName());
  }
  return new Text(host + ":" + addr.getPort());
}
 
Example 32
Project: hadoop-oss   File: SecurityUtil.java   Source Code and License 5 votes vote down vote up
public static AuthenticationMethod getAuthenticationMethod(Configuration conf) {
  String value = conf.get(HADOOP_SECURITY_AUTHENTICATION, "simple");
  try {
    return Enum.valueOf(AuthenticationMethod.class,
        StringUtils.toUpperCase(value));
  } catch (IllegalArgumentException iae) {
    throw new IllegalArgumentException("Invalid attribute value for " +
        HADOOP_SECURITY_AUTHENTICATION + " of " + value);
  }
}
 
Example 33
Project: hadoop-oss   File: SecurityUtil.java   Source Code and License 5 votes vote down vote up
public static void setAuthenticationMethod(
    AuthenticationMethod authenticationMethod, Configuration conf) {
  if (authenticationMethod == null) {
    authenticationMethod = AuthenticationMethod.SIMPLE;
  }
  conf.set(HADOOP_SECURITY_AUTHENTICATION,
      StringUtils.toLowerCase(authenticationMethod.toString()));
}
 
Example 34
Project: hadoop-oss   File: ActiveStandbyElector.java   Source Code and License 5 votes vote down vote up
/**
 * If there is a breadcrumb node indicating that another node may need
 * fencing, try to fence that node.
 * @return the Stat of the breadcrumb node that was read, or null
 * if no breadcrumb node existed
 */
private Stat fenceOldActive() throws InterruptedException, KeeperException {
  final Stat stat = new Stat();
  byte[] data;
  LOG.info("Checking for any old active which needs to be fenced...");
  try {
    data = zkDoWithRetries(new ZKAction<byte[]>() {
      @Override
      public byte[] run() throws KeeperException, InterruptedException {
        return zkClient.getData(zkBreadCrumbPath, false, stat);
      }
    });
  } catch (KeeperException ke) {
    if (isNodeDoesNotExist(ke.code())) {
      LOG.info("No old node to fence");
      return null;
    }
    
    // If we failed to read for any other reason, then likely we lost
    // our session, or we don't have permissions, etc. In any case,
    // we probably shouldn't become active, and failing the whole
    // thing is the best bet.
    throw ke;
  }

  LOG.info("Old node exists: " + StringUtils.byteToHexString(data));
  if (Arrays.equals(data, appData)) {
    LOG.info("But old node has our own data, so don't need to fence it.");
  } else {
    appClient.fenceOldActive(data);
  }
  return stat;
}
 
Example 35
Project: hadoop   File: TestFileSystemAccessService.java   Source Code and License 5 votes vote down vote up
@Test
@TestException(exception = ServiceException.class, msgRegExp = "H09.*")
@TestDir
public void invalidSecurity() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "foo");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 
Example 36
Project: hadoop-oss   File: MetricsSystemImpl.java   Source Code and License 5 votes vote down vote up
private InitMode initMode() {
  LOG.debug("from system property: "+ System.getProperty(MS_INIT_MODE_KEY));
  LOG.debug("from environment variable: "+ System.getenv(MS_INIT_MODE_KEY));
  String m = System.getProperty(MS_INIT_MODE_KEY);
  String m2 = m == null ? System.getenv(MS_INIT_MODE_KEY) : m;
  return InitMode.valueOf(
      StringUtils.toUpperCase((m2 == null ? InitMode.NORMAL.name() : m2)));
}
 
Example 37
Project: hadoop   File: DistCpUtils.java   Source Code and License 5 votes vote down vote up
/**
 * Converts a FileStatus to a CopyListingFileStatus.  If preserving ACLs,
 * populates the CopyListingFileStatus with the ACLs. If preserving XAttrs,
 * populates the CopyListingFileStatus with the XAttrs.
 *
 * @param fileSystem FileSystem containing the file
 * @param fileStatus FileStatus of file
 * @param preserveAcls boolean true if preserving ACLs
 * @param preserveXAttrs boolean true if preserving XAttrs
 * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs
 * @throws IOException if there is an I/O error
 */
public static CopyListingFileStatus toCopyListingFileStatus(
    FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
    boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException {
  CopyListingFileStatus copyListingFileStatus =
    new CopyListingFileStatus(fileStatus);
  if (preserveAcls) {
    FsPermission perm = fileStatus.getPermission();
    if (perm.getAclBit()) {
      List<AclEntry> aclEntries = fileSystem.getAclStatus(
        fileStatus.getPath()).getEntries();
      copyListingFileStatus.setAclEntries(aclEntries);
    }
  }
  if (preserveXAttrs || preserveRawXAttrs) {
    Map<String, byte[]> srcXAttrs = fileSystem.getXAttrs(fileStatus.getPath());
    if (preserveXAttrs && preserveRawXAttrs) {
       copyListingFileStatus.setXAttrs(srcXAttrs);
    } else {
      Map<String, byte[]> trgXAttrs = Maps.newHashMap();
      final String rawNS =
          StringUtils.toLowerCase(XAttr.NameSpace.RAW.name());
      for (Map.Entry<String, byte[]> ent : srcXAttrs.entrySet()) {
        final String xattrName = ent.getKey();
        if (xattrName.startsWith(rawNS)) {
          if (preserveRawXAttrs) {
            trgXAttrs.put(xattrName, ent.getValue());
          }
        } else if (preserveXAttrs) {
          trgXAttrs.put(xattrName, ent.getValue());
        }
      }
      copyListingFileStatus.setXAttrs(trgXAttrs);
    }
  }
  return copyListingFileStatus;
}
 
Example 38
Project: hadoop   File: SliveReducer.java   Source Code and License 5 votes vote down vote up
@Override // Reducer
public void reduce(Text key, Iterator<Text> values,
    OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
  OperationOutput collector = null;
  int reduceAm = 0;
  int errorAm = 0;
  logAndSetStatus(reporter, "Iterating over reduction values for key " + key);
  while (values.hasNext()) {
    Text value = values.next();
    try {
      OperationOutput val = new OperationOutput(key, value);
      if (collector == null) {
        collector = val;
      } else {
        collector = OperationOutput.merge(collector, val);
      }
      LOG.info("Combined " + val + " into/with " + collector);
      ++reduceAm;
    } catch (Exception e) {
      ++errorAm;
      logAndSetStatus(reporter, "Error iterating over reduction input "
          + value + " due to : " + StringUtils.stringifyException(e));
      if (getConfig().shouldExitOnFirstError()) {
        break;
      }
    }
  }
  logAndSetStatus(reporter, "Reduced " + reduceAm + " values with " + errorAm
      + " errors");
  if (collector != null) {
    logAndSetStatus(reporter, "Writing output " + collector.getKey() + " : "
        + collector.getOutputValue());
    output.collect(collector.getKey(), collector.getOutputValue());
  }
}
 
Example 39
Project: hadoop-oss   File: FileSystem.java   Source Code and License 5 votes vote down vote up
Key(URI uri, Configuration conf, long unique) throws IOException {
  scheme = uri.getScheme()==null ?
      "" : StringUtils.toLowerCase(uri.getScheme());
  authority = uri.getAuthority()==null ?
      "" : StringUtils.toLowerCase(uri.getAuthority());
  this.unique = unique;
  
  this.ugi = UserGroupInformation.getCurrentUser();
}
 
Example 40
Project: ditb   File: HBaseAdmin.java   Source Code and License 5 votes vote down vote up
private boolean isEncodedRegionName(byte[] regionName) throws IOException {
  try {
    HRegionInfo.parseRegionName(regionName);
    return false;
  } catch (IOException e) {
    if (StringUtils.stringifyException(e)
      .contains(HRegionInfo.INVALID_REGION_NAME_FORMAT_MESSAGE)) {
      return true;
    }
    throw e;
  }
}