Java Code Examples for org.apache.hadoop.mapred.JobConf#iterator()

The following examples show how to use org.apache.hadoop.mapred.JobConf#iterator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HadoopUtils.java    From incubator-hivemall with Apache License 2.0 6 votes vote down vote up
@Nonnull
public static String toString(@Nonnull JobConf jobconf, @Nullable String regexKey) {
    final Iterator<Entry<String, String>> itor = jobconf.iterator();
    boolean hasNext = itor.hasNext();
    if (!hasNext) {
        return "";
    }
    final StringBuilder buf = new StringBuilder(1024);
    do {
        Entry<String, String> e = itor.next();
        hasNext = itor.hasNext();
        String k = e.getKey();
        if (k == null) {
            continue;
        }
        if (regexKey == null || k.matches(regexKey)) {
            String v = e.getValue();
            buf.append(k).append('=').append(v);
            if (hasNext) {
                buf.append(',');
            }
        }
    } while (hasNext);
    return buf.toString();
}
 
Example 2
Source File: Utils.java    From spork with Apache License 2.0 6 votes vote down vote up
/**
 * Method to apply pig properties to JobConf (replaces properties with
 * resulting jobConf values).
 *
 * @param conf JobConf with appropriate hadoop resource files
 * @param properties Pig properties that will override hadoop properties;
 * properties might be modified
 */
public static void recomputeProperties(JobConf jobConf, Properties properties) {
    // We need to load the properties from the hadoop configuration
    // We want to override these with any existing properties we have.
    if (jobConf != null && properties != null) {
        // set user properties on the jobConf to ensure that defaults
        // and deprecation is applied correctly
        Enumeration<Object> propertiesIter = properties.keys();
        while (propertiesIter.hasMoreElements()) {
            String key = (String) propertiesIter.nextElement();
            String val = properties.getProperty(key);
            // We do not put user.name, See PIG-1419
            if (!key.equals("user.name")) {
                jobConf.set(key, val);
            }
        }
        // clear user defined properties and re-populate
        properties.clear();
        Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            properties.put(entry.getKey(), entry.getValue());
        }
    }
}
 
Example 3
Source File: PipeMapRed.java    From RDFS with Apache License 2.0 6 votes vote down vote up
void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 
Example 4
Source File: PipeMapRed.java    From RDFS with Apache License 2.0 6 votes vote down vote up
void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 
Example 5
Source File: PipeMapRed.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
void addJobConfToEnvironment(JobConf conf, Properties env) {
  if (debug_) {
    logprintln("addJobConfToEnvironment: begin");
  }
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
  if (debug_) {
    logprintln("addJobConfToEnvironment: end");
  }
}
 
Example 6
Source File: PipeMapRed.java    From hadoop with Apache License 2.0 5 votes vote down vote up
void addJobConfToEnvironment(JobConf jobconf, Properties env) {
  JobConf conf = new JobConf(jobconf);
  conf.setDeprecatedProperties();
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
}
 
Example 7
Source File: PipeMapRed.java    From big-c with Apache License 2.0 5 votes vote down vote up
void addJobConfToEnvironment(JobConf jobconf, Properties env) {
  JobConf conf = new JobConf(jobconf);
  conf.setDeprecatedProperties();
  Iterator it = conf.iterator();
  while (it.hasNext()) {
    Map.Entry en = (Map.Entry) it.next();
    String name = (String) en.getKey();
    //String value = (String)en.getValue(); // does not apply variable expansion
    String value = conf.get(name); // does variable expansion 
    name = safeEnvVarName(name);
    envPut(env, name, value);
  }
}
 
Example 8
Source File: HExecutionEngine.java    From spork with Apache License 2.0 5 votes vote down vote up
public JobConf getS3Conf() throws ExecException {
    JobConf jc = new JobConf();
    jc.addResource(CORE_SITE);
    Iterator<Entry<String, String>> i = jc.iterator();
    while (i.hasNext()) {
        Entry<String, String> e = i.next();
        String key = e.getKey();
        String value = e.getValue();
        if (key.startsWith("fs.s3") || key.startsWith("fs.s3n")) {
            jc.set(key, value);
        }
    }
    return jc;
}