Java Code Examples for org.apache.hadoop.conf.Configuration#getStringCollection()

The following examples show how to use org.apache.hadoop.conf.Configuration#getStringCollection() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testCsvBlurDriverTest3() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000", "-p", "SNAPPY");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
  assertEquals("true", configuration.get(CsvBlurDriver.MAPRED_COMPRESS_MAP_OUTPUT));
  assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC));
}
 
Example 2
@Test
public void testCsvBlurDriverTest() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString());
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 
Example 3
Source Project: hadoop   File: KeyProviderFactory.java    License: Apache License 2.0 6 votes vote down vote up
public static List<KeyProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<KeyProvider> result = new ArrayList<KeyProvider>();
  for(String path: conf.getStringCollection(KEY_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      KeyProvider kp = get(uri, conf);
      if (kp != null) {
        result.add(kp);
      } else {
        throw new IOException("No KeyProviderFactory for " + uri + " in " +
            KEY_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + KEY_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 
Example 4
public NodeWriter(Configuration conf, String host) {
    super(conf, host);
    String opTypeStr = conf.get(NODE_OPERATION_TYPE);
    if (opTypeStr == null || opTypeStr.isEmpty()) {
        throw new IllegalArgumentException(
                NODE_OPERATION_TYPE + " is not specified.");
    }
    NodeOpType opType = NodeOpType.valueOf(opTypeStr);
    Collection<String> nsCol = conf.getStringCollection(OUTPUT_NAMESPACE);
    StringBuilder buf = new StringBuilder();
    if (nsCol != null) {
        for (Iterator<String> nsIt = nsCol.iterator(); nsIt.hasNext();) {
            String ns = nsIt.next();
            buf.append('"').append(ns).append('"');
            if (nsIt.hasNext()) {
                buf.append(',');
            }
        }
    }
    query = opType.getQuery(buf.toString());
}
 
Example 5
Source Project: big-c   File: CredentialProviderFactory.java    License: Apache License 2.0 6 votes vote down vote up
public static List<CredentialProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<CredentialProvider> result = new ArrayList<CredentialProvider>();
  for(String path: conf.getStringCollection(CREDENTIAL_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      boolean found = false;
      for(CredentialProviderFactory factory: serviceLoader) {
        CredentialProvider kp = factory.createProvider(uri, conf);
        if (kp != null) {
          result.add(kp);
          found = true;
          break;
        }
      }
      if (!found) {
        throw new IOException("No CredentialProviderFactory for " + uri + " in " +
            CREDENTIAL_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + CREDENTIAL_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 
Example 6
Source Project: hbase   File: IOTestProvider.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable,
    long blocksize) throws IOException, CommonFSUtils.StreamLacksCapabilityException {
  Collection<String> operations = conf.getStringCollection(ALLOWED_OPERATIONS);
  if (operations.isEmpty() || operations.contains(AllowedOperations.all.name())) {
    doAppends = doSyncs = true;
  } else if (operations.contains(AllowedOperations.none.name())) {
    doAppends = doSyncs = false;
  } else {
    doAppends = operations.contains(AllowedOperations.append.name());
    doSyncs = operations.contains(AllowedOperations.sync.name());
  }
  LOG.info("IOTestWriter initialized with appends " + (doAppends ? "enabled" : "disabled") +
      " and syncs " + (doSyncs ? "enabled" : "disabled"));
  super.init(fs, path, conf, overwritable, blocksize);
}
 
Example 7
Source Project: hbase   File: TableMapReduceUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
 * Also exposed to shell scripts via `bin/hbase mapredcp`.
 */
public static String buildDependencyClasspath(Configuration conf) {
  if (conf == null) {
    throw new IllegalArgumentException("Must provide a configuration object.");
  }
  Set<String> paths = new HashSet<>(conf.getStringCollection("tmpjars"));
  if (paths.isEmpty()) {
    throw new IllegalArgumentException("Configuration contains no tmpjars.");
  }
  StringBuilder sb = new StringBuilder();
  for (String s : paths) {
    // entries can take the form 'file:/path/to/file.jar'.
    int idx = s.indexOf(":");
    if (idx != -1) s = s.substring(idx + 1);
    if (sb.length() > 0) sb.append(File.pathSeparator);
    sb.append(s);
  }
  return sb.toString();
}
 
Example 8
public static void main(String[] args) throws IOException {
  Path p = new Path("./src/main/scripts/conf/hdfs");

  Configuration configuration = mergeHdfsConfigs(p.getFileSystem(new Configuration()), p);

  // configuration.writeXml(System.out);

  Collection<String> nameServices = configuration.getStringCollection(DFS_NAMESERVICES);
  for (String name : nameServices) {
    Path path = new Path("hdfs://" + name + "/");
    FileSystem fileSystem = path.getFileSystem(configuration);
    FileStatus[] listStatus = fileSystem.listStatus(path);
    for (FileStatus fileStatus : listStatus) {
      System.out.println(fileStatus.getPath());
    }
  }
}
 
Example 9
@Test
public void testCsvBlurDriverTest2() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 
Example 10
Source Project: datawave   File: BulkInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets the columns to be mapped over from this configuration object.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @return a set of columns
 * @see #fetchColumns(Configuration, Collection)
 */
protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
    Set<Pair<Text,Text>> columns = new HashSet<>();
    for (String col : conf.getStringCollection(COLUMNS)) {
        int idx = col.indexOf(":");
        Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes()) : Base64.decodeBase64(col.substring(0, idx).getBytes()));
        Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes()));
        columns.add(new Pair<>(cf, cq));
    }
    return columns;
}
 
Example 11
@Override
protected void setup(Context context) throws IOException, InterruptedException {
  super.setup(context);
  Configuration configuration = context.getConfiguration();
  _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
  _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
  if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
    try {
      _digest = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IOException(e);
    }
  }
  _columnNameMap = getFamilyAndColumnNameMap(configuration);
  _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)), UTF_8);
  _splitter = Splitter.on(_separator);
  Path fileCurrentlyProcessing = getCurrentFile(context);
  Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
  OUTER: for (String family : families) {
    Collection<String> pathStrCollection = configuration
        .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
    for (String pathStr : pathStrCollection) {
      Path path = new Path(pathStr);
      FileSystem fileSystem = path.getFileSystem(configuration);
      path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
      if (isParent(path, fileCurrentlyProcessing)) {
        _familyFromPath = family;
        _familyNotInFile = true;
        break OUTER;
      }
    }
  }
}
 
Example 12
Source Project: Halyard   File: QueryInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public static void addQuery(Configuration conf, String name, String query, int repeatCount) {
    Collection<String> qNames = conf.getStringCollection(QUERIES);
    qNames.add(name);
    conf.set(PREFIX + name + QUERY_SUFFIX, query);
    conf.setInt(PREFIX + name + REPEAT_SUFFIX, repeatCount);
    conf.setStrings(QUERIES, qNames.toArray(new String[qNames.size()]));
}
 
Example 13
Source Project: Halyard   File: QueryInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    ArrayList<InputSplit> splits = new ArrayList<>();
    Configuration conf = context.getConfiguration();
    for (String qName : conf.getStringCollection(QUERIES)) {
        int repeatCount = conf.getInt(PREFIX + qName + REPEAT_SUFFIX, 1);
        String query = conf.get(PREFIX + qName + QUERY_SUFFIX);
        for (int i=0; i<repeatCount; i++) {
            splits.add(new QueryInputSplit(qName, query , i));
        }
    }
    return splits;
}
 
Example 14
Source Project: hadoop-gpu   File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
static Collection<File> getCheckpointEditsDirs(Configuration conf,
                                                String defaultName) {
   Collection<String> dirNames = 
               conf.getStringCollection("fs.checkpoint.edits.dir");
if (dirNames.size() == 0 && defaultName != null) {
  dirNames.add(defaultName);
}
Collection<File> dirs = new ArrayList<File>(dirNames.size());
for(String name : dirNames) {
  dirs.add(new File(name));
}
return dirs;    
 }
 
Example 15
Source Project: RDFS   File: FSImage.java    License: Apache License 2.0 5 votes vote down vote up
static Collection<File> getCheckpointEditsDirs(Configuration conf,
    String defaultName) {
  Collection<String> dirNames = conf
      .getStringCollection("fs.checkpoint.edits.dir");
  if (dirNames.size() == 0 && defaultName != null) {
    dirNames.add(defaultName);
  }
  Collection<File> dirs = new ArrayList<File>(dirNames.size());
  for (String name : dirNames) {
    dirs.add(new File(name));
  }
  return dirs;
}
 
Example 16
public static HConnection createConnection(Configuration conf)
    throws IOException {

  Logger LOG = Logger.getLogger(HConnectionManagerMultiClusterWrapper.class);

  Collection < String > failoverClusters = conf
          .getStringCollection(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG);

  if (failoverClusters.size() == 0) {
    LOG.info(" -- Getting a signle cluster connection !!");
    return HConnectionManager.createConnection(conf);
  } else {

    Map<String, Configuration> configMap = HBaseMultiClusterConfigUtil
        .splitMultiConfigFile(conf);

    LOG.info(" -- Getting primary Connction");
    HConnection primaryConnection = HConnectionManager
        .createConnection(configMap
            .get(HBaseMultiClusterConfigUtil.PRIMARY_NAME));
    LOG.info(" --- Got primary Connction");

    ArrayList<HConnection> failoverConnections = new ArrayList<HConnection>();

    for (Entry<String, Configuration> entry : configMap.entrySet()) {
      if (!entry.getKey().equals(HBaseMultiClusterConfigUtil.PRIMARY_NAME)) {
        LOG.info(" -- Getting failure Connction");
        failoverConnections.add(HConnectionManager.createConnection(entry
            .getValue()));
        LOG.info(" --- Got failover Connction");
      }
    }
    
    return new HConnectionMultiCluster(conf, primaryConnection,
        failoverConnections.toArray(new HConnection[0]));
  }
}
 
Example 17
Source Project: RDFS   File: DataNode.java    License: Apache License 2.0 4 votes vote down vote up
static Collection<URI> getStorageDirs(Configuration conf) {
  Collection<String> dirNames =
    conf.getStringCollection("dfs.data.dir");
  return Util.stringCollectionAsURIs(dirNames);
}
 
Example 18
/**
 * Adds the column layout for the given family.
 * 
 * @param configuration
 *          the configuration to apply the layout.
 * @param family
 *          the family name.
 * @param columns
 *          the column names.
 */
public static void addColumns(Configuration configuration, String family, String... columns) {
  Collection<String> families = new TreeSet<String>(configuration.getStringCollection(BLUR_CSV_FAMILIES));
  families.add(family);
  configuration.setStrings(BLUR_CSV_FAMILIES, families.toArray(new String[] {}));
  configuration.setStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family, columns);
}
 
Example 19
Source Project: big-c   File: HAUtil.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * @param conf Configuration. Please use getRMHAIds to check.
 * @return RM Ids on success
 */
public static Collection<String> getRMHAIds(Configuration conf) {
  return  conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
}
 
Example 20
Source Project: RDFS   File: DFSUtil.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Returns collection of nameservice Ids from the configuration.
 * @param conf configuration
 * @return collection of nameservice Ids
 */
public static Collection<String> getNameServiceIds(Configuration conf) {
  return conf.getStringCollection(FSConstants.DFS_FEDERATION_NAMESERVICES);
}