Java Code Examples for org.apache.hadoop.conf.Configuration#getStringCollection()

The following examples show how to use org.apache.hadoop.conf.Configuration#getStringCollection() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CsvBlurDriverTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvBlurDriverTest2() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 
Example 2
Source File: CsvBlurDriverTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvBlurDriverTest() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString());
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 
Example 3
Source File: HdfsConfigurationNamespaceMerge.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws IOException {
  Path p = new Path("./src/main/scripts/conf/hdfs");

  Configuration configuration = mergeHdfsConfigs(p.getFileSystem(new Configuration()), p);

  // configuration.writeXml(System.out);

  Collection<String> nameServices = configuration.getStringCollection(DFS_NAMESERVICES);
  for (String name : nameServices) {
    Path path = new Path("hdfs://" + name + "/");
    FileSystem fileSystem = path.getFileSystem(configuration);
    FileStatus[] listStatus = fileSystem.listStatus(path);
    for (FileStatus fileStatus : listStatus) {
      System.out.println(fileStatus.getPath());
    }
  }
}
 
Example 4
Source File: KeyProviderFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static List<KeyProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<KeyProvider> result = new ArrayList<KeyProvider>();
  for(String path: conf.getStringCollection(KEY_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      KeyProvider kp = get(uri, conf);
      if (kp != null) {
        result.add(kp);
      } else {
        throw new IOException("No KeyProviderFactory for " + uri + " in " +
            KEY_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + KEY_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 
Example 5
Source File: NodeWriter.java    From marklogic-contentpump with Apache License 2.0 6 votes vote down vote up
public NodeWriter(Configuration conf, String host) {
    super(conf, host);
    String opTypeStr = conf.get(NODE_OPERATION_TYPE);
    if (opTypeStr == null || opTypeStr.isEmpty()) {
        throw new IllegalArgumentException(
                NODE_OPERATION_TYPE + " is not specified.");
    }
    NodeOpType opType = NodeOpType.valueOf(opTypeStr);
    Collection<String> nsCol = conf.getStringCollection(OUTPUT_NAMESPACE);
    StringBuilder buf = new StringBuilder();
    if (nsCol != null) {
        for (Iterator<String> nsIt = nsCol.iterator(); nsIt.hasNext();) {
            String ns = nsIt.next();
            buf.append('"').append(ns).append('"');
            if (nsIt.hasNext()) {
                buf.append(',');
            }
        }
    }
    query = opType.getQuery(buf.toString());
}
 
Example 6
Source File: CredentialProviderFactory.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static List<CredentialProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<CredentialProvider> result = new ArrayList<CredentialProvider>();
  for(String path: conf.getStringCollection(CREDENTIAL_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      boolean found = false;
      for(CredentialProviderFactory factory: serviceLoader) {
        CredentialProvider kp = factory.createProvider(uri, conf);
        if (kp != null) {
          result.add(kp);
          found = true;
          break;
        }
      }
      if (!found) {
        throw new IOException("No CredentialProviderFactory for " + uri + " in " +
            CREDENTIAL_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + CREDENTIAL_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 
Example 7
Source File: TableMapReduceUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
 * Also exposed to shell scripts via `bin/hbase mapredcp`.
 */
public static String buildDependencyClasspath(Configuration conf) {
  if (conf == null) {
    throw new IllegalArgumentException("Must provide a configuration object.");
  }
  Set<String> paths = new HashSet<>(conf.getStringCollection("tmpjars"));
  if (paths.isEmpty()) {
    throw new IllegalArgumentException("Configuration contains no tmpjars.");
  }
  StringBuilder sb = new StringBuilder();
  for (String s : paths) {
    // entries can take the form 'file:/path/to/file.jar'.
    int idx = s.indexOf(":");
    if (idx != -1) s = s.substring(idx + 1);
    if (sb.length() > 0) sb.append(File.pathSeparator);
    sb.append(s);
  }
  return sb.toString();
}
 
Example 8
Source File: IOTestProvider.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable,
    long blocksize) throws IOException, CommonFSUtils.StreamLacksCapabilityException {
  Collection<String> operations = conf.getStringCollection(ALLOWED_OPERATIONS);
  if (operations.isEmpty() || operations.contains(AllowedOperations.all.name())) {
    doAppends = doSyncs = true;
  } else if (operations.contains(AllowedOperations.none.name())) {
    doAppends = doSyncs = false;
  } else {
    doAppends = operations.contains(AllowedOperations.append.name());
    doSyncs = operations.contains(AllowedOperations.sync.name());
  }
  LOG.info("IOTestWriter initialized with appends " + (doAppends ? "enabled" : "disabled") +
      " and syncs " + (doSyncs ? "enabled" : "disabled"));
  super.init(fs, path, conf, overwritable, blocksize);
}
 
Example 9
Source File: CsvBlurDriverTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvBlurDriverTest3() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000", "-p", "SNAPPY");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
  assertEquals("true", configuration.get(CsvBlurDriver.MAPRED_COMPRESS_MAP_OUTPUT));
  assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC));
}
 
Example 10
Source File: QueryInputFormat.java    From Halyard with Apache License 2.0 5 votes vote down vote up
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    ArrayList<InputSplit> splits = new ArrayList<>();
    Configuration conf = context.getConfiguration();
    for (String qName : conf.getStringCollection(QUERIES)) {
        int repeatCount = conf.getInt(PREFIX + qName + REPEAT_SUFFIX, 1);
        String query = conf.get(PREFIX + qName + QUERY_SUFFIX);
        for (int i=0; i<repeatCount; i++) {
            splits.add(new QueryInputSplit(qName, query , i));
        }
    }
    return splits;
}
 
Example 11
Source File: HConnectionManagerMultiClusterWrapper.java    From HBase.MCC with Apache License 2.0 5 votes vote down vote up
public static HConnection createConnection(Configuration conf)
    throws IOException {

  Logger LOG = Logger.getLogger(HConnectionManagerMultiClusterWrapper.class);

  Collection < String > failoverClusters = conf
          .getStringCollection(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG);

  if (failoverClusters.size() == 0) {
    LOG.info(" -- Getting a signle cluster connection !!");
    return HConnectionManager.createConnection(conf);
  } else {

    Map<String, Configuration> configMap = HBaseMultiClusterConfigUtil
        .splitMultiConfigFile(conf);

    LOG.info(" -- Getting primary Connction");
    HConnection primaryConnection = HConnectionManager
        .createConnection(configMap
            .get(HBaseMultiClusterConfigUtil.PRIMARY_NAME));
    LOG.info(" --- Got primary Connction");

    ArrayList<HConnection> failoverConnections = new ArrayList<HConnection>();

    for (Entry<String, Configuration> entry : configMap.entrySet()) {
      if (!entry.getKey().equals(HBaseMultiClusterConfigUtil.PRIMARY_NAME)) {
        LOG.info(" -- Getting failure Connction");
        failoverConnections.add(HConnectionManager.createConnection(entry
            .getValue()));
        LOG.info(" --- Got failover Connction");
      }
    }
    
    return new HConnectionMultiCluster(conf, primaryConnection,
        failoverConnections.toArray(new HConnection[0]));
  }
}
 
Example 12
Source File: FSImage.java    From RDFS with Apache License 2.0 5 votes vote down vote up
static Collection<File> getCheckpointEditsDirs(Configuration conf,
    String defaultName) {
  Collection<String> dirNames = conf
      .getStringCollection("fs.checkpoint.edits.dir");
  if (dirNames.size() == 0 && defaultName != null) {
    dirNames.add(defaultName);
  }
  Collection<File> dirs = new ArrayList<File>(dirNames.size());
  for (String name : dirNames) {
    dirs.add(new File(name));
  }
  return dirs;
}
 
Example 13
Source File: FSImage.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
static Collection<File> getCheckpointEditsDirs(Configuration conf,
                                                String defaultName) {
   Collection<String> dirNames = 
               conf.getStringCollection("fs.checkpoint.edits.dir");
if (dirNames.size() == 0 && defaultName != null) {
  dirNames.add(defaultName);
}
Collection<File> dirs = new ArrayList<File>(dirNames.size());
for(String name : dirNames) {
  dirs.add(new File(name));
}
return dirs;    
 }
 
Example 14
Source File: QueryInputFormat.java    From Halyard with Apache License 2.0 5 votes vote down vote up
public static void addQuery(Configuration conf, String name, String query, int repeatCount) {
    Collection<String> qNames = conf.getStringCollection(QUERIES);
    qNames.add(name);
    conf.set(PREFIX + name + QUERY_SUFFIX, query);
    conf.setInt(PREFIX + name + REPEAT_SUFFIX, repeatCount);
    conf.setStrings(QUERIES, qNames.toArray(new String[qNames.size()]));
}
 
Example 15
Source File: CsvBlurMapper.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
@Override
protected void setup(Context context) throws IOException, InterruptedException {
  super.setup(context);
  Configuration configuration = context.getConfiguration();
  _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
  _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
  if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
    try {
      _digest = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IOException(e);
    }
  }
  _columnNameMap = getFamilyAndColumnNameMap(configuration);
  _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)), UTF_8);
  _splitter = Splitter.on(_separator);
  Path fileCurrentlyProcessing = getCurrentFile(context);
  Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
  OUTER: for (String family : families) {
    Collection<String> pathStrCollection = configuration
        .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
    for (String pathStr : pathStrCollection) {
      Path path = new Path(pathStr);
      FileSystem fileSystem = path.getFileSystem(configuration);
      path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
      if (isParent(path, fileCurrentlyProcessing)) {
        _familyFromPath = family;
        _familyNotInFile = true;
        break OUTER;
      }
    }
  }
}
 
Example 16
Source File: BulkInputFormat.java    From datawave with Apache License 2.0 5 votes vote down vote up
/**
 * Gets the columns to be mapped over from this configuration object.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @return a set of columns
 * @see #fetchColumns(Configuration, Collection)
 */
protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
    Set<Pair<Text,Text>> columns = new HashSet<>();
    for (String col : conf.getStringCollection(COLUMNS)) {
        int idx = col.indexOf(":");
        Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes()) : Base64.decodeBase64(col.substring(0, idx).getBytes()));
        Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes()));
        columns.add(new Pair<>(cf, cq));
    }
    return columns;
}
 
Example 17
Source File: DataNode.java    From RDFS with Apache License 2.0 4 votes vote down vote up
static Collection<URI> getStorageDirs(Configuration conf) {
  Collection<String> dirNames =
    conf.getStringCollection("dfs.data.dir");
  return Util.stringCollectionAsURIs(dirNames);
}
 
Example 18
Source File: CsvBlurMapper.java    From incubator-retired-blur with Apache License 2.0 3 votes vote down vote up
/**
 * Adds the column layout for the given family.
 * 
 * @param configuration
 *          the configuration to apply the layout.
 * @param family
 *          the family name.
 * @param columns
 *          the column names.
 */
public static void addColumns(Configuration configuration, String family, String... columns) {
  Collection<String> families = new TreeSet<String>(configuration.getStringCollection(BLUR_CSV_FAMILIES));
  families.add(family);
  configuration.setStrings(BLUR_CSV_FAMILIES, families.toArray(new String[] {}));
  configuration.setStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family, columns);
}
 
Example 19
Source File: HAUtil.java    From big-c with Apache License 2.0 2 votes vote down vote up
/**
 * @param conf Configuration. Please use getRMHAIds to check.
 * @return RM Ids on success
 */
public static Collection<String> getRMHAIds(Configuration conf) {
  return  conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
}
 
Example 20
Source File: DFSUtil.java    From RDFS with Apache License 2.0 2 votes vote down vote up
/**
 * Returns collection of nameservice Ids from the configuration.
 * @param conf configuration
 * @return collection of nameservice Ids
 */
public static Collection<String> getNameServiceIds(Configuration conf) {
  return conf.getStringCollection(FSConstants.DFS_FEDERATION_NAMESERVICES);
}