Java Code Examples for com.typesafe.config.ConfigFactory#parseMap()

@Test
public void testCustomDelimiterHash() {
  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("dep1", testDataFrame());

  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(HashDeriver.DELIMITER_CONFIG, ":::");
  Config config = ConfigFactory.parseMap(configMap);

  HashDeriver d = new HashDeriver();
  assertNoValidationFailures(d, config);
  d.configure(config);

  Dataset<Row> derived = d.derive(dependencies);

  assertEquals(1, derived.count());
  assertEquals(testDataFrame().schema().size() + 1, derived.schema().size());
  assertTrue(Lists.newArrayList(derived.schema().fieldNames()).contains(HashDeriver.DEFAULT_HASH_FIELD_NAME));
  assertEquals(
      "d85bcfeacb088fcc7e8ded019ed48ec2",
      derived.collectAsList().get(0).get(derived.schema().size() - 1));
}
 

@Test
public void readInputFormat() throws Exception {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(FileSystemInput.FORMAT_CONFIG, "input-format");
  paramMap.put(FileSystemInput.PATH_CONFIG, FileSystemInput.class.getResource(CSV_DATA).getPath());
  paramMap.put(FileSystemInput.INPUT_FORMAT_TYPE_CONFIG, TextInputFormat.class.getCanonicalName());
  paramMap.put("translator" + "." + ComponentFactory.TYPE_CONFIG_NAME,
      DummyInputFormatTranslator.class.getCanonicalName());
  config = ConfigFactory.parseMap(paramMap);

  FileSystemInput formatInput = new FileSystemInput();
  assertNoValidationFailures(formatInput, config);
  formatInput.configure(config);

  Dataset<Row> results = formatInput.read();

  assertEquals("Invalid number of rows", 4, results.count());
  assertEquals("Invalid first row result", 0L, results.first().getLong(0));
  assertEquals("Invalid first row result", "One,Two,Three,Four", results.first().getString(1));
}
 

@Override
public Config apply(final String systemVcapServices) {
    checkNotNull(systemVcapServices, "system VCAP services string");
    if (systemVcapServices.isEmpty()) {
        return ConfigFactory.empty();
    }

    final Config vcapServicesConfig = tryToParseString(systemVcapServices);
    final Set<Map.Entry<String, ConfigValue>> vcapServicesConfigEntries = vcapServicesConfig.entrySet();

    final Map<String, Object> result = new HashMap<>(vcapServicesConfigEntries.size());
    for (final Map.Entry<String, ConfigValue> serviceConfigEntry : vcapServicesConfigEntries) {
        result.put(serviceConfigEntry.getKey(), convertConfigListToConfigObject(serviceConfigEntry.getValue()));
    }
    return ConfigFactory.parseMap(result);
}
 

@Test
public void testNoUUIDKey() {
  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(AppendPlanner.KEY_FIELD_NAMES_CONFIG_NAME, Lists.newArrayList("key"));
  Config config = ConfigFactory.parseMap(configMap);

  AppendPlanner ap = new AppendPlanner();
  assertNoValidationFailures(ap, config);
  ap.configure(config);

  List<Tuple2<MutationType, Dataset<Row>>> planned = ap.planMutationsForSet(dataFrame);

  assertEquals(planned.size(), 1);

  Dataset<Row> plannedDF = planned.get(0)._2();

  assertEquals(planned.get(0)._1(), MutationType.INSERT);
  assertEquals(plannedDF.count(), 1);

  Row plannedRow = plannedDF.collectAsList().get(0);

  assertNull(plannedRow.get(plannedRow.fieldIndex("key")));
}
 

@Test
public void testSpecifiedDependency() {
  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("dep1", testDataFrame());

  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(HashDeriver.STEP_NAME_CONFIG, "dep1");
  Config config = ConfigFactory.parseMap(configMap);

  HashDeriver d = new HashDeriver();
  assertNoValidationFailures(d, config);
  d.configure(config);

  Dataset<Row> derived = d.derive(dependencies);

  assertEquals(1, derived.count());
  assertEquals(testDataFrame().schema().size() + 1, derived.schema().size());
  assertTrue(Lists.newArrayList(derived.schema().fieldNames()).contains(HashDeriver.DEFAULT_HASH_FIELD_NAME));
  assertEquals(
      "4891a9d87f8f46a5c8c19c3059864146",
      derived.collectAsList().get(0).get(derived.schema().size() - 1));
}
 

@Test
public void test() throws Exception {

  URL prefix = getClass().getResource("/configStore");

  Config config = ConfigFactory.parseMap(ImmutableMap.of(
     ConfigClientBasedPolicyFactory.CONFIG_KEY_URI_PREFIX_KEY, "simple-" + prefix.toString()
  ));

  SharedResourcesBroker<ThrottlingServerScopes> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(
      ConfigFactory.empty(), ThrottlingServerScopes.GLOBAL.defaultScopeInstance());

  ConfigClientBasedPolicyFactory policyFactory = new ConfigClientBasedPolicyFactory();

  ThrottlingPolicy policy =
      policyFactory.createPolicy(new SharedLimiterKey("ConfigBasedPolicyTest/resource1"), broker, config);
  Assert.assertEquals(policy.getClass(), QPSPolicy.class);
  Assert.assertEquals(((QPSPolicy) policy).getQps(), 100);

  policy =
      policyFactory.createPolicy(new SharedLimiterKey("ConfigBasedPolicyTest/resource2"), broker, config);
  Assert.assertEquals(policy.getClass(), CountBasedPolicy.class);
  Assert.assertEquals(((CountBasedPolicy) policy).getCount(), 50);
}
 

@Test
public void testSameDbDifferentUser() throws IOException {

  Config config1 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url1",
      ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user1",
      ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd"));

  Config config2 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url1",
      ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user2",
      ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd"));

  BasicDataSource basicDataSource1 = MysqlDataSourceFactory.get(config1,
      SharedResourcesBrokerFactory.getImplicitBroker());

  BasicDataSource basicDataSource2 = MysqlDataSourceFactory.get(config2,
      SharedResourcesBrokerFactory.getImplicitBroker());

  Assert.assertNotEquals(basicDataSource1, basicDataSource2);
}
 

@Test
public void testSelectionPolicyInit() throws Exception {

  // Using alias
  AccessControlAction testRetentionAction =
      new AccessControlAction(ConfigFactory.parseMap(ImmutableMap.<String, String> of("selection.policy.class",
          "SelectAfterTimeBasedPolicy", "selection.timeBased.lookbackTime", "7d")), null, ConfigFactory.empty());

  Assert.assertEquals(testRetentionAction.getSelectionPolicy().getClass(), SelectAfterTimeBasedPolicy.class);

  // Using complete class name
  testRetentionAction =
      new AccessControlAction(ConfigFactory.parseMap(ImmutableMap.<String, String> of("selection.policy.class",
          SelectAfterTimeBasedPolicy.class.getName(), "selection.timeBased.lookbackTime", "7d")), null,
          ConfigFactory.empty());

  Assert.assertEquals(testRetentionAction.getSelectionPolicy().getClass(), SelectAfterTimeBasedPolicy.class);
}
 

@Test
public void testBuildLdapAuthConnectionString() {
  Map<String, Object> configMap = new HashMap<>();
  configMap.put(HOST_CONFIG, "testhost");
  configMap.put(QUERY_TYPE_CONFIG, "refresh");
  configMap.put(QUERY_TABLE_CONFIG, "testtable");
  configMap.put(AUTH_CONFIG, "ldap");
  configMap.put(USERNAME_CONFIG, "user");
  configMap.put(PASSWORD_CONFIG, "password");
  Config config = ConfigFactory.parseMap(configMap);
  ImpalaMetadataTask metadataTask = new ImpalaMetadataTask();
  metadataTask.configure(config);

  String connectionString = metadataTask.buildConnectionString();
  assertEquals("jdbc:hive2://testhost:21050/", connectionString);
}
 

@Test
public void testDeriveAddRangePartitionValueQuery() {
  Map<String, Object> configMap = new HashMap<>();
  configMap.put(HOST_CONFIG, "testhost");
  configMap.put(QUERY_TYPE_CONFIG, "add_partition");
  configMap.put(QUERY_TABLE_CONFIG, "testtable");
  configMap.put(QUERY_PART_RANGE_VAL_CONFIG, "20190122");
  configMap.put(AUTH_CONFIG, "none");
  Config config = ConfigFactory.parseMap(configMap);
  ImpalaMetadataTask metadataTask = new ImpalaMetadataTask();
  metadataTask.configure(config);

  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  String query = metadataTask.deriveQuery(dependencies);

  assertEquals("ALTER TABLE testtable ADD IF NOT EXISTS RANGE PARTITION VALUE = 20190122", query);
}
 

@Test
public void testGetTimeUnitValid() {
  String key = "a.b.c";
  TimeUnit expectedTimeUnit = TimeUnit.DAYS;
  TimeUnit defaultTimeUnit = TimeUnit.MILLISECONDS;
  Config cfg = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder()
      .put(key, TimeUnit.DAYS.name())
      .build());
  TimeUnit timeUnit = ConfigUtils.getTimeUnit(cfg, key, defaultTimeUnit);
  Assert.assertEquals(timeUnit, expectedTimeUnit);
}
 

private static Config loadFromEnvVars() {
    final Map<String, String> envMap = new HashMap<>();
    System.getenv().entrySet().stream()
            .filter((e) -> isHaystackAgentEnvVar(e.getKey()))
            .forEach((e) -> {
                final String normalizedKey = getNormalizedKey(e.getKey());
                envMap.put(normalizedKey, e.getValue());
            });

    return ConfigFactory.parseMap(envMap);
}
 

@Test (expected = RuntimeException.class)
public void testMissingDependency() {
  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("dep1", testDataFrame());

  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(HashDeriver.STEP_NAME_CONFIG, "dep2");
  Config config = ConfigFactory.parseMap(configMap);

  HashDeriver d = new HashDeriver();
  assertNoValidationFailures(d, config);
  d.configure(config);

  d.derive(dependencies);
}
 

@Test
public void missingNames() {
  Map<String, Object> paramMap = new HashMap<>();
  paramMap.put(ComponentFactory.TYPE_CONFIG_NAME, "flat");
  paramMap.put(FlatSchema.FIELD_TYPES_CONFIG, Lists.newArrayList("long", "int"));
  config = ConfigFactory.parseMap(paramMap);
  FlatSchema flatSchema = new FlatSchema(); 
  assertValidationFailures(flatSchema, config);
}
 

private static Config arrayToConfig(final KnownConfigValue[] knownConfigValues) {
    final Map<String, Object> fallbackValues = new HashMap<>(knownConfigValues.length);
    for (final KnownConfigValue knownConfigValue : knownConfigValues) {
        fallbackValues.put(knownConfigValue.getConfigPath(), knownConfigValue.getDefaultValue());
    }
    return ConfigFactory.parseMap(fallbackValues);
}
 

@Test
public void tryToGetInstanceWithMissingConfigAtConfigPath() {
    final Config config = ConfigFactory.parseMap(Collections.singletonMap("foo", "bar"));

    assertThatExceptionOfType(DittoConfigError.class)
            .isThrownBy(() -> DefaultScopedConfig.newInstance(config, KNOWN_CONFIG_PATH))
            .withMessage("Failed to get nested Config at <%s>!", KNOWN_CONFIG_PATH)
            .withCauseInstanceOf(ConfigException.Missing.class);
}
 

static Config getDefaultConfig() {
  return ConfigFactory.parseMap(ImmutableMap.<String, Object>builder()
         .put(EMBEDDED_MYSQL_ENABLED_FULL_KEY, true)
         .put(DBUSER_NAME_FULL_KEY, "testUser")
         .put(DBUSER_PASSWORD_FULL_KEY, "testPassword")
         .put(DBHOST_FULL_KEY, "localhost")
         .put(DBPORT_FULL_KEY, 3306)
         .build());
}
 

@Test
public void tryToCreateInstanceWithNullFallBackValues() {
    final Config config = ConfigFactory.parseMap(Collections.singletonMap(KNOWN_CONFIG_PATH, "nothing"));

    assertThatExceptionOfType(DittoConfigError.class)
            .isThrownBy(() -> ConfigWithFallback.newInstance(config, KNOWN_CONFIG_PATH, null))
            .withCause(new NullPointerException("The fall-back values must not be null!"));
}
 

@Test
public void testMultipleKeyFieldNames() throws Exception {
  StructType ordersSchema = DataTypes.createStructType(Lists.newArrayList(
      DataTypes.createStructField("order_id", DataTypes.IntegerType, true),
      DataTypes.createStructField("product_name", DataTypes.StringType, true),
      DataTypes.createStructField("customer_first", DataTypes.StringType, true),
      DataTypes.createStructField("customer_last", DataTypes.StringType, true)));

  StructType customersSchema = DataTypes.createStructType(Lists.newArrayList(
      DataTypes.createStructField("customer_first", DataTypes.StringType, true),
      DataTypes.createStructField("customer_last", DataTypes.StringType, true),
      DataTypes.createStructField("state", DataTypes.StringType, true)));

  List<Row> orderRows = Lists.newArrayList();
  orderRows.add(RowFactory.create(1000, "Envelopes", "Jane", "Smith"));
  orderRows.add(RowFactory.create(1001, "Stamps", "Jane", "Smith"));
  orderRows.add(RowFactory.create(1002, "Pens", "Jane", "Smith"));
  orderRows.add(RowFactory.create(1003, "Paper", "Jane", "Bloggs"));

  List<Row> customerRows = Lists.newArrayList();
  customerRows.add(RowFactory.create("Jane", "Smith", "NY"));
  customerRows.add(RowFactory.create("Jane", "Bloggs", "CA"));

  Dataset<Row> orders = Contexts.getSparkSession().createDataFrame(orderRows, ordersSchema);
  Dataset<Row> customers = Contexts.getSparkSession().createDataFrame(customerRows, customersSchema);

  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("orders", orders);
  dependencies.put("customers", customers);

  Map<String, Object> configMap = Maps.newHashMap();
  configMap.put(NestDeriver.NEST_FROM_CONFIG_NAME, "orders");
  configMap.put(NestDeriver.NEST_INTO_CONFIG_NAME, "customers");
  configMap.put(NestDeriver.KEY_FIELD_NAMES_CONFIG_NAME, Lists.newArrayList("customer_first", "customer_last"));
  configMap.put(NestDeriver.NESTED_FIELD_NAME_CONFIG_NAME, "orders");
  Config config = ConfigFactory.parseMap(configMap);

  NestDeriver deriver = new NestDeriver();
  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  Dataset<Row> nested = deriver.derive(dependencies);

  assertEquals(nested.count(), 2);

  List<Row> smith = nested.where("customer_first = 'Jane' AND customer_last = 'Smith'").collectAsList();
  assertEquals(smith.size(), 1);
  Row smithRow = smith.get(0);
  assertEquals(smithRow.getList(smithRow.fieldIndex("orders")).size(), 3);

  List<Row> bloggs = nested.where("customer_first = 'Jane' AND customer_last = 'Bloggs'").collectAsList();
  assertEquals(bloggs.size(), 1);
  Row bloggsRow = bloggs.get(0);
  assertEquals(bloggsRow.getList(bloggsRow.fieldIndex("orders")).size(), 1);
}
 

@Test
public void shouldCreateAllAuthorizationPolicy() {
  final Config config = ConfigFactory.parseMap(Map.of(AUTHORIZATION_REQUIRE_ALL_CONFIG, "true"));
  final AuthorizationPolicy policy = AuthorizationPolicy.fromConfig(config);
  assertThat(policy, is(instanceOf(AllAuthorizationPolicy.class)));
}