com.typesafe.config.ConfigValueFactory Java Examples

@Test
public void testCarryForwardWhenNull() {
  p = new EventTimeHistoryPlanner();
  config = config.withValue(EventTimeHistoryPlanner.CARRY_FORWARD_CONFIG_NAME, ConfigValueFactory.fromAnyRef(true));
  assertNoValidationFailures(p, config);
  p.configure(config);

  existing.add(new RowWithSchema(existingSchema, "a", "hello", 100L, 100L, 253402214400000L, EventTimeHistoryPlanner.CURRENT_FLAG_DEFAULT_YES, ""));
  arriving.add(new RowWithSchema(arrivingSchema, "a", null, 200L));
  key = new RowWithSchema(keySchema, "a");

  List<Row> planned = p.planMutationsForKey(key, arriving, existing);

  assertEquals(planned.size(), 2);
  assertEquals(PlannerUtils.getMutationType(planned.get(0)), MutationType.UPDATE);
  assertEquals(planned.get(0).getAs("value"), "hello");
  assertEquals(planned.get(0).getAs("startdate"), 100L);
  assertEquals(planned.get(0).getAs("enddate"), 199L);
  assertEquals(planned.get(0).getAs("currentflag"), EventTimeHistoryPlanner.CURRENT_FLAG_DEFAULT_NO);
  assertEquals(PlannerUtils.getMutationType(planned.get(1)), MutationType.INSERT);
  assertEquals(planned.get(1).getAs("value"), "hello");
  assertEquals(planned.get(1).getAs("startdate"), 200L);
  assertEquals(planned.get(1).getAs("enddate"), 253402214400000L);
  assertEquals(planned.get(1).getAs("currentflag"), EventTimeHistoryPlanner.CURRENT_FLAG_DEFAULT_YES);
}
 

@Test
public void testCustomFormat() throws ParseException {
  format = new SimpleDateFormat("dd-MMM-yyyy HH mm ss SSS");
  config = config.withValue(StringDateTimeModel.DATETIME_FORMAT_CONFIG,
      ConfigValueFactory.fromAnyRef("dd-MMM-yyyy HH mm ss SSS"));
  assertNoValidationFailures(tm, config);
  tm.configure(config);
  tm.configureFieldNames(Lists.newArrayList(field.name()));
  
  first = new RowWithSchema(schema, "31-DEC-2017 23 45 12 345");
  second = new RowWithSchema(schema, "01-JAN-2018 00 00 00 000");
  
  testSchema();
  testBefore();
  testSimultaneous();
  testAfter();
  testFarFuture();
  testCurrentSystemTime();
  testPrecedingSystemTime();
  testAppendFields();
}
 

@Test
public void testEmptyValue() {
  ProvidesValidations validee = new ProvidesValidations() {
    @Override
    public Validations getValidations() {
      return Validations.builder().mandatoryPath("hello").build();
    }
  };

  Properties configProps = new Properties();
  configProps.setProperty("hello", "");
  Config config = ConfigFactory.parseProperties(configProps);

  assertValidationFailures(validee, config);

  config = config.withValue("hello", ConfigValueFactory.fromAnyRef(null));
  assertValidationFailures(validee, config);
}
 

private static ConfigObject getAsConfigObject(final ConfigList configList) {
    final Map<String, ConfigValue> flattenedConfigValues = new HashMap<>(configList.size());

    for (int i = 0; i < configList.size(); i++) {
        final ConfigValue configValue = configList.get(i);
        final String configPath;
        if (ConfigValueType.OBJECT == configValue.valueType()) {
            configPath = getName((ConfigObject) configValue);
        } else {
            configPath = String.valueOf(i);
        }
        flattenedConfigValues.put(configPath, configValue);
    }

    return ConfigValueFactory.fromMap(flattenedConfigValues);
}
 

@DataProvider(value = {
    "true",
    "false"
})
@Test
public void codahaleMetricsEngine_is_configured_with_jvm_metrics_on_or_off_based_on_property(boolean reportJvmMetrics) {
    // given
    configForTesting = generateAppConfigWithMetricsEnabledOrDisabled(true, true, false)
        .withValue("metrics.reportJvmMetrics", ConfigValueFactory.fromAnyRef(reportJvmMetrics));
    appGuiceModule = new AppGuiceModule(configForTesting);
    injector = generateInjector(appGuiceModule, configForTesting);

    // when
    CodahaleMetricsEngine engine = injector.getInstance(CodahaleMetricsEngine.class);

    // then
    assertThat(Whitebox.getInternalState(engine, "jvmMetricsAdded")).isEqualTo(reportJvmMetrics);
}
 

@Test
public void testFindReplaceStringValues() {
  Config baseConfig = ConfigFactory.parseString("a: ${replaceme}, b: \"${replaceme}\", " +
      "c: [${replaceme}, \"hello\", ${replaceme}], d: [\"${replaceme}\", \"world\", " +
      "\"${replaceme}\"], e: { f: \"${replaceme}\", g: [\"${replaceme}\"] }");
  Config resolvedConfig = baseConfig.resolveWith(
      ConfigFactory.empty().withValue("replaceme", ConfigValueFactory.fromAnyRef("REPLACED")));
  Config replacedConfig = ConfigUtils.findReplaceStringValues(
      resolvedConfig, "\\$\\{replaceme\\}", "REPLACED");

  assertEquals(replacedConfig.getString("a"), "REPLACED");
  assertEquals(replacedConfig.getString("b"), "REPLACED");
  assertEquals(replacedConfig.getStringList("c").get(0), "REPLACED");
  assertEquals(replacedConfig.getStringList("c").get(1), "hello");
  assertEquals(replacedConfig.getStringList("c").get(2), "REPLACED");
  assertEquals(replacedConfig.getStringList("d").get(0), "REPLACED");
  assertEquals(replacedConfig.getStringList("d").get(1), "world");
  assertEquals(replacedConfig.getStringList("d").get(2), "REPLACED");
  assertEquals(replacedConfig.getConfig("e").getString("f"), "REPLACED");
  assertEquals(replacedConfig.getConfig("e").getStringList("g").get(0), "REPLACED");
}
 

@Test
public void testAcksWithSingleData() throws Exception {
  sourceConfiguration = sourceConfiguration.withValue("akka.remote.auto-ack",
    ConfigValueFactory.fromAnyRef("on"));
  source = new AkkaTestSource(sourceConfiguration);

  feederActorSystem.actorOf(
    Props.create(FeederActor.class, FeederActor.MessageTypes.SINGLE_DATA),
    feederActorName);

  source.open(config);
  sourceThread.start();

  while (DummySourceContext.numElementsCollected != 1) {
    Thread.sleep(5);
  }

  int noOfRetries = 1;
  while (Message.ACK_MESSAGE == null && noOfRetries <= 5) {
    Thread.sleep(5);
    noOfRetries++;
  }
  Assertions.assertEquals("ack", Message.ACK_MESSAGE);
}
 

@Test
public void testHandlerReflection() {
    Config config =
            ConfigFactory.load().withValue("bag.30", ConfigValueFactory.fromAnyRef(BagForTest.class.getName()));
    BaggageHandlerRegistry registry = BaggageHandlerRegistry.create(config);

    assertEquals(1, registry.registrations.keys.length);
    assertEquals(1, registry.registrations.handlers.length);
    assertEquals(BagKey.indexed(30), registry.registrations.keys[0]);
    assertEquals(handler, registry.registrations.handlers[0]);

    BaggageHandlerForTest handler2 = new BaggageHandlerForTest();
    registry.doAdd(BagKey.indexed(5), handler2);

    assertEquals(2, registry.registrations.keys.length);
    assertEquals(2, registry.registrations.handlers.length);
    assertEquals(BagKey.indexed(5), registry.registrations.keys[0]);
    assertEquals(handler2, registry.registrations.handlers[0]);
    assertEquals(BagKey.indexed(30), registry.registrations.keys[1]);
    assertEquals(handler, registry.registrations.handlers[1]);

}
 

@Test
public void createByUriWithExtraSettings() {
    // prepare
    final boolean sslEnabled = false;
    final Duration maxIdleTime = Duration.ofMinutes(10);
    final Duration maxLifeTime = Duration.ofMinutes(25);
    final String uri = createUri(sslEnabled) + "&maxIdleTimeMS=" + maxIdleTime.toMillis()
            + "&maxLifeTimeMS=" + maxLifeTime.toMillis();

    final Config config = CONFIG.withValue(MONGO_URI_CONFIG_KEY, ConfigValueFactory.fromAnyRef(uri));
    final DefaultMongoDbConfig mongoDbConfig = DefaultMongoDbConfig.of(config);

    // test
    final MongoClientWrapper underTest = MongoClientWrapper.newInstance(mongoDbConfig);

    // verify
    assertThat(underTest.getSettings().getConnectionPoolSettings().
             getMaxConnectionIdleTime(TimeUnit.MILLISECONDS)).isEqualTo(maxIdleTime.toMillis());
    assertThat(underTest.getSettings().getConnectionPoolSettings().
            getMaxConnectionLifeTime(TimeUnit.MILLISECONDS)).isEqualTo(maxLifeTime.toMillis());
}
 

@Test
public void testCustomFormat() throws ParseException {
  format = new SimpleDateFormat("dd-MMM-yyyy");
  config = config.withValue(StringDateTimeModel.DATETIME_FORMAT_CONFIG,
      ConfigValueFactory.fromAnyRef("dd-MMM-yyyy"));
  assertNoValidationFailures(tm, config);
  tm.configure(config);
  tm.configureFieldNames(Lists.newArrayList(field.name()));
  
  first = new RowWithSchema(schema, "31-DEC-2017");
  second = new RowWithSchema(schema, "01-JAN-2018");
  
  testSchema();
  testBefore();
  testSimultaneous();
  testAfter();
  testFarFuture();
  testCurrentSystemTime();
  testPrecedingSystemTime();
  testAppendFields();
}
 

private Config appendDittoInfo(final Config config) {
    final String instanceId = InstanceIdentifierSupplier.getInstance().get();

    final ConfigValue service = ConfigFactory.empty()
            .withValue("name", ConfigValueFactory.fromAnyRef(serviceName))
            .withValue("instance-id", ConfigValueFactory.fromAnyRef(instanceId))
            .root();

    final ConfigValue vmArgs =
            ConfigValueFactory.fromIterable(ManagementFactory.getRuntimeMXBean().getInputArguments());

    final ConfigValue env = ConfigValueFactory.fromMap(System.getenv());

    return config.withValue("ditto.info",
            ConfigFactory.empty()
                    .withValue("service", service)
                    .withValue("vm-args", vmArgs)
                    .withValue("env", env)
                    .root());
}
 

@Test
public void testMissingField() throws Exception {
  Dataset<Row> source = createTestDataframe();

  List<String> inListLiteral = Arrays.asList("1", "2", "3");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_VALUES_CONFIG, ConfigValueFactory.fromIterable(inListLiteral));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  // Doesn't return anything because the IN values don't match the target column values
  List<Row> results = deriver.derive(dependencies).select("value").collectAsList();
  assertThat(results.size(), is(0));
}
 

@Test
public void testMissingRefField() throws Exception {
  Dataset<Row> source = createTestDataframe();
  Dataset<Row> ref = createTestDataframe();

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);
  dependencies.put("df2", ref);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_FIELD_CONFIG, ConfigValueFactory.fromAnyRef("value"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df2"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  // Doesn't return anything because the IN values don't match the target column values
  List<Row> results = deriver.derive(dependencies).select("value").collectAsList();
  assertThat(results.size(), is(0));
}
 

@Test
public void testWrongRefField() throws Exception {
  thrown.expect(RuntimeException.class);
  thrown.expectMessage("Error executing IN list filtering");

  Dataset<Row> source = createTestDataframe();
  Dataset<Row> ref = createTestDataframe();

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);
  dependencies.put("df2", ref);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_FIELD_CONFIG, ConfigValueFactory.fromAnyRef("value"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df2"))
      .withValue(InListDeriver.INLIST_REFFIELD_CONFIG, ConfigValueFactory.fromAnyRef("non_existent_ref_field"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  deriver.derive(dependencies);
}
 

@Test
public void testWrongSourceTargetDependency() throws Exception {
  thrown.expect(RuntimeException.class);
  thrown.expectMessage("In-list deriver does not have step 'step_df_missing' in its dependencies");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", null);
  dependencies.put("df2", null);

  List<String> inListLiteral = Arrays.asList("1", "2", "3");

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_VALUES_CONFIG, ConfigValueFactory.fromIterable(inListLiteral))
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("step_df_missing"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  deriver.derive(dependencies);
}
 

@Test
public void testWrongReferenceDependency() throws Exception {
  thrown.expect(RuntimeException.class);
  thrown.expectMessage("Targeted step, 'df1', is null");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", null);
  dependencies.put("df2", null);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df_missing"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("ref_df_missing"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  deriver.derive(dependencies);
}
 

@Test
public void testWrongField() throws Exception {
  thrown.expect(RuntimeException.class);
  thrown.expectMessage("Error executing IN list filtering");

  Dataset<Row> source = createTestDataframe();
  List<String> inListLiteral = Arrays.asList("1", "2", "3");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_FIELD_CONFIG, ConfigValueFactory.fromAnyRef("non_existing_field"))
      .withValue(InListDeriver.INLIST_VALUES_CONFIG, ConfigValueFactory.fromIterable(inListLiteral));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  deriver.derive(dependencies);
}
 

@Test
public void testAllowEmptyValue() {
  ProvidesValidations validee = new ProvidesValidations() {
    @Override
    public Validations getValidations() {
      return Validations.builder().mandatoryPath("hello").allowEmptyValue("hello").build();
    }
  };

  Properties configProps = new Properties();
  configProps.setProperty("hello", "");
  Config config = ConfigFactory.parseProperties(configProps);
  assertNoValidationFailures(validee, config);

  config = config.withValue("hello", ConfigValueFactory.fromIterable(
      Lists.newArrayList("")));
  assertNoValidationFailures(validee, config);
}
 

@Test
public void testDeriveAsFields() {
  Config config = ConfigFactory.empty()
      .withValue(ParseJSONDeriver.STEP_NAME_CONFIG, ConfigValueFactory.fromAnyRef("source"))
      .withValue(ParseJSONDeriver.FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("value"))
      .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME,
          ConfigValueFactory.fromAnyRef(new AvroSchema().getAlias()))
      .withValue(ParseJSONDeriver.SCHEMA_CONFIG + "." + AvroSchema.AVRO_FILE_CONFIG,
          ConfigValueFactory.fromAnyRef(
              getClass().getResource("/json/json-schema.avsc").getFile()));

  Row row = deriveRow(config);

  assertEquals(6, row.length());
  assertEquals("hello", row.getAs("f1"));
  assertEquals(1.2f, row.getAs("f2"));
  assertEquals(true, row.getAs("f3"));
  assertEquals("world", row.getStruct(row.fieldIndex("f4")).getAs("f41"));
  assertEquals("v5", row.getList(row.fieldIndex("f5")).get(0));
  assertEquals(false, ((Row)row.getList(row.fieldIndex("f6")).get(0)).getList(0).get(0));
  assertEquals(false, ((Row)row.getList(row.fieldIndex("f6")).get(0)).getList(0).get(1));
}
 

@Test
public void testRegexDelimiter() {
  String delimited = "val1 \"val2 ...\" val3 \"val4 val5\"";
  
  Config config = ConfigFactory.empty()
      .withValue(DelimitedTranslator.SCHEMA_CONFIG + "." + ComponentFactory.TYPE_CONFIG_NAME,
          ConfigValueFactory.fromAnyRef("flat"))
      .withValue(DelimitedTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_NAMES_CONFIG,
          ConfigValueFactory.fromIterable(
              Lists.newArrayList("field1", "field2", "field3", "field4")))
      .withValue(DelimitedTranslator.SCHEMA_CONFIG + "." + FlatSchema.FIELD_TYPES_CONFIG,
          ConfigValueFactory.fromIterable(
              Lists.newArrayList("string", "string", "string", "string")))
      .withValue(DelimitedTranslator.DELIMITER_CONFIG_NAME, 
          ConfigValueFactory.fromAnyRef(" (?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"))
      .withValue(DelimitedTranslator.DELIMITER_REGEX_CONFIG_NAME,
          ConfigValueFactory.fromAnyRef(true));

  DelimitedTranslator t = new DelimitedTranslator();
  assertNoValidationFailures(t, config);
  t.configure(config);
  Row raw = TestingMessageFactory.get("testkey", DataTypes.StringType, delimited, DataTypes.StringType);
  Row r = t.translate(raw).iterator().next();
  assertEquals(r.length(), 4);
  assertEquals(r.get(0), "val1");
  assertEquals(r.get(1), "\"val2 ...\"");
  assertEquals(r.get(2), "val3");
  assertEquals(r.get(3), "\"val4 val5\"");
}
 

@Test
public void testDeriveReference() throws Exception {
  Dataset<Row> source = createTestDataframe();
  Dataset<Row> ref = createTestDataframe().withColumnRenamed("id", "fk").filter("value < 6");

  List<String> ids = Arrays.asList("A", "B", "C", "D", "E");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);
  dependencies.put("df2", ref);
  dependencies.put("df3", null);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_FIELD_CONFIG, ConfigValueFactory.fromAnyRef("id"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df2"))
      .withValue(InListDeriver.INLIST_REFFIELD_CONFIG, ConfigValueFactory.fromAnyRef("fk"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  List<Row> results = deriver.derive(dependencies).select("id").collectAsList();
  assertThat(results.size(), is(5));

  for (Row row : results) {
    assertThat(row.getString(0), in(ids));
  }
}
 

@Test
public void testIntegerDataType() throws Exception {
  List<Row> sourceList = Lists.newArrayList(
      RowFactory.create("A", "hello", 1),
      RowFactory.create("A", "world", 2),
      RowFactory.create("B", "hello", 3),
      RowFactory.create("C", "world", 4));
  StructType schema = DataTypes.createStructType(Lists.newArrayList(
    DataTypes.createStructField("entity_id", DataTypes.StringType, true),
    DataTypes.createStructField("key", DataTypes.StringType, true),
    DataTypes.createStructField("value", DataTypes.IntegerType, true)
  ));
  Dataset<Row> source = Contexts.getSparkSession().createDataFrame(sourceList, schema);

  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("source", source);
  
  Config config = ConfigFactory.empty()
      .withValue(PivotDeriver.STEP_NAME_CONFIG, ConfigValueFactory.fromAnyRef("source"))
      .withValue(PivotDeriver.ENTITY_KEY_FIELD_NAMES_CONFIG, ConfigValueFactory.fromAnyRef(Lists.newArrayList("entity_id")))
      .withValue(PivotDeriver.PIVOT_KEY_FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("key"))
      .withValue(PivotDeriver.PIVOT_VALUE_FIELD_NAME_CONFIG, ConfigValueFactory.fromAnyRef("value"));

  PivotDeriver d = new PivotDeriver();
  assertNoValidationFailures(d, config);
  d.configure(config);
  
  List<Row> results = d.derive(dependencies).collectAsList();
  
  assertEquals(results.size(), 3);
  assertTrue(results.contains(RowFactory.create("A", 1, 2)));
  assertTrue(results.contains(RowFactory.create("B", 3, null)));
  assertTrue(results.contains(RowFactory.create("C", null, 4)));
}
 

@Test
public void testAmbiguousConfig() {
  List<String> inListLiteral = Arrays.asList("1", "2", "3");
  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_VALUES_CONFIG, ConfigValueFactory.fromIterable(inListLiteral))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df2"));
  assertValidationFailures(new InListDeriver(), config);
}
 

@Test
public void testBatchSize() throws Exception {
  Dataset<Row> source = createTestDataframe();
  Dataset<Row> ref = createTestDataframe().withColumnRenamed("id", "fk").filter("value < 6");

  List<String> ids = Arrays.asList("A", "B", "C", "D", "E");

  Map<String, Dataset<Row>> dependencies = new HashMap<>();
  dependencies.put("df1", source);
  dependencies.put("df2", ref);

  Config config = ConfigFactory.empty()
      .withValue(InListDeriver.INLIST_BATCH_SIZE, ConfigValueFactory.fromAnyRef(1))
      .withValue(InListDeriver.INLIST_STEP_CONFIG, ConfigValueFactory.fromAnyRef("df1"))
      .withValue(InListDeriver.INLIST_FIELD_CONFIG, ConfigValueFactory.fromAnyRef("id"))
      .withValue(InListDeriver.INLIST_REFSTEP_CONFIG, ConfigValueFactory.fromAnyRef("df2"))
      .withValue(InListDeriver.INLIST_REFFIELD_CONFIG, ConfigValueFactory.fromAnyRef("fk"));

  InListDeriver deriver = new InListDeriver();

  assertNoValidationFailures(deriver, config);
  deriver.configure(config);

  List<Row> results = deriver.derive(dependencies).select("id").collectAsList();
  assertThat(results.size(), is(5));

  for (Row row : results) {
    assertThat(row.getString(0), in(ids));
  }
}
 

@Test(expected = RuntimeException.class)
public void wrongConfig() throws Exception {
  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  dependencies.put("df1", null);
  dependencies.put("df2", null);
  Config config = ConfigFactory.empty().withValue(DistinctDeriver.DISTINCT_STEP_CONFIG,
      ConfigValueFactory.fromAnyRef("df3"));
  DistinctDeriver deriver = new DistinctDeriver();
  assertNoValidationFailures(deriver, config);
  deriver.configure(config);
  deriver.derive(dependencies);
}
 

@Test
public void testElectionDisconnection() throws Exception {
  final CountDownLatch elected = new CountDownLatch(1);
  final CountDownLatch cancelled = new CountDownLatch(1);

  try(ZKClusterClient client = new ZKClusterClient(
      DEFAULT_SABOT_CONFIG
      .withValue(ClusterCoordinator.Options.ZK_ELECTION_POLLING, ConfigValueFactory.fromAnyRef("20ms"))
      .withValue(ClusterCoordinator.Options.ZK_ELECTION_TIMEOUT, ConfigValueFactory.fromAnyRef("100ms")),
      String.format("%s/dremio/test/test-cluster-id", zooKeeperServer.getConnectString()))
  ) {
    client.start();
    ElectionRegistrationHandle node1 = client.joinElection("test-election", new ElectionListener() {

      @Override
      public void onElected() {
        elected.countDown();
      }

      @Override
      public void onCancelled() {
        cancelled.countDown();
      }
    });

    assertTrue("No election happened", elected.await(5, TimeUnit.SECONDS));

    // Kill the server to force disconnection
    zooKeeperServer.closeServer();

    assertTrue("Node was not notified about cancellation", cancelled.await(5, TimeUnit.SECONDS));
  }
}
 

private Config getOrCreateMandatoryProperties(Config properties) {
    if (!properties.hasPath("akka.actor.provider")) {
        properties = properties.withValue("akka.actor.provider",
                ConfigValueFactory.fromAnyRef("akka.remote.RemoteActorRefProvider"));
    }

    if (!properties.hasPath("akka.remote.enabled-transports")) {
        properties = properties.withValue("akka.remote.enabled-transports",
                ConfigValueFactory.fromAnyRef(Collections.singletonList("akka.remote.netty.tcp")));
    }
    return properties;
}
 

@Test(expected = RuntimeException.class)
public void wrongConfigUnknownColumnIncludeFields() throws Exception {
  List<String> includeFields = Arrays.asList("sourceid", "sourcename", "sourcestatus", "TestField");

  Map<String, Dataset<Row>> dependencies = Maps.newHashMap();
  Dataset<Row> inputdataframe = testDataframe();
  dependencies.put("dataSource1", inputdataframe); 
  Config config = ConfigFactory.empty()
          .withValue(SelectDeriver.INCLUDE_FIELDS,ConfigValueFactory.fromAnyRef(includeFields))
          .withValue(SelectDeriver.STEP_NAME_CONFIG,ConfigValueFactory.fromAnyRef("dataSource1"));
  SelectDeriver selectderiver = new SelectDeriver();
  assertNoValidationFailures(selectderiver, config);
  selectderiver.configure(config);
  selectderiver.derive(dependencies);  
}
 

private Map<Config, EventHandler> getEventHandlers(Config config, boolean configure) {
  Map<Config, EventHandler> handlers = Maps.newHashMap();
  Set<String> nonConfiguredDefaultHandlerAliases = Sets.newHashSet(
      new LogEventHandler().getAlias()
  );

  if (ConfigUtils.getApplicationConfig(config).hasPath(EVENT_HANDLERS_CONFIG)) {
    List<? extends ConfigObject> handlerConfigObjects;
    try {
      handlerConfigObjects =
          ConfigUtils.getApplicationConfig(config).getObjectList(EVENT_HANDLERS_CONFIG);
    }
    catch (ConfigException.WrongType e) {
      throw new RuntimeException("Event handler configuration must be a list of event handler objects");
    }

    for (ConfigObject handlerConfigObject : handlerConfigObjects) {
      Config handlerConfig = handlerConfigObject.toConfig();
      EventHandler handler = ComponentFactory.create(EventHandler.class, handlerConfig, configure);
      handlers.put(handlerConfig, handler);

      // If this handler is a default handler then because it was configured we remove it from the
      // non-configured set. If this handler is not a default handler then this will be a no-op.
      nonConfiguredDefaultHandlerAliases.remove(handlerConfig.getString(ComponentFactory.TYPE_CONFIG_NAME));
    }
  }

  // Create the default handlers that were not already created above. These are created with
  // no configurations, so default handlers must only use optional configurations.
  for (String defaultHandlerAlias : nonConfiguredDefaultHandlerAliases) {
    Config defaultHandlerConfig = ConfigFactory.empty().withValue(
        ComponentFactory.TYPE_CONFIG_NAME, ConfigValueFactory.fromAnyRef(defaultHandlerAlias));
    EventHandler defaultHandler = ComponentFactory.create(EventHandler.class, defaultHandlerConfig, configure);
    handlers.put(defaultHandlerConfig, defaultHandler);
  }

  return handlers;
}
 

private Config getOrCreateMandatoryProperties(Config properties) {
  if (!properties.hasPath("akka.actor.provider")) {
    properties = properties.withValue("akka.actor.provider",
      ConfigValueFactory.fromAnyRef("akka.remote.RemoteActorRefProvider"));
  }

  if (!properties.hasPath("akka.remote.enabled-transports")) {
    properties = properties.withValue("akka.remote.enabled-transports",
      ConfigValueFactory.fromAnyRef(Collections.singletonList("akka.remote.netty.tcp")));
  }
  return properties;
}