Java Code Examples for org.apache.commons.collections4.MapUtils#getString()

The following examples show how to use org.apache.commons.collections4.MapUtils#getString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LegacyDimensionWrapper.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
/**
 * Instantiates a new Legacy dimension wrapper.
 *
 * @param provider the provider
 * @param config the config
 * @param startTime the start time
 * @param endTime the end time
 * @throws Exception the exception
 */
public LegacyDimensionWrapper(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime) throws Exception {
  super(provider, augmentConfig(config), startTime, endTime);

  this.anomalyFunctionClassName = MapUtils.getString(config.getProperties(), PROP_ANOMALY_FUNCTION_CLASS);
  this.anomalyFunctionSpecs = ConfigUtils.getMap(config.getProperties().get(PROP_SPEC));
  this.anomalyFunction = (BaseAnomalyFunction) Class.forName(this.anomalyFunctionClassName).newInstance();

  String specs = OBJECT_MAPPER.writeValueAsString(this.anomalyFunctionSpecs);
  this.anomalyFunction.init(OBJECT_MAPPER.readValue(specs, AnomalyFunctionDTO.class));
  if (!StringUtils.isBlank(this.anomalyFunction.getSpec().getExploreDimensions())) {
    this.dimensions.add(this.anomalyFunction.getSpec().getExploreDimensions());
  }

  if (this.nestedProperties.isEmpty()) {
    this.nestedProperties.add(Collections.singletonMap(PROP_CLASS_NAME, (Object) LegacyAnomalyFunctionAlgorithm.class.getName()));
  }
}
 
Example 2
Source File: DataQualityPropertiesBuilder.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
private Map<String, Object> buildDataQualityWrapperProperties(String subEntityName, String metricUrn,
    Map<String, Object> yamlConfig, Map<String, Object> mergerProperties) {
  String qualityType = MapUtils.getString(yamlConfig, PROP_TYPE);
  String name = MapUtils.getString(yamlConfig, PROP_NAME);

  Map<String, Object> properties = new HashMap<>();
  properties.put(PROP_CLASS_NAME, DataSlaWrapper.class.getName());
  properties.put(PROP_SUB_ENTITY_NAME, subEntityName);
  properties.put(PROP_METRIC_URN, metricUrn);

  String qualityRefKey = makeComponentRefKey(qualityType, name);
  properties.put(PROP_QUALITY_CHECK, qualityRefKey);

  buildComponentSpec(metricUrn, yamlConfig, qualityType, qualityRefKey);

  properties.putAll(mergerProperties);
  return properties;
}
 
Example 3
Source File: YamlResource.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
private DetectionAlertConfigDTO fetchExistingSubscriptionGroup(@NotNull String payload) {
  DetectionAlertConfigDTO existingSubscriptionConfig = null;

  // Extract the subscription group name from payload
  Map<String, Object> subscriptionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
  String subscriptionGroupName = MapUtils.getString(subscriptionConfigMap, PROP_SUBS_GROUP_NAME);
  Preconditions.checkNotNull(subscriptionGroupName, "Missing property " + PROP_SUBS_GROUP_NAME
      + " in the subscription config.");

  // Check if subscription already exists
  Collection<DetectionAlertConfigDTO> subscriptionConfigs = this.subscriptionConfigDAO
      .findByPredicate(Predicate.EQ("name", subscriptionGroupName));
  if (subscriptionConfigs != null && !subscriptionConfigs.isEmpty()) {
    existingSubscriptionConfig = subscriptionConfigs.iterator().next();
  }

  return existingSubscriptionConfig;
}
 
Example 4
Source File: DetectionConfigTranslator.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
/**
 * Fill in common fields of detection config. Properties of the pipeline is filled by the subclass.
 */
private DetectionConfigDTO generateDetectionConfig(Map<String, Object> yamlConfigMap,
    Map<String, Object> detectionProperties, Map<String, Object> qualityProperties, String cron) {
  DetectionConfigDTO config = new DetectionConfigDTO();
  config.setName(MapUtils.getString(yamlConfigMap, PROP_DETECTION_NAME));
  config.setDescription(MapUtils.getString(yamlConfigMap, PROP_DESC_NAME));
  config.setDescription(MapUtils.getString(yamlConfigMap, PROP_DESC_NAME));
  config.setLastTimestamp(System.currentTimeMillis());
  config.setOwners(filterOwners(ConfigUtils.getList(yamlConfigMap.get(PROP_OWNERS))));

  config.setProperties(detectionProperties);
  config.setDataQualityProperties(qualityProperties);
  config.setComponentSpecs(this.metricAttributesMap.getAllComponenets());
  config.setCron(cron);
  config.setActive(MapUtils.getBooleanValue(yamlConfigMap, PROP_ACTIVE, true));
  config.setYaml(yamlConfig);

  //TODO: data-availability trigger is only enabled for detections running on PINOT daily dataset only
  List<DatasetConfigDTO> datasetConfigs = this.metricAttributesMap.getAllDatasets();
  if (MapUtils.getString(yamlConfigMap, PROP_CRON) == null
      && datasetConfigs.stream().allMatch(c -> c.bucketTimeGranularity().getUnit().equals(TimeUnit.DAYS))
      && datasetConfigs.stream().allMatch(c -> c.getDataSource().equals(PinotThirdEyeDataSource.DATA_SOURCE_NAME))) {
    config.setDataAvailabilitySchedule(true);
  }
  return config;
}
 
Example 5
Source File: DetectionMetricAttributeHolder.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
private String loadMetricCache(Map<String, Object> metricAlertConfigMap) {
  String metricName = MapUtils.getString(metricAlertConfigMap, PROP_METRIC);
  String datasetName = MapUtils.getString(metricAlertConfigMap, PROP_DATASET);
  String cron = MapUtils.getString(metricAlertConfigMap, PROP_CRON);
  String metricAliasKey = ThirdEyeUtils.constructMetricAlias(datasetName, metricName);
  if (metricAttributesMap.containsKey(metricAliasKey)) {
    return metricAliasKey;
  }

  DatasetConfigDTO datasetConfig = fetchDatasetConfigDTO(this.dataProvider, datasetName);
  datasetConfigs.add(datasetConfig);

  MetricConfigDTO metricConfig = this.dataProvider.fetchMetric(metricName, datasetConfig.getDataset());

  cron = cron == null ? buildCron(datasetConfig.bucketTimeGranularity()) : cron;

  metricAttributesMap.put(metricAliasKey, new DetectionMetricProperties(cron, metricConfig, datasetConfig));

  return metricAliasKey;
}
 
Example 6
Source File: TriggerConditionGrouper.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Groups the anomalies based on the parsed operator tree
 */
private List<MergedAnomalyResultDTO> groupAnomaliesByOperator(Map<String, Object> operatorNode, List<MergedAnomalyResultDTO> anomalies) {
  Preconditions.checkNotNull(operatorNode);

  // Base condition - If reached leaf node of operator tree, then return the anomalies corresponding to the entity/metric
  String value = MapUtils.getString(operatorNode, "value");
  if (value != null) {
    return anomalies.stream().filter(anomaly ->
        anomaly.getProperties() != null
        && anomaly.getProperties().containsKey(PROP_SUB_ENTITY_NAME)
        && anomaly.getProperties().get(PROP_SUB_ENTITY_NAME).equals(value)
    ).collect(Collectors.toList());
  }

  String operator = MapUtils.getString(operatorNode, PROP_OPERATOR);
  Preconditions.checkNotNull(operator, "No operator provided!");
  Map<String, Object> leftOp = ConfigUtils.getMap(operatorNode.get(PROP_LEFT_OP));
  Map<String, Object> rightOp = ConfigUtils.getMap(operatorNode.get(PROP_RIGHT_OP));

  // Post-order traversal - find anomalies from left subtree and right sub-tree and then group them
  List<MergedAnomalyResultDTO> leftAnomalies = groupAnomaliesByOperator(leftOp, anomalies);
  List<MergedAnomalyResultDTO> rightAnomalies = groupAnomaliesByOperator(rightOp, anomalies);
  if (operator.equalsIgnoreCase(PROP_AND)) {
    return andGrouping(leftAnomalies, rightAnomalies);
  } else if (operator.equalsIgnoreCase(PROP_OR)) {
    return orGrouping(leftAnomalies, rightAnomalies);
  } else {
    throw new RuntimeException("Unsupported operator");
  }
}
 
Example 7
Source File: DetectionConfigValidator.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Validates the detection or filter rule accordingly based on {@param ruleType}
 */
private void validateRule(String alertName, Map<String, Object> ruleYaml, int ruleIndex, String ruleType, Set<String> ruleNamesTaken) {
  Preconditions.checkArgument(ruleYaml.containsKey(PROP_TYPE),
      "Missing property " + ruleType + " (" + ruleType + ") for sub-alert " + alertName + " rule no. " + ruleIndex);

  Preconditions.checkArgument(ruleYaml.containsKey(PROP_NAME),
      "Missing property " + ruleType + " (" + PROP_NAME + ") for sub-alert " + alertName + " rule no. " + ruleIndex);
  String name = MapUtils.getString(ruleYaml, PROP_NAME);

  Preconditions.checkArgument(!ruleNamesTaken.contains(name),
      "Duplicate rule name (" + name + ") found for sub-alert " + alertName + " rule no. " + ruleIndex + ". Names have to be unique within a config.");

  Preconditions.checkArgument(!name.contains(":"),
      "Illegal character (:) found in " + ruleType + " (" + PROP_NAME + ") for sub-alert " + alertName + " rule no. " + ruleIndex);
}
 
Example 8
Source File: DataQualityPropertiesBuilder.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Constructs the data quality properties mapping by translating the quality yaml
 *
 * @param metricAlertConfigMap holds the parsed yaml for a single metric alert
 */
@Override
public Map<String, Object> buildMetricAlertProperties(Map<String, Object> metricAlertConfigMap) {
  Map<String, Object> properties = new HashMap<>();
  MetricConfigDTO metricConfigDTO = metricAttributesMap.fetchMetric(metricAlertConfigMap);

  String subEntityName = MapUtils.getString(metricAlertConfigMap, PROP_NAME);
  Map<String, Object> mergerProperties = ConfigUtils.getMap(metricAlertConfigMap.get(PROP_MERGER));

  Map<String, Collection<String>> dimensionFiltersMap = ConfigUtils.getMap(metricAlertConfigMap.get(PROP_FILTERS));
  String metricUrn = MetricEntity.fromMetric(dimensionFiltersMap, metricConfigDTO.getId()).getUrn();

  // Translate all the rules
  List<Map<String, Object>> ruleYamls = getList(metricAlertConfigMap.get(PROP_RULES));
  List<Map<String, Object>> nestedPipelines = new ArrayList<>();
  for (Map<String, Object> ruleYaml : ruleYamls) {
    List<Map<String, Object>> qualityYamls = ConfigUtils.getList(ruleYaml.get(PROP_QUALITY));
    if (qualityYamls.isEmpty()) {
      continue;
    }
    List<Map<String, Object>> qualityProperties = buildListOfDataQualityProperties(
        subEntityName, metricUrn, qualityYamls, mergerProperties);
    nestedPipelines.addAll(qualityProperties);
  }
  if (nestedPipelines.isEmpty()) {
    // No data quality rules
    return properties;
  }

  properties.putAll(buildWrapperProperties(DataQualityMergeWrapper.class.getName(), nestedPipelines, mergerProperties));
  return properties;
}
 
Example 9
Source File: ThirdEyeEventsPipeline.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Alternate constructor for RCAFrameworkLoader
 *
 * @param outputName pipeline output name
 * @param inputNames input pipeline names
 * @param properties configuration properties ({@code PROP_STRATEGY})
 */
public ThirdEyeEventsPipeline(String outputName, Set<String> inputNames, Map<String, Object> properties) {
  super(outputName, inputNames);
  this.eventDAO = DAORegistry.getInstance().getEventDAO();
  this.strategy = StrategyType.valueOf(MapUtils.getString(properties, PROP_STRATEGY, PROP_STRATEGY_DEFAULT));
  this.eventType = MapUtils.getString(properties, PROP_EVENT_TYPE, "holiday");
  this.k = MapUtils.getInteger(properties, PROP_K, PROP_K_DEFAULT);
}
 
Example 10
Source File: LegacyMergeWrapper.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Instantiates a new Legacy merge wrapper.
 *
 * @param provider the provider
 * @param config the config
 * @param startTime the start time
 * @param endTime the end time
 * @throws Exception the exception
 */
public LegacyMergeWrapper(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime) throws Exception {
  super(provider, config, startTime, endTime);

  this.anomalyFunctionClassName = MapUtils.getString(config.getProperties(), PROP_ANOMALY_FUNCTION_CLASS);
  this.anomalyFunctionSpecs = ConfigUtils.getMap(config.getProperties().get(PROP_SPEC));
  this.anomalyFunction = (BaseAnomalyFunction) Class.forName(this.anomalyFunctionClassName).newInstance();

  String specs = OBJECT_MAPPER.writeValueAsString(this.anomalyFunctionSpecs);
  this.anomalyFunction.init(OBJECT_MAPPER.readValue(specs, AnomalyFunctionDTO.class));

  AnomalyMergeConfig mergeConfig = this.anomalyFunction.getSpec().getAnomalyMergeConfig();
  if (mergeConfig == null) {
    mergeConfig = DEFAULT_TIME_BASED_MERGE_CONFIG;
  }

  this.mergeConfig = mergeConfig;
  this.maxGap = mergeConfig.getSequentialAllowedGap();
  this.slice = new AnomalySlice()
      .withStart(startTime)
      .withEnd(endTime);

  if (config.getProperties().containsKey(PROP_NESTED)) {
    this.nestedProperties = ConfigUtils.getList(config.getProperties().get(PROP_NESTED));
  } else {
    this.nestedProperties = new ArrayList<>(Collections.singletonList(Collections.singletonMap(PROP_CLASS_NAME, (Object) LegacyDimensionWrapper.class.getName())));
  }
}
 
Example 11
Source File: DetectionConfigTranslator.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
@Override
DetectionConfigDTO translateConfig(Map<String, Object> yamlConfigMap) throws IllegalArgumentException {
  // Hack to support 'detectionName' attribute at root level and 'name' attribute elsewhere
  // We consistently use 'name' as a convention to define the sub-alerts. However, at the root
  // level, as a convention, we will use 'detectionName' which defines the name of the complete alert.
  String alertName = MapUtils.getString(yamlConfigMap, PROP_DETECTION_NAME);
  yamlConfigMap.put(PROP_NAME, alertName);

  // By default if 'type' is not specified, we assume it as a METRIC_ALERT
  yamlConfigMap.putIfAbsent(PROP_TYPE, METRIC_ALERT);

  // Translate config depending on the type (METRIC_ALERT OR COMPOSITE_ALERT)
  Map<String, Object> detectionProperties;
  Map<String, Object> qualityProperties;
  String cron;
  if (yamlConfigMap.get(PROP_TYPE).equals(COMPOSITE_ALERT)) {
    detectionProperties = detectionTranslatorBuilder.buildCompositeAlertProperties(yamlConfigMap);
    qualityProperties = dataQualityTranslatorBuilder.buildCompositeAlertProperties(yamlConfigMap);

    // TODO: discuss strategy for default cron
    Preconditions.checkArgument(yamlConfigMap.containsKey(PROP_CRON), "Missing property (" + PROP_CRON + ") in alert");
    cron = MapUtils.getString(yamlConfigMap, PROP_CRON);
  } else {
    // The legacy type 'COMPOSITE' will be treated as a metric alert along with the new convention METRIC_ALERT.
    // This is applicable only at the root level to maintain backward compatibility.
    detectionProperties = detectionTranslatorBuilder.buildMetricAlertProperties(yamlConfigMap);
    qualityProperties = dataQualityTranslatorBuilder.buildMetricAlertProperties(yamlConfigMap);
    cron = metricAttributesMap.fetchCron(yamlConfigMap);
  }

  return generateDetectionConfig(yamlConfigMap, detectionProperties, qualityProperties, cron);
}
 
Example 12
Source File: PinotThirdEyeDataSourceConfig.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
/**
 * Returns PinotThirdEyeDataSourceConfig from the given property map.
 *
 * @param properties the properties to setup a PinotThirdEyeDataSourceConfig.
 *
 * @return a PinotThirdEyeDataSourceConfig.
 *
 * @throws IllegalArgumentException is thrown if the property map does not contain all necessary fields, i.e.,
 *                                  controller host and port, cluster name, and the URL to zoo keeper.
 */
static PinotThirdEyeDataSourceConfig createFromProperties(Map<String, Object> properties) {
  ImmutableMap<String, Object> processedProperties = processPropertyMap(properties);
  if (processedProperties == null) {
    throw new IllegalArgumentException(
        "Invalid properties for data source: " + PinotThirdEyeDataSource.DATA_SOURCE_NAME + ", properties="
            + properties);
  }

  String controllerHost = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.CONTROLLER_HOST.getValue());
  int controllerPort = MapUtils.getInteger(processedProperties, PinotThirdeyeDataSourceProperties.CONTROLLER_PORT.getValue());
  String controllerConnectionScheme = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.CONTROLLER_CONNECTION_SCHEME.getValue());
  String zookeeperUrl = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.ZOOKEEPER_URL.getValue());
  String clusterName = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.CLUSTER_NAME.getValue());

  // brokerUrl and tag are optional
  String brokerUrl = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.BROKER_URL.getValue());
  String tag = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.TAG.getValue());

  Builder builder =
      PinotThirdEyeDataSourceConfig.builder().setControllerHost(controllerHost).setControllerPort(controllerPort)
          .setZookeeperUrl(zookeeperUrl).setClusterName(clusterName);
  if (StringUtils.isNotBlank(brokerUrl)) {
    builder.setBrokerUrl(brokerUrl);
  }
  if (StringUtils.isNotBlank(tag)) {
    builder.setTag(tag);
  }
  if (StringUtils.isNotBlank(controllerConnectionScheme)) {
    builder.setControllerConnectionScheme(controllerConnectionScheme);
  }

  return builder.build();
}
 
Example 13
Source File: MapUtilsUnitTest.java    From tutorials with MIT License 5 votes vote down vote up
@Test
public void whenGetOnNullMap_thenMustReturnDefaultValue() {
    String defaultColorStr = "COLOR_NOT_FOUND";
    String color = MapUtils.getString(null, "RED", defaultColorStr);

    assertEquals(color, defaultColorStr);
}
 
Example 14
Source File: BaselineRuleFilterWrapper.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
public BaselineRuleFilterWrapper(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime) {
  super(provider, config, startTime, endTime);
  int weeks = MapUtils.getIntValue(config.getProperties(), PROP_WEEKS, PROP_WEEKS_DEFAULT);
  DateTimeZone timezone =
      DateTimeZone.forID(MapUtils.getString(this.config.getProperties(), PROP_TIMEZONE, PROP_TIMEZONE_DEFAULT));
  this.baseline = BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEDIAN, weeks, 1, timezone);
  // percentage change
  this.change = MapUtils.getDoubleValue(config.getProperties(), PROP_CHANGE, PROP_CHANGE_DEFAULT);
  // absolute change
  this.difference = MapUtils.getDoubleValue(config.getProperties(), PROP_DIFFERENCE, PROP_DIFFERENCE_DEFAULT);
  // site wide impact
  this.siteWideImpactThreshold = MapUtils.getDoubleValue(config.getProperties(), PROP_SITEWIDE_THRESHOLD, PROP_SITEWIDE_THRESHOLD_DEFAULT);
  this.siteWideMetricUrn = MapUtils.getString(config.getProperties(), PROP_SITEWIDE_METRIC);
}
 
Example 15
Source File: WeiXinOAuth2Template.java    From FEBS-Security with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private AccessGrant getAccessToken(StringBuilder accessTokenRequestUrl) {

    logger.info("获取access_token, 请求URL{} ", accessTokenRequestUrl.toString());
    String response = getRestTemplate().getForObject(accessTokenRequestUrl.toString(), String.class);
    logger.info("获取access_token, 响应内容{} ", response);

    Map<String, Object> result = null;
    try {
        result = new ObjectMapper().readValue(response, Map.class);
    } catch (Exception e) {
        logger.error("获取微信AccessToken失败", e);
    }

    if (StringUtils.isNotBlank(MapUtils.getString(result, "errcode"))) {
        String errcode = MapUtils.getString(result, "errcode");
        String errmsg = MapUtils.getString(result, "errmsg");
        throw new FebsCredentialExcetion("获取access token失败, errcode:" + errcode + ", errmsg:" + errmsg);
    }

    WeiXinAccessGrant accessToken = new WeiXinAccessGrant(
            MapUtils.getString(result, "access_token"),
            MapUtils.getString(result, "scope"),
            MapUtils.getString(result, "refresh_token"),
            MapUtils.getLong(result, "expires_in"));

    accessToken.setOpenId(MapUtils.getString(result, "openid"));

    return accessToken;
}
 
Example 16
Source File: ThresholdAlgorithm.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
public ThresholdAlgorithm(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime) {
  super(provider, config, startTime, endTime);
  this.min = MapUtils.getDoubleValue(config.getProperties(), "min", Double.NaN);
  this.max = MapUtils.getDoubleValue(config.getProperties(), "max", Double.NaN);

  String metricUrn = MapUtils.getString(config.getProperties(), "metricUrn");
  MetricEntity me = MetricEntity.fromURN(metricUrn);
  this.slice = MetricSlice.from(me.getId(), this.startTime, this.endTime, me.getFilters());
}
 
Example 17
Source File: CacheUtils.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
public static String getBucketName() {
  Map<String, Object> config = CacheConfig.getInstance().getCentralizedCacheSettings().getDataSourceConfig().getConfig();
  return MapUtils.getString(config, BUCKET_NAME);
}
 
Example 18
Source File: AnomalyDetectorWrapper.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
public AnomalyDetectorWrapper(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime) {
  super(provider, config, startTime, endTime);

  Preconditions.checkArgument(this.config.getProperties().containsKey(PROP_SUB_ENTITY_NAME));
  this.entityName = MapUtils.getString(config.getProperties(), PROP_SUB_ENTITY_NAME);

  this.metricUrn = MapUtils.getString(config.getProperties(), PROP_METRIC_URN);
  this.metricEntity = MetricEntity.fromURN(this.metricUrn);
  this.metric = provider.fetchMetrics(Collections.singleton(this.metricEntity.getId())).get(this.metricEntity.getId());

  Preconditions.checkArgument(this.config.getProperties().containsKey(PROP_DETECTOR));
  this.detectorName = DetectionUtils.getComponentKey(MapUtils.getString(config.getProperties(), PROP_DETECTOR));
  Preconditions.checkArgument(this.config.getComponents().containsKey(this.detectorName));
  this.anomalyDetector = (AnomalyDetector) this.config.getComponents().get(this.detectorName);

  // emulate moving window or now
  this.isMovingWindowDetection = MapUtils.getBooleanValue(config.getProperties(), PROP_MOVING_WINDOW_DETECTION, false);
  // delays to wait for data becomes available
  this.windowDelay = MapUtils.getIntValue(config.getProperties(), PROP_WINDOW_DELAY, 0);
  // window delay unit
  this.windowDelayUnit = TimeUnit.valueOf(MapUtils.getString(config.getProperties(), PROP_WINDOW_DELAY_UNIT, "DAYS"));
  // detection window size
  this.windowSize = MapUtils.getIntValue(config.getProperties(), PROP_WINDOW_SIZE, 1);
  // detection window unit
  this.windowUnit = TimeUnit.valueOf(MapUtils.getString(config.getProperties(), PROP_WINDOW_UNIT, "DAYS"));
  // run frequency, used to determine moving windows for minute-level detection
  Map<String, Object> frequency = (Map<String, Object>) MapUtils.getMap(config.getProperties(), PROP_FREQUENCY);
  this.functionFrequency = new TimeGranularity(MapUtils.getIntValue(frequency, "size", 15), TimeUnit.valueOf(MapUtils.getString(frequency, "unit", "MINUTES")));

  MetricConfigDTO metricConfigDTO = this.provider.fetchMetrics(Collections.singletonList(this.metricEntity.getId())).get(this.metricEntity.getId());
  this.dataset = this.provider.fetchDatasets(Collections.singletonList(metricConfigDTO.getDataset()))
      .get(metricConfigDTO.getDataset());
  // date time zone for moving windows. use dataset time zone as default
  this.dateTimeZone = DateTimeZone.forID(MapUtils.getString(config.getProperties(), PROP_TIMEZONE, this.dataset.getTimezone()));

  String bucketStr = MapUtils.getString(config.getProperties(), PROP_BUCKET_PERIOD);
  this.bucketPeriod = bucketStr == null ? this.getBucketSizePeriodForDataset() : Period.parse(bucketStr);
  this.cachingPeriodLookback = config.getProperties().containsKey(PROP_CACHE_PERIOD_LOOKBACK) ?
      MapUtils.getLong(config.getProperties(), PROP_CACHE_PERIOD_LOOKBACK) : ThirdEyeUtils.getCachingPeriodLookback(this.dataset.bucketTimeGranularity());

  speedUpMinuteLevelDetection();
}
 
Example 19
Source File: DetectionConfigValidator.java    From incubator-pinot with Apache License 2.0 4 votes vote down vote up
private void validateMetricAlertConfig(Map<String, Object> detectionYaml, String parentAlertName)
    throws IllegalArgumentException {
  validateBasicAttributes(detectionYaml, parentAlertName);
  String alertName = MapUtils.getString(detectionYaml, PROP_NAME);

  // Validate all compulsory fields
  String metric = MapUtils.getString(detectionYaml, PROP_METRIC);
  Preconditions.checkArgument(StringUtils.isNotEmpty(metric),
      "Missing property (" + PROP_METRIC + ") in sub-alert " + alertName);
  String dataset = MapUtils.getString(detectionYaml, PROP_DATASET);
  Preconditions.checkArgument(StringUtils.isNotEmpty(dataset),
      "Missing property (" + PROP_DATASET + ") in sub-alert " + alertName);
  Preconditions.checkArgument(detectionYaml.containsKey(PROP_RULES),
      "Missing property (" + PROP_RULES + ") in sub-alert " + alertName);

  // Validate fields which shouldn't be defined at this level
  Preconditions.checkArgument(!detectionYaml.containsKey(PROP_FILTER),
      "For sub-alert " + alertName + ", please double check the filter config. Adding dimensions filters"
          + " should be in the yaml root level using 'filters' as the key. Anomaly filter should be added in to the"
          + " indentation level of detection yaml it applies to.");
  // Check if the dataset defined in the config exists
  DatasetConfigDTO datasetConfig = fetchDatasetConfigDTO(this.provider, dataset);

  // Check if the metric defined in the config exists
  MetricConfigDTO metricConfig = provider.fetchMetric(metric, datasetConfig.getDataset());
  Preconditions.checkArgument(metricConfig != null,
      "Metric doesn't exist in our records. Metric " + metric + " Dataset " + dataset + " in sub-alert " + alertName);

  // We support only one grouper per metric
  Preconditions.checkArgument(ConfigUtils.getList(detectionYaml.get(PROP_GROUPER)).size() <= 1,
      "Multiple groupers detected for metric in sub-alert " + alertName);

  // Validate all the rules
  Set<String> names = new HashSet<>();
  List<Map<String, Object>> ruleYamls = ConfigUtils.getList(detectionYaml.get(PROP_RULES));
  for (int ruleIndex = 1; ruleIndex <= ruleYamls.size(); ruleIndex++) {
    Map<String, Object> ruleYaml = ruleYamls.get(ruleIndex - 1);

    // Validate detection rules
    Preconditions.checkArgument(ruleYaml.containsKey(PROP_DETECTION),
        "Detection rule missing for sub-alert " + alertName + " rule no. " + ruleIndex);
    // Validate detection rules
    List<Map<String, Object>> detectionRuleYamls = ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
    for (Map<String, Object> detectionRuleYaml : detectionRuleYamls) {
      validateRule(alertName, detectionRuleYaml, ruleIndex, "detection", names);
      names.add(MapUtils.getString(detectionRuleYaml, PROP_NAME));
    }

    // Validate filter rules
    if (ruleYaml.containsKey(PROP_FILTER)) {
      List<Map<String, Object>> filterRuleYamls = ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
      for (Map<String, Object> filterRuleYaml : filterRuleYamls) {
        validateRule(alertName, filterRuleYaml, ruleIndex, "filter", names);
        names.add(MapUtils.getString(filterRuleYaml, PROP_NAME));
      }
    }
  }

  // Safety condition: Validate if maxDuration is greater than 15 minutes
  Map<String, Object> mergerProperties = ConfigUtils.getMap(detectionYaml.get(PROP_MERGER));
  if (mergerProperties.get(PROP_MAX_DURATION) != null) {
    Preconditions.checkArgument(
        MapUtils.getLong(mergerProperties, PROP_MAX_DURATION) >= datasetConfig.bucketTimeGranularity().toMillis(),
        "The maxDuration field set is not acceptable. Please check the the document and set it correctly.");
  }
}
 
Example 20
Source File: DetectionRegistry.java    From incubator-pinot with Apache License 2.0 2 votes vote down vote up
/**
 * Look up the class name for a given component
 * @param type the type used in the YAML configs
 * @return component class name
 */
public String lookup(String type) {
  Preconditions.checkArgument(REGISTRY_MAP.containsKey(type), type + " not found in registry");
  return MapUtils.getString(REGISTRY_MAP.get(type), KEY_CLASS_NAME);
}