org.kitesdk.morphline.api.CommandBuilder Java Examples

The following examples show how to use org.kitesdk.morphline.api.CommandBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConvertHTMLBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public ConvertHTML(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws SAXNotRecognizedException, SAXNotSupportedException {
  super(builder, config, parent, child, context);
  this.charset = getConfigs().getCharset(config, "charset", null);
  this.omitXMLDeclaration = getConfigs().getBoolean(config, "omitXMLDeclaration", false);      
  this.xmlReader = new Parser(); // no reuse?
  xmlReader.setProperty(Parser.schemaProperty, htmlSchema);
  xmlReader.setFeature(Parser.CDATAElementsFeature, getConfigs().getBoolean(config, "noCDATA", false));
  xmlReader.setFeature(Parser.namespacesFeature, !getConfigs().getBoolean(config, "noNamespaces", true));
  xmlReader.setFeature(Parser.ignoreBogonsFeature, getConfigs().getBoolean(config, "noBogons", false)); // also see TIKA-599
  xmlReader.setFeature(Parser.bogonsEmptyFeature, getConfigs().getBoolean(config, "emptyBogons", false));
  xmlReader.setFeature(Parser.rootBogonsFeature, getConfigs().getBoolean(config, "noRootBogons", false));
  xmlReader.setFeature(Parser.defaultAttributesFeature, getConfigs().getBoolean(config, "noDefaultAttributes", false));
  xmlReader.setFeature(Parser.translateColonsFeature, getConfigs().getBoolean(config, "noColons", false));
  xmlReader.setFeature(Parser.restartElementsFeature, getConfigs().getBoolean(config, "noRestart", false));
  xmlReader.setFeature(Parser.ignorableWhitespaceFeature, !getConfigs().getBoolean(config, "suppressIgnorableWhitespace", true));
  validateArguments();
}
 
Example #2
Source File: FindReplaceBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public FindReplace(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  GrokDictionaries dict = new GrokDictionaries(config, getConfigs());
  String replacementStr = getConfigs().getString(config, "replacement");
  String pattern = getConfigs().getString(config, "pattern");
  if (getConfigs().getBoolean(config, "isRegex", false)) {
    Pattern regex = dict.compileExpression(pattern);
    this.matcher = regex.pattern().matcher("");
    replacementStr = regex.replaceProperties(replacementStr);
    this.literalPattern = null;
  } else {
    this.matcher = null;
    this.literalPattern = pattern;
  }
  this.replacement = replacementStr;
  this.replaceFirst = getConfigs().getBoolean(config, "replaceFirst", false);
  validateArguments();
}
 
Example #3
Source File: TokenizeTextBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");      
  String solrFieldType = getConfigs().getString(config, "solrFieldType");      
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
  if (fieldType == null) {
    throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
  }
  this.analyzer = fieldType.getIndexAnalyzer();
  Preconditions.checkNotNull(analyzer);
  // register CharTermAttribute for later (implicit) reuse
  this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class);
  Preconditions.checkNotNull(token);
  validateArguments();
}
 
Example #4
Source File: LoadSolrBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM);
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  RetryPolicyFactory retryPolicyFactory = parseRetryPolicyFactory(
      getConfigs().getConfig(config, "retryPolicy", null));
  this.loader = locator.getLoader(retryPolicyFactory, new CodahaleMetricsFacade(context.getMetricRegistry()));

  Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty());
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) {
    String fieldName = entry.getKey();        
    float boost = Float.parseFloat(entry.getValue().toString().trim());
    boosts.put(fieldName, boost);
  }
  this.rateLimiter = RateLimiter.create(getConfigs().getDouble(config, "maxRecordsPerSecond", Double.MAX_VALUE));
  this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false);
  validateArguments();
  this.elapsedTime = getTimer(Metrics.ELAPSED_TIME);
}
 
Example #5
Source File: SplitKeyValueBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public SplitKeyValue(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");      
  this.outputFieldPrefix = getConfigs().getString(config, "outputFieldPrefix", "");
  this.separator = getConfigs().getString(config, "separator", "=");
  if (separator.length() == 0) {
    throw new MorphlineCompilationException("separator must not be the empty string", config);
  }
  if (getConfigs().getBoolean(config, "isRegex", false)) {
    GrokDictionaries dict = new GrokDictionaries(config, getConfigs());
    this.regex = dict.compileExpression(separator).pattern().matcher("");
  } else {
    this.regex = null;
  }
  this.addEmptyStrings = getConfigs().getBoolean(config, "addEmptyStrings", false);      
  this.trim = getConfigs().getBoolean(config, "trim", true);
  validateArguments();
}
 
Example #6
Source File: TryRulesBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public TryRules(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.throwExceptionIfAllRulesFailed = getConfigs().getBoolean(config, "throwExceptionIfAllRulesFailed", true);
  this.catchExceptions = getConfigs().getBoolean(config, "catchExceptions", false);
  this.copyRecords = getConfigs().getBoolean(config, "copyRecords", true);
  
  List<? extends Config> ruleConfigs = getConfigs().getConfigList(config, "rules", Collections.EMPTY_LIST);
  for (Config ruleConfig : ruleConfigs) {
    List<Command> commands = buildCommandChain(ruleConfig, "commands", child, true);
    if (commands.size() > 0) {
      childRules.add(commands.get(0));
    }
  }
  validateArguments();
  numExceptionsCaught = getMeter("numExceptionsCaught");
}
 
Example #7
Source File: GenerateUUIDBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public GenerateUUID(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  this.fieldName = getConfigs().getString(config, FIELD_NAME, Fields.ID);
  this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true);
  this.prefix = getConfigs().getString(config, PREFIX_NAME, "");
  Type type = new Validator<Type>().validateEnum(
      config,
      getConfigs().getString(config, "type", Type.secure.toString()),
      Type.class);
  if (type == Type.secure) {
    prng = null; // secure & slow
  } else {
    Random rand = new SecureRandom();
    int[] seed = new int[624];
    for (int i = 0; i < seed.length; i++) {
      seed[i] = rand.nextInt();
    }
    prng = new Well19937c(seed); // non-secure & fast
  }
  validateArguments();
}
 
Example #8
Source File: AddLocalHostBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public AddLocalHost(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  this.fieldName = getConfigs().getString(config, FIELD_NAME, "host");
  this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true);
  boolean useIP = getConfigs().getBoolean(config, USE_IP, true);      
  validateArguments();
  
  InetAddress addr = null;
  try {
    addr = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    LOG.warn("Cannot get address of local host", e);
  }
  
  if (addr == null) {
    host = null;
  } else if (useIP) {
    host = addr.getHostAddress();
  } else {
    host = addr.getCanonicalHostName();
  }
}
 
Example #9
Source File: ReadAvroBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public ReadAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  String schemaString = getConfigs().getString(config, "writerSchemaString", null);
  if (schemaString != null) {
    this.writerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "writerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.writerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro writer schema file: " + schemaFile, config, e);
      }
    } else {
      this.writerSchema = null;
    }
  }
  
  this.isJson = getConfigs().getBoolean(config, "isJson", false);
  validateArguments();      
}
 
Example #10
Source File: AbstractCommand.java    From kite with Apache License 2.0 6 votes vote down vote up
/**
 * Using the given <code>builder</code>, constructs a command rooted at the given morphline JSON
 * <code>config</code>.
 * 
 * The command will feed records into <code>child</code>. The command will have
 * <code>parent</code> as it's parent. Additional parameters can be passed via the morphline
 * <code>context</code>.
 */
protected AbstractCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  Preconditions.checkNotNull(builder);
  Preconditions.checkNotNull(config);
  Preconditions.checkNotNull(parent);
  Preconditions.checkNotNull(child);
  Preconditions.checkNotNull(context);
  this.config = config;
  this.parent = parent;
  this.child = child;
  this.context = context;
  Preconditions.checkArgument(builder.getNames().size() > 0);
  this.name = "morphline." + builder.getNames().iterator().next();
  this.configs = new Configs();
  this.numProcessCallsMeter = getMeter(Metrics.NUM_PROCESS_CALLS);
  this.numNotifyCallsMeter = getMeter(Metrics.NUM_NOTIFY_CALLS);
}
 
Example #11
Source File: Pipe.java    From kite with Apache License 2.0 6 votes vote down vote up
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.id = getConfigs().getString(config, "id");

  List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", 
      Arrays.asList("com.**", "org.**", "net.**"));    
  context.importCommandBuilders(importCommandSpecs);

  getConfigs().getConfigList(config, "commands", null);
  List<Command> childCommands = buildCommandChain(config, "commands", child, false);
  if (childCommands.size() > 0) {
    this.realChild = childCommands.get(0);
  } else {
    this.realChild = child;
  }
  validateArguments();
}
 
Example #12
Source File: DownloadHdfsFileBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public DownloadHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) 
    throws IOException {
  
  super(builder, config, parent, child, context);
  List<String> uris = getConfigs().getStringList(config, "inputFiles", Collections.<String>emptyList()); 
  File dstRootDir = new File(getConfigs().getString(config, "outputDir", "."));
  Configuration conf = new Configuration();
  String defaultFileSystemUri = getConfigs().getString(config, "fs", null);
  if (defaultFileSystemUri != null) {
    FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser
  }
  for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) {
    conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser
  }
  validateArguments();
  download(uris, conf, dstRootDir);
}
 
Example #13
Source File: IfThenElseBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public IfThenElse(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  Command devNull = new DropRecordBuilder().build(null, this, null, context); // pipes into /dev/null
  List<Command> conditions = buildCommandChain(config, "conditions", devNull, true);
  if (conditions.size() == 0) {
    throw new MorphlineCompilationException("Missing conditions", config);
  } else {
    this.conditionChain = conditions.get(0);
  }

  List<Command> thenCommands = buildCommandChain(config, "then", child, true);
  if (thenCommands.size() > 0) {
    this.thenChain = thenCommands.get(0);
  }
  
  List<Command> elseCommands = buildCommandChain(config, "else", child, true);
  if (elseCommands.size() > 0) {
    this.elseChain = elseCommands.get(0);
  }
  validateArguments();
}
 
Example #14
Source File: WriteAvroToByteArrayBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public WriteAvroToByteArray(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.format = new Validator<Format>().validateEnum(
      config,
      getConfigs().getString(config, "format", Format.container.toString()),
      Format.class);
  
  String codec = getConfigs().getString(config, "codec", null);
  if (codec == null) {
    this.codecFactory = null;
  } else {
    this.codecFactory = CodecFactory.fromString(codec);
  }
  
  Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty());
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) {
    this.metadata.put(entry.getKey(), entry.getValue().toString());
  }
  
  validateArguments();
}
 
Example #15
Source File: JavaBuilder.java    From kite with Apache License 2.0 6 votes vote down vote up
public Java(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws ScriptException {
  super(builder, config, parent, child, context);
  
  String javaImports = getConfigs().getString(config, "imports", DEFAULT_IMPORTS);
  String javaCodeBlock = getConfigs().getString(config, "code");
  validateArguments();
  this.script = new ScriptEvaluator<Boolean>(
      javaImports, 
      javaCodeBlock, 
      Boolean.class,
      new String[] {"record", "config", "parent", "child", "context", "logger"}, 
      new Class[] {Record.class, Config.class, Command.class, Command.class, MorphlineContext.class, Logger.class}, 
      new Class[] {Exception.class},
      javaCodeBlock
      );
}
 
Example #16
Source File: StartReportingMetricsToSLF4JBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public StartReportingMetricsToSLF4J(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  
  MetricFilter filter = PatternMetricFilter.parse(getConfigs(), config);
  TimeUnit defaultDurationUnit = getConfigs().getTimeUnit(config, "defaultDurationUnit", TimeUnit.MILLISECONDS);
  TimeUnit defaultRateUnit = getConfigs().getTimeUnit(config, "defaultRateUnit", TimeUnit.SECONDS); 
  long frequency = getConfigs().getNanoseconds(config, "frequency", 10 * 1000L * 1000 * 1000); // 10 secs, also see https://github.com/typesafehub/config/blob/master/HOCON.md#duration-format
  this.logger = getConfigs().getString(config, "logger", "metrics");
  String marker = getConfigs().getString(config, "marker", null);      
  validateArguments();
  
  MetricRegistry registry = context.getMetricRegistry();
  synchronized (REGISTRIES) {
    Map<String, Slf4jReporter> reporters = REGISTRIES.get(registry);
    if (reporters == null) {
      reporters = Maps.newHashMap();
      REGISTRIES.put(registry, reporters);
    }
    Slf4jReporter reporter = reporters.get(logger);
    if (reporter == null) {
      Builder reporterBuilder = Slf4jReporter.forRegistry(registry)
          .filter(filter)
          .convertDurationsTo(defaultDurationUnit)
          .convertRatesTo(defaultRateUnit)
          .outputTo(LoggerFactory.getLogger(logger));
      
      if (marker != null) {
        reporterBuilder = reporterBuilder.markWith(new BasicMarkerFactory().getMarker(marker));
      }
          
      reporter = reporterBuilder.build();
      reporter.start(frequency, TimeUnit.NANOSECONDS);
      reporters.put(logger, reporter);
    }
  }
}
 
Example #17
Source File: ReadJsonTestTweetsBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public ReadJsonTestTweets(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  
  this.isLengthDelimited = getConfigs().getBoolean(config, "isLengthDelimited", true);
  this.idPrefix = getConfigs().getString(config, "idPrefix", null);
  if ("random".equals(idPrefix)) {
    idPrefix = String.valueOf(new Random().nextInt());
  } else if (idPrefix == null) {
    idPrefix = "";
  }
  validateArguments();
}
 
Example #18
Source File: GeoIPBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public GeoIP(CommandBuilder builder, Config config, Command parent, 
                                   Command child, final MorphlineContext context) {
  
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.databaseFile = new File(getConfigs().getString(config, "database", "GeoLite2-City.mmdb"));
  try {
    this.databaseReader = new Reader(databaseFile);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot read Maxmind database: " + databaseFile, config, e);
  }
  validateArguments();
}
 
Example #19
Source File: ExtractAvroPathsBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public ExtractAvroPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create();
  this.flatten = getConfigs().getBoolean(config, "flatten", true);
  Config paths = getConfigs().getConfig(config, "paths");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) {
    String fieldName = entry.getKey();        
    String path = entry.getValue().toString().trim();
    if (path.contains("//")) {
      throw new MorphlineCompilationException("No support for descendant axis available yet", config);
    }
    if (path.startsWith("/")) {
      path = path.substring(1);
    }
    if (path.endsWith("/")) {
      path = path.substring(0, path.length() - 1);
    }
    path = path.trim();
    for (String step : path.split("/")) {
      step = step.trim();
      if (step.length() > ARRAY_TOKEN.length() && step.endsWith(ARRAY_TOKEN)) {
        step = step.substring(0,  step.length() - ARRAY_TOKEN.length());
        stepMultiMap.put(fieldName, normalize(step));
        stepMultiMap.put(fieldName, ARRAY_TOKEN);
      } else {
        stepMultiMap.put(fieldName, normalize(step));
      }
    }
  }
  this.stepMap = stepMultiMap.asMap();
  LOG.debug("stepMap: {}", stepMap);
  validateArguments();
}
 
Example #20
Source File: ReadAvroContainerBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {   
  super(builder, config, parent, child, context);

  String schemaString = getConfigs().getString(config, "readerSchemaString", null);
  if (schemaString != null) {
    this.readerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "readerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.readerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e);
      }
    } else {
      this.readerSchema = null;
    }
  }
  
  if (getClass() == ReadAvroContainer.class) {
    resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>(
        getConfigs().getInt(config, "schemaCacheCapacity", 100));
    
    validateArguments();
  } else {
    resolverCache = null;
  }
}
 
Example #21
Source File: ToAvroMapBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public ToAvroMap(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  InputStream in = null;
  try {
    in = getClass().getResourceAsStream("morphlineRecord.avsc");
    this.schema = new Schema.Parser().parse(in);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot parse morphlineRecord schema", config, e, builder);
  } finally {
    Closeables.closeQuietly(in);
  }
  
  validateArguments();
}
 
Example #22
Source File: OpenHdfsFileBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public OpenHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.conf = new Configuration();
  String defaultFileSystemUri = getConfigs().getString(config, "fs", null);
  if (defaultFileSystemUri != null) {
    FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser
  }
  for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) {
    conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser
  }
  validateArguments();
}
 
Example #23
Source File: SaxonCommand.java    From kite with Apache License 2.0 5 votes vote down vote up
public SaxonCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    
    this.isTracing = getConfigs().getBoolean(config, "isTracing", false);
    boolean isLicensedSaxonEdition = getConfigs().getBoolean(config, "isLicensedSaxonEdition", false);
    this.processor = new Processor(isLicensedSaxonEdition);
    this.documentBuilder = processor.newDocumentBuilder();
    
    Config features = getConfigs().getConfig(config, "features", ConfigFactory.empty());
    for (Map.Entry<String, Object> entry : new Configs().getEntrySet(features)) {
      processor.setConfigurationProperty(entry.getKey(), entry.getValue());
    }
    
    for (String clazz : getConfigs().getStringList(config, "extensionFunctions", Collections.<String>emptyList())) {
      Object function;
      try {
        function = Class.forName(clazz).newInstance();
      } catch (Exception e) {
        throw new MorphlineCompilationException("Cannot instantiate extension function: " + clazz, config);
      }
      
      if (function instanceof ExtensionFunction) {
        processor.registerExtensionFunction((ExtensionFunction) function);              
//      }
//      else if (function instanceof ExtensionFunctionDefinition) {
//        processor.registerExtensionFunction((ExtensionFunctionDefinition) function);              
      } else {
        throw new MorphlineCompilationException("Extension function has wrong class: " + clazz, config);
      }
    }
  }
 
Example #24
Source File: RegisterJVMMetricsBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public RegisterJVMMetrics(CommandBuilder builder, Config config, Command parent, 
                                   Command child, final MorphlineContext context) {
  
  super(builder, config, parent, child, context);      
  validateArguments();
  
  MetricRegistry registry = context.getMetricRegistry();
  BufferPoolMetricSet bufferPoolMetrics = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer());
  registerAll("jvm.buffers", bufferPoolMetrics, registry);
  registerAll("jvm.gc", new GarbageCollectorMetricSet(), registry);
  registerAll("jvm.memory", new MemoryUsageGaugeSet(), registry);
  registerAll("jvm.threads", new ThreadStatesGaugeSet(), registry);
  register("jvm.fileDescriptorCountRatio", new FileDescriptorRatioGauge(), registry);
  context.getHealthCheckRegistry().register("deadlocks", new ThreadDeadlockHealthCheck());
}
 
Example #25
Source File: ReplaceValues.java    From kite with Apache License 2.0 5 votes vote down vote up
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context,
    boolean isRemoving) {
  super(builder, config, parent, child, context);
  List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*"));
  List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList());
  List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*"));
  List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList());
  int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000);
  this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity);
  int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0);
  this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity);
  this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement");
  validateArguments();
}
 
Example #26
Source File: ToLowerCaseBuilder.java    From sequenceiq-samples with Apache License 2.0 5 votes vote down vote up
public ToLowerCase(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    this.fieldName = getConfigs().getString(config, "field");
    this.locale = getConfigs().getLocale(config, "locale", Locale.ROOT);
    LOG.debug("fieldName: {}", fieldName);
    validateArguments();
}
 
Example #27
Source File: HashDigestBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public HashDigest(CommandBuilder builder, Config config, Command parent,
    Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);

  this.inputFieldName = getConfigs().getString(config, INPUT_FIELD);
  this.outputFieldName = getConfigs().getString(config, OUTPUT_FIELD);
  this.hashType = getConfigs().getString(config, HASH_TYPE);
  this.preserveExisting = getConfigs().getBoolean(config,
      PRESERVE_EXISTING_NAME, PRESERVE_EXISTING_DEFAULT);
  this.charset = getConfigs().getCharset(config, CHARSET_FIELD, Charsets.UTF_8);
  

  try {
    this.digest = MessageDigest.getInstance(hashType);
  } catch (NoSuchAlgorithmException e) {
    throw new MorphlineCompilationException("Unable to initialise digest", config, e);
  }
  
  validateArguments();

  if (LOG.isTraceEnabled()) {
    LOG.trace("inputField: {}", inputFieldName);
    LOG.trace("outputField: {}", outputFieldName);
    LOG.trace("hashType: {}", hashType);
    LOG.trace("preserveExisting: {}", preserveExisting );
  }
}
 
Example #28
Source File: ExtractURIQueryParametersBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public ExtractURIQueryParameters(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.parameterName = getConfigs().getString(config, "parameter");
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");
  this.failOnInvalidURI = getConfigs().getBoolean(config, "failOnInvalidURI", false);
  this.maxParameters = getConfigs().getInt(config, "maxParameters", Integer.MAX_VALUE);
  this.charset = getConfigs().getString(config, "charset", "UTF-8");
  Charset.forName(charset); // fail fast if charset is unsupported
  validateArguments();
}
 
Example #29
Source File: LogCommand.java    From kite with Apache License 2.0 5 votes vote down vote up
public LogCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.format = getConfigs().getString(config, "format");
  List<String> argList = getConfigs().getStringList(config, "args", Collections.<String>emptyList());
  this.expressions = new FieldExpression[argList.size()];
  for (int i = 0; i < argList.size(); i++) {
    this.expressions[i] = new FieldExpression(argList.get(i), getConfig()); 
  }
  validateArguments();
}
 
Example #30
Source File: TranslateBuilder.java    From kite with Apache License 2.0 5 votes vote down vote up
public Translate(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.fieldName = getConfigs().getString(config, "field");
  Config dict = getConfigs().getConfig(config, "dictionary");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(dict)) {
    dictionary.put(entry.getKey(), entry.getValue());
  }
  this.fallback = getConfigs().getString(config, "fallback", null);
  validateArguments();
}