org.kitesdk.morphline.api.CommandBuilder Java Examples

public ConvertHTML(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws SAXNotRecognizedException, SAXNotSupportedException {
  super(builder, config, parent, child, context);
  this.charset = getConfigs().getCharset(config, "charset", null);
  this.omitXMLDeclaration = getConfigs().getBoolean(config, "omitXMLDeclaration", false);      
  this.xmlReader = new Parser(); // no reuse?
  xmlReader.setProperty(Parser.schemaProperty, htmlSchema);
  xmlReader.setFeature(Parser.CDATAElementsFeature, getConfigs().getBoolean(config, "noCDATA", false));
  xmlReader.setFeature(Parser.namespacesFeature, !getConfigs().getBoolean(config, "noNamespaces", true));
  xmlReader.setFeature(Parser.ignoreBogonsFeature, getConfigs().getBoolean(config, "noBogons", false)); // also see TIKA-599
  xmlReader.setFeature(Parser.bogonsEmptyFeature, getConfigs().getBoolean(config, "emptyBogons", false));
  xmlReader.setFeature(Parser.rootBogonsFeature, getConfigs().getBoolean(config, "noRootBogons", false));
  xmlReader.setFeature(Parser.defaultAttributesFeature, getConfigs().getBoolean(config, "noDefaultAttributes", false));
  xmlReader.setFeature(Parser.translateColonsFeature, getConfigs().getBoolean(config, "noColons", false));
  xmlReader.setFeature(Parser.restartElementsFeature, getConfigs().getBoolean(config, "noRestart", false));
  xmlReader.setFeature(Parser.ignorableWhitespaceFeature, !getConfigs().getBoolean(config, "suppressIgnorableWhitespace", true));
  validateArguments();
}
 

public FindReplace(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  GrokDictionaries dict = new GrokDictionaries(config, getConfigs());
  String replacementStr = getConfigs().getString(config, "replacement");
  String pattern = getConfigs().getString(config, "pattern");
  if (getConfigs().getBoolean(config, "isRegex", false)) {
    Pattern regex = dict.compileExpression(pattern);
    this.matcher = regex.pattern().matcher("");
    replacementStr = regex.replaceProperties(replacementStr);
    this.literalPattern = null;
  } else {
    this.matcher = null;
    this.literalPattern = pattern;
  }
  this.replacement = replacementStr;
  this.replaceFirst = getConfigs().getBoolean(config, "replaceFirst", false);
  validateArguments();
}
 

public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");      
  String solrFieldType = getConfigs().getString(config, "solrFieldType");      
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
  if (fieldType == null) {
    throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
  }
  this.analyzer = fieldType.getIndexAnalyzer();
  Preconditions.checkNotNull(analyzer);
  // register CharTermAttribute for later (implicit) reuse
  this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class);
  Preconditions.checkNotNull(token);
  validateArguments();
}
 

public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM);
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  RetryPolicyFactory retryPolicyFactory = parseRetryPolicyFactory(
      getConfigs().getConfig(config, "retryPolicy", null));
  this.loader = locator.getLoader(retryPolicyFactory, new CodahaleMetricsFacade(context.getMetricRegistry()));

  Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty());
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) {
    String fieldName = entry.getKey();        
    float boost = Float.parseFloat(entry.getValue().toString().trim());
    boosts.put(fieldName, boost);
  }
  this.rateLimiter = RateLimiter.create(getConfigs().getDouble(config, "maxRecordsPerSecond", Double.MAX_VALUE));
  this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false);
  validateArguments();
  this.elapsedTime = getTimer(Metrics.ELAPSED_TIME);
}
 

public SplitKeyValue(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");      
  this.outputFieldPrefix = getConfigs().getString(config, "outputFieldPrefix", "");
  this.separator = getConfigs().getString(config, "separator", "=");
  if (separator.length() == 0) {
    throw new MorphlineCompilationException("separator must not be the empty string", config);
  }
  if (getConfigs().getBoolean(config, "isRegex", false)) {
    GrokDictionaries dict = new GrokDictionaries(config, getConfigs());
    this.regex = dict.compileExpression(separator).pattern().matcher("");
  } else {
    this.regex = null;
  }
  this.addEmptyStrings = getConfigs().getBoolean(config, "addEmptyStrings", false);      
  this.trim = getConfigs().getBoolean(config, "trim", true);
  validateArguments();
}
 

@SuppressWarnings("unchecked")
public TryRules(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.throwExceptionIfAllRulesFailed = getConfigs().getBoolean(config, "throwExceptionIfAllRulesFailed", true);
  this.catchExceptions = getConfigs().getBoolean(config, "catchExceptions", false);
  this.copyRecords = getConfigs().getBoolean(config, "copyRecords", true);
  
  List<? extends Config> ruleConfigs = getConfigs().getConfigList(config, "rules", Collections.EMPTY_LIST);
  for (Config ruleConfig : ruleConfigs) {
    List<Command> commands = buildCommandChain(ruleConfig, "commands", child, true);
    if (commands.size() > 0) {
      childRules.add(commands.get(0));
    }
  }
  validateArguments();
  numExceptionsCaught = getMeter("numExceptionsCaught");
}
 

public GenerateUUID(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  this.fieldName = getConfigs().getString(config, FIELD_NAME, Fields.ID);
  this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true);
  this.prefix = getConfigs().getString(config, PREFIX_NAME, "");
  Type type = new Validator<Type>().validateEnum(
      config,
      getConfigs().getString(config, "type", Type.secure.toString()),
      Type.class);
  if (type == Type.secure) {
    prng = null; // secure & slow
  } else {
    Random rand = new SecureRandom();
    int[] seed = new int[624];
    for (int i = 0; i < seed.length; i++) {
      seed[i] = rand.nextInt();
    }
    prng = new Well19937c(seed); // non-secure & fast
  }
  validateArguments();
}
 

public AddLocalHost(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  this.fieldName = getConfigs().getString(config, FIELD_NAME, "host");
  this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true);
  boolean useIP = getConfigs().getBoolean(config, USE_IP, true);      
  validateArguments();
  
  InetAddress addr = null;
  try {
    addr = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    LOG.warn("Cannot get address of local host", e);
  }
  
  if (addr == null) {
    host = null;
  } else if (useIP) {
    host = addr.getHostAddress();
  } else {
    host = addr.getCanonicalHostName();
  }
}
 

public ReadAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  String schemaString = getConfigs().getString(config, "writerSchemaString", null);
  if (schemaString != null) {
    this.writerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "writerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.writerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro writer schema file: " + schemaFile, config, e);
      }
    } else {
      this.writerSchema = null;
    }
  }
  
  this.isJson = getConfigs().getBoolean(config, "isJson", false);
  validateArguments();      
}
 

/**
 * Using the given <code>builder</code>, constructs a command rooted at the given morphline JSON
 * <code>config</code>.
 * 
 * The command will feed records into <code>child</code>. The command will have
 * <code>parent</code> as it's parent. Additional parameters can be passed via the morphline
 * <code>context</code>.
 */
protected AbstractCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  Preconditions.checkNotNull(builder);
  Preconditions.checkNotNull(config);
  Preconditions.checkNotNull(parent);
  Preconditions.checkNotNull(child);
  Preconditions.checkNotNull(context);
  this.config = config;
  this.parent = parent;
  this.child = child;
  this.context = context;
  Preconditions.checkArgument(builder.getNames().size() > 0);
  this.name = "morphline." + builder.getNames().iterator().next();
  this.configs = new Configs();
  this.numProcessCallsMeter = getMeter(Metrics.NUM_PROCESS_CALLS);
  this.numNotifyCallsMeter = getMeter(Metrics.NUM_NOTIFY_CALLS);
}
 

public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.id = getConfigs().getString(config, "id");

  List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", 
      Arrays.asList("com.**", "org.**", "net.**"));    
  context.importCommandBuilders(importCommandSpecs);

  getConfigs().getConfigList(config, "commands", null);
  List<Command> childCommands = buildCommandChain(config, "commands", child, false);
  if (childCommands.size() > 0) {
    this.realChild = childCommands.get(0);
  } else {
    this.realChild = child;
  }
  validateArguments();
}
 

public DownloadHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) 
    throws IOException {
  
  super(builder, config, parent, child, context);
  List<String> uris = getConfigs().getStringList(config, "inputFiles", Collections.<String>emptyList()); 
  File dstRootDir = new File(getConfigs().getString(config, "outputDir", "."));
  Configuration conf = new Configuration();
  String defaultFileSystemUri = getConfigs().getString(config, "fs", null);
  if (defaultFileSystemUri != null) {
    FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser
  }
  for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) {
    conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser
  }
  validateArguments();
  download(uris, conf, dstRootDir);
}
 

public IfThenElse(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  Command devNull = new DropRecordBuilder().build(null, this, null, context); // pipes into /dev/null
  List<Command> conditions = buildCommandChain(config, "conditions", devNull, true);
  if (conditions.size() == 0) {
    throw new MorphlineCompilationException("Missing conditions", config);
  } else {
    this.conditionChain = conditions.get(0);
  }

  List<Command> thenCommands = buildCommandChain(config, "then", child, true);
  if (thenCommands.size() > 0) {
    this.thenChain = thenCommands.get(0);
  }
  
  List<Command> elseCommands = buildCommandChain(config, "else", child, true);
  if (elseCommands.size() > 0) {
    this.elseChain = elseCommands.get(0);
  }
  validateArguments();
}
 

public WriteAvroToByteArray(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.format = new Validator<Format>().validateEnum(
      config,
      getConfigs().getString(config, "format", Format.container.toString()),
      Format.class);
  
  String codec = getConfigs().getString(config, "codec", null);
  if (codec == null) {
    this.codecFactory = null;
  } else {
    this.codecFactory = CodecFactory.fromString(codec);
  }
  
  Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty());
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) {
    this.metadata.put(entry.getKey(), entry.getValue().toString());
  }
  
  validateArguments();
}
 

public Java(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws ScriptException {
  super(builder, config, parent, child, context);
  
  String javaImports = getConfigs().getString(config, "imports", DEFAULT_IMPORTS);
  String javaCodeBlock = getConfigs().getString(config, "code");
  validateArguments();
  this.script = new ScriptEvaluator<Boolean>(
      javaImports, 
      javaCodeBlock, 
      Boolean.class,
      new String[] {"record", "config", "parent", "child", "context", "logger"}, 
      new Class[] {Record.class, Config.class, Command.class, Command.class, MorphlineContext.class, Logger.class}, 
      new Class[] {Exception.class},
      javaCodeBlock
      );
}
 

public StartReportingMetricsToSLF4J(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  
  MetricFilter filter = PatternMetricFilter.parse(getConfigs(), config);
  TimeUnit defaultDurationUnit = getConfigs().getTimeUnit(config, "defaultDurationUnit", TimeUnit.MILLISECONDS);
  TimeUnit defaultRateUnit = getConfigs().getTimeUnit(config, "defaultRateUnit", TimeUnit.SECONDS); 
  long frequency = getConfigs().getNanoseconds(config, "frequency", 10 * 1000L * 1000 * 1000); // 10 secs, also see https://github.com/typesafehub/config/blob/master/HOCON.md#duration-format
  this.logger = getConfigs().getString(config, "logger", "metrics");
  String marker = getConfigs().getString(config, "marker", null);      
  validateArguments();
  
  MetricRegistry registry = context.getMetricRegistry();
  synchronized (REGISTRIES) {
    Map<String, Slf4jReporter> reporters = REGISTRIES.get(registry);
    if (reporters == null) {
      reporters = Maps.newHashMap();
      REGISTRIES.put(registry, reporters);
    }
    Slf4jReporter reporter = reporters.get(logger);
    if (reporter == null) {
      Builder reporterBuilder = Slf4jReporter.forRegistry(registry)
          .filter(filter)
          .convertDurationsTo(defaultDurationUnit)
          .convertRatesTo(defaultRateUnit)
          .outputTo(LoggerFactory.getLogger(logger));
      
      if (marker != null) {
        reporterBuilder = reporterBuilder.markWith(new BasicMarkerFactory().getMarker(marker));
      }
          
      reporter = reporterBuilder.build();
      reporter.start(frequency, TimeUnit.NANOSECONDS);
      reporters.put(logger, reporter);
    }
  }
}
 

public ReadJsonTestTweets(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { 
  super(builder, config, parent, child, context);
  
  this.isLengthDelimited = getConfigs().getBoolean(config, "isLengthDelimited", true);
  this.idPrefix = getConfigs().getString(config, "idPrefix", null);
  if ("random".equals(idPrefix)) {
    idPrefix = String.valueOf(new Random().nextInt());
  } else if (idPrefix == null) {
    idPrefix = "";
  }
  validateArguments();
}
 

public GeoIP(CommandBuilder builder, Config config, Command parent, 
                                   Command child, final MorphlineContext context) {
  
  super(builder, config, parent, child, context);      
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.databaseFile = new File(getConfigs().getString(config, "database", "GeoLite2-City.mmdb"));
  try {
    this.databaseReader = new Reader(databaseFile);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot read Maxmind database: " + databaseFile, config, e);
  }
  validateArguments();
}
 

public ExtractAvroPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create();
  this.flatten = getConfigs().getBoolean(config, "flatten", true);
  Config paths = getConfigs().getConfig(config, "paths");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) {
    String fieldName = entry.getKey();        
    String path = entry.getValue().toString().trim();
    if (path.contains("//")) {
      throw new MorphlineCompilationException("No support for descendant axis available yet", config);
    }
    if (path.startsWith("/")) {
      path = path.substring(1);
    }
    if (path.endsWith("/")) {
      path = path.substring(0, path.length() - 1);
    }
    path = path.trim();
    for (String step : path.split("/")) {
      step = step.trim();
      if (step.length() > ARRAY_TOKEN.length() && step.endsWith(ARRAY_TOKEN)) {
        step = step.substring(0,  step.length() - ARRAY_TOKEN.length());
        stepMultiMap.put(fieldName, normalize(step));
        stepMultiMap.put(fieldName, ARRAY_TOKEN);
      } else {
        stepMultiMap.put(fieldName, normalize(step));
      }
    }
  }
  this.stepMap = stepMultiMap.asMap();
  LOG.debug("stepMap: {}", stepMap);
  validateArguments();
}
 

public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {   
  super(builder, config, parent, child, context);

  String schemaString = getConfigs().getString(config, "readerSchemaString", null);
  if (schemaString != null) {
    this.readerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "readerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.readerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e);
      }
    } else {
      this.readerSchema = null;
    }
  }
  
  if (getClass() == ReadAvroContainer.class) {
    resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>(
        getConfigs().getInt(config, "schemaCacheCapacity", 100));
    
    validateArguments();
  } else {
    resolverCache = null;
  }
}
 

public ToAvroMap(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  
  InputStream in = null;
  try {
    in = getClass().getResourceAsStream("morphlineRecord.avsc");
    this.schema = new Schema.Parser().parse(in);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot parse morphlineRecord schema", config, e, builder);
  } finally {
    Closeables.closeQuietly(in);
  }
  
  validateArguments();
}
 

public OpenHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.conf = new Configuration();
  String defaultFileSystemUri = getConfigs().getString(config, "fs", null);
  if (defaultFileSystemUri != null) {
    FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser
  }
  for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) {
    conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser
  }
  validateArguments();
}
 

public SaxonCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    
    this.isTracing = getConfigs().getBoolean(config, "isTracing", false);
    boolean isLicensedSaxonEdition = getConfigs().getBoolean(config, "isLicensedSaxonEdition", false);
    this.processor = new Processor(isLicensedSaxonEdition);
    this.documentBuilder = processor.newDocumentBuilder();
    
    Config features = getConfigs().getConfig(config, "features", ConfigFactory.empty());
    for (Map.Entry<String, Object> entry : new Configs().getEntrySet(features)) {
      processor.setConfigurationProperty(entry.getKey(), entry.getValue());
    }
    
    for (String clazz : getConfigs().getStringList(config, "extensionFunctions", Collections.<String>emptyList())) {
      Object function;
      try {
        function = Class.forName(clazz).newInstance();
      } catch (Exception e) {
        throw new MorphlineCompilationException("Cannot instantiate extension function: " + clazz, config);
      }
      
      if (function instanceof ExtensionFunction) {
        processor.registerExtensionFunction((ExtensionFunction) function);              
//      }
//      else if (function instanceof ExtensionFunctionDefinition) {
//        processor.registerExtensionFunction((ExtensionFunctionDefinition) function);              
      } else {
        throw new MorphlineCompilationException("Extension function has wrong class: " + clazz, config);
      }
    }
  }
 

public RegisterJVMMetrics(CommandBuilder builder, Config config, Command parent, 
                                   Command child, final MorphlineContext context) {
  
  super(builder, config, parent, child, context);      
  validateArguments();
  
  MetricRegistry registry = context.getMetricRegistry();
  BufferPoolMetricSet bufferPoolMetrics = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer());
  registerAll("jvm.buffers", bufferPoolMetrics, registry);
  registerAll("jvm.gc", new GarbageCollectorMetricSet(), registry);
  registerAll("jvm.memory", new MemoryUsageGaugeSet(), registry);
  registerAll("jvm.threads", new ThreadStatesGaugeSet(), registry);
  register("jvm.fileDescriptorCountRatio", new FileDescriptorRatioGauge(), registry);
  context.getHealthCheckRegistry().register("deadlocks", new ThreadDeadlockHealthCheck());
}
 

public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context,
    boolean isRemoving) {
  super(builder, config, parent, child, context);
  List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*"));
  List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList());
  List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*"));
  List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList());
  int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000);
  this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity);
  int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0);
  this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity);
  this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement");
  validateArguments();
}
 

public ToLowerCase(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    this.fieldName = getConfigs().getString(config, "field");
    this.locale = getConfigs().getLocale(config, "locale", Locale.ROOT);
    LOG.debug("fieldName: {}", fieldName);
    validateArguments();
}
 

public HashDigest(CommandBuilder builder, Config config, Command parent,
    Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);

  this.inputFieldName = getConfigs().getString(config, INPUT_FIELD);
  this.outputFieldName = getConfigs().getString(config, OUTPUT_FIELD);
  this.hashType = getConfigs().getString(config, HASH_TYPE);
  this.preserveExisting = getConfigs().getBoolean(config,
      PRESERVE_EXISTING_NAME, PRESERVE_EXISTING_DEFAULT);
  this.charset = getConfigs().getCharset(config, CHARSET_FIELD, Charsets.UTF_8);
  

  try {
    this.digest = MessageDigest.getInstance(hashType);
  } catch (NoSuchAlgorithmException e) {
    throw new MorphlineCompilationException("Unable to initialise digest", config, e);
  }
  
  validateArguments();

  if (LOG.isTraceEnabled()) {
    LOG.trace("inputField: {}", inputFieldName);
    LOG.trace("outputField: {}", outputFieldName);
    LOG.trace("hashType: {}", hashType);
    LOG.trace("preserveExisting: {}", preserveExisting );
  }
}
 

public ExtractURIQueryParameters(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.parameterName = getConfigs().getString(config, "parameter");
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");
  this.failOnInvalidURI = getConfigs().getBoolean(config, "failOnInvalidURI", false);
  this.maxParameters = getConfigs().getInt(config, "maxParameters", Integer.MAX_VALUE);
  this.charset = getConfigs().getString(config, "charset", "UTF-8");
  Charset.forName(charset); // fail fast if charset is unsupported
  validateArguments();
}
 

public LogCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.format = getConfigs().getString(config, "format");
  List<String> argList = getConfigs().getStringList(config, "args", Collections.<String>emptyList());
  this.expressions = new FieldExpression[argList.size()];
  for (int i = 0; i < argList.size(); i++) {
    this.expressions[i] = new FieldExpression(argList.get(i), getConfig()); 
  }
  validateArguments();
}
 

public Translate(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  this.fieldName = getConfigs().getString(config, "field");
  Config dict = getConfigs().getConfig(config, "dictionary");
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(dict)) {
    dictionary.put(entry.getKey(), entry.getValue());
  }
  this.fallback = getConfigs().getString(config, "fallback", null);
  validateArguments();
}