org.kitesdk.morphline.api.CommandBuilder Java Examples
The following examples show how to use
org.kitesdk.morphline.api.CommandBuilder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConvertHTMLBuilder.java From kite with Apache License 2.0 | 6 votes |
public ConvertHTML(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws SAXNotRecognizedException, SAXNotSupportedException { super(builder, config, parent, child, context); this.charset = getConfigs().getCharset(config, "charset", null); this.omitXMLDeclaration = getConfigs().getBoolean(config, "omitXMLDeclaration", false); this.xmlReader = new Parser(); // no reuse? xmlReader.setProperty(Parser.schemaProperty, htmlSchema); xmlReader.setFeature(Parser.CDATAElementsFeature, getConfigs().getBoolean(config, "noCDATA", false)); xmlReader.setFeature(Parser.namespacesFeature, !getConfigs().getBoolean(config, "noNamespaces", true)); xmlReader.setFeature(Parser.ignoreBogonsFeature, getConfigs().getBoolean(config, "noBogons", false)); // also see TIKA-599 xmlReader.setFeature(Parser.bogonsEmptyFeature, getConfigs().getBoolean(config, "emptyBogons", false)); xmlReader.setFeature(Parser.rootBogonsFeature, getConfigs().getBoolean(config, "noRootBogons", false)); xmlReader.setFeature(Parser.defaultAttributesFeature, getConfigs().getBoolean(config, "noDefaultAttributes", false)); xmlReader.setFeature(Parser.translateColonsFeature, getConfigs().getBoolean(config, "noColons", false)); xmlReader.setFeature(Parser.restartElementsFeature, getConfigs().getBoolean(config, "noRestart", false)); xmlReader.setFeature(Parser.ignorableWhitespaceFeature, !getConfigs().getBoolean(config, "suppressIgnorableWhitespace", true)); validateArguments(); }
Example #2
Source File: FindReplaceBuilder.java From kite with Apache License 2.0 | 6 votes |
public FindReplace(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); GrokDictionaries dict = new GrokDictionaries(config, getConfigs()); String replacementStr = getConfigs().getString(config, "replacement"); String pattern = getConfigs().getString(config, "pattern"); if (getConfigs().getBoolean(config, "isRegex", false)) { Pattern regex = dict.compileExpression(pattern); this.matcher = regex.pattern().matcher(""); replacementStr = regex.replaceProperties(replacementStr); this.literalPattern = null; } else { this.matcher = null; this.literalPattern = pattern; } this.replacement = replacementStr; this.replaceFirst = getConfigs().getBoolean(config, "replaceFirst", false); validateArguments(); }
Example #3
Source File: TokenizeTextBuilder.java From kite with Apache License 2.0 | 6 votes |
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); String solrFieldType = getConfigs().getString(config, "solrFieldType"); Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator"); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); IndexSchema schema = locator.getIndexSchema(); FieldType fieldType = schema.getFieldTypeByName(solrFieldType); if (fieldType == null) { throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config); } this.analyzer = fieldType.getIndexAnalyzer(); Preconditions.checkNotNull(analyzer); // register CharTermAttribute for later (implicit) reuse this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class); Preconditions.checkNotNull(token); validateArguments(); }
Example #4
Source File: LoadSolrBuilder.java From kite with Apache License 2.0 | 6 votes |
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Config solrLocatorConfig = getConfigs().getConfig(config, SOLR_LOCATOR_PARAM); SolrLocator locator = new SolrLocator(solrLocatorConfig, context); LOG.debug("solrLocator: {}", locator); RetryPolicyFactory retryPolicyFactory = parseRetryPolicyFactory( getConfigs().getConfig(config, "retryPolicy", null)); this.loader = locator.getLoader(retryPolicyFactory, new CodahaleMetricsFacade(context.getMetricRegistry())); Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) { String fieldName = entry.getKey(); float boost = Float.parseFloat(entry.getValue().toString().trim()); boosts.put(fieldName, boost); } this.rateLimiter = RateLimiter.create(getConfigs().getDouble(config, "maxRecordsPerSecond", Double.MAX_VALUE)); this.isDryRun = context.getTypedSettings().getBoolean(TypedSettings.DRY_RUN_SETTING_NAME, false); validateArguments(); this.elapsedTime = getTimer(Metrics.ELAPSED_TIME); }
Example #5
Source File: SplitKeyValueBuilder.java From kite with Apache License 2.0 | 6 votes |
public SplitKeyValue(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldPrefix = getConfigs().getString(config, "outputFieldPrefix", ""); this.separator = getConfigs().getString(config, "separator", "="); if (separator.length() == 0) { throw new MorphlineCompilationException("separator must not be the empty string", config); } if (getConfigs().getBoolean(config, "isRegex", false)) { GrokDictionaries dict = new GrokDictionaries(config, getConfigs()); this.regex = dict.compileExpression(separator).pattern().matcher(""); } else { this.regex = null; } this.addEmptyStrings = getConfigs().getBoolean(config, "addEmptyStrings", false); this.trim = getConfigs().getBoolean(config, "trim", true); validateArguments(); }
Example #6
Source File: TryRulesBuilder.java From kite with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") public TryRules(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.throwExceptionIfAllRulesFailed = getConfigs().getBoolean(config, "throwExceptionIfAllRulesFailed", true); this.catchExceptions = getConfigs().getBoolean(config, "catchExceptions", false); this.copyRecords = getConfigs().getBoolean(config, "copyRecords", true); List<? extends Config> ruleConfigs = getConfigs().getConfigList(config, "rules", Collections.EMPTY_LIST); for (Config ruleConfig : ruleConfigs) { List<Command> commands = buildCommandChain(ruleConfig, "commands", child, true); if (commands.size() > 0) { childRules.add(commands.get(0)); } } validateArguments(); numExceptionsCaught = getMeter("numExceptionsCaught"); }
Example #7
Source File: GenerateUUIDBuilder.java From kite with Apache License 2.0 | 6 votes |
public GenerateUUID(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, FIELD_NAME, Fields.ID); this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true); this.prefix = getConfigs().getString(config, PREFIX_NAME, ""); Type type = new Validator<Type>().validateEnum( config, getConfigs().getString(config, "type", Type.secure.toString()), Type.class); if (type == Type.secure) { prng = null; // secure & slow } else { Random rand = new SecureRandom(); int[] seed = new int[624]; for (int i = 0; i < seed.length; i++) { seed[i] = rand.nextInt(); } prng = new Well19937c(seed); // non-secure & fast } validateArguments(); }
Example #8
Source File: AddLocalHostBuilder.java From kite with Apache License 2.0 | 6 votes |
public AddLocalHost(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, FIELD_NAME, "host"); this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, true); boolean useIP = getConfigs().getBoolean(config, USE_IP, true); validateArguments(); InetAddress addr = null; try { addr = InetAddress.getLocalHost(); } catch (UnknownHostException e) { LOG.warn("Cannot get address of local host", e); } if (addr == null) { host = null; } else if (useIP) { host = addr.getHostAddress(); } else { host = addr.getCanonicalHostName(); } }
Example #9
Source File: ReadAvroBuilder.java From kite with Apache License 2.0 | 6 votes |
public ReadAvro(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "writerSchemaString", null); if (schemaString != null) { this.writerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "writerSchemaFile", null); if (schemaFile != null) { try { this.writerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro writer schema file: " + schemaFile, config, e); } } else { this.writerSchema = null; } } this.isJson = getConfigs().getBoolean(config, "isJson", false); validateArguments(); }
Example #10
Source File: AbstractCommand.java From kite with Apache License 2.0 | 6 votes |
/** * Using the given <code>builder</code>, constructs a command rooted at the given morphline JSON * <code>config</code>. * * The command will feed records into <code>child</code>. The command will have * <code>parent</code> as it's parent. Additional parameters can be passed via the morphline * <code>context</code>. */ protected AbstractCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { Preconditions.checkNotNull(builder); Preconditions.checkNotNull(config); Preconditions.checkNotNull(parent); Preconditions.checkNotNull(child); Preconditions.checkNotNull(context); this.config = config; this.parent = parent; this.child = child; this.context = context; Preconditions.checkArgument(builder.getNames().size() > 0); this.name = "morphline." + builder.getNames().iterator().next(); this.configs = new Configs(); this.numProcessCallsMeter = getMeter(Metrics.NUM_PROCESS_CALLS); this.numNotifyCallsMeter = getMeter(Metrics.NUM_NOTIFY_CALLS); }
Example #11
Source File: Pipe.java From kite with Apache License 2.0 | 6 votes |
public Pipe(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.id = getConfigs().getString(config, "id"); List<String> importCommandSpecs = getConfigs().getStringList(config, "importCommands", Arrays.asList("com.**", "org.**", "net.**")); context.importCommandBuilders(importCommandSpecs); getConfigs().getConfigList(config, "commands", null); List<Command> childCommands = buildCommandChain(config, "commands", child, false); if (childCommands.size() > 0) { this.realChild = childCommands.get(0); } else { this.realChild = child; } validateArguments(); }
Example #12
Source File: DownloadHdfsFileBuilder.java From kite with Apache License 2.0 | 6 votes |
public DownloadHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws IOException { super(builder, config, parent, child, context); List<String> uris = getConfigs().getStringList(config, "inputFiles", Collections.<String>emptyList()); File dstRootDir = new File(getConfigs().getString(config, "outputDir", ".")); Configuration conf = new Configuration(); String defaultFileSystemUri = getConfigs().getString(config, "fs", null); if (defaultFileSystemUri != null) { FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser } for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) { conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser } validateArguments(); download(uris, conf, dstRootDir); }
Example #13
Source File: IfThenElseBuilder.java From kite with Apache License 2.0 | 6 votes |
public IfThenElse(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); Command devNull = new DropRecordBuilder().build(null, this, null, context); // pipes into /dev/null List<Command> conditions = buildCommandChain(config, "conditions", devNull, true); if (conditions.size() == 0) { throw new MorphlineCompilationException("Missing conditions", config); } else { this.conditionChain = conditions.get(0); } List<Command> thenCommands = buildCommandChain(config, "then", child, true); if (thenCommands.size() > 0) { this.thenChain = thenCommands.get(0); } List<Command> elseCommands = buildCommandChain(config, "else", child, true); if (elseCommands.size() > 0) { this.elseChain = elseCommands.get(0); } validateArguments(); }
Example #14
Source File: WriteAvroToByteArrayBuilder.java From kite with Apache License 2.0 | 6 votes |
public WriteAvroToByteArray(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = new Validator<Format>().validateEnum( config, getConfigs().getString(config, "format", Format.container.toString()), Format.class); String codec = getConfigs().getString(config, "codec", null); if (codec == null) { this.codecFactory = null; } else { this.codecFactory = CodecFactory.fromString(codec); } Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) { this.metadata.put(entry.getKey(), entry.getValue().toString()); } validateArguments(); }
Example #15
Source File: JavaBuilder.java From kite with Apache License 2.0 | 6 votes |
public Java(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) throws ScriptException { super(builder, config, parent, child, context); String javaImports = getConfigs().getString(config, "imports", DEFAULT_IMPORTS); String javaCodeBlock = getConfigs().getString(config, "code"); validateArguments(); this.script = new ScriptEvaluator<Boolean>( javaImports, javaCodeBlock, Boolean.class, new String[] {"record", "config", "parent", "child", "context", "logger"}, new Class[] {Record.class, Config.class, Command.class, Command.class, MorphlineContext.class, Logger.class}, new Class[] {Exception.class}, javaCodeBlock ); }
Example #16
Source File: StartReportingMetricsToSLF4JBuilder.java From kite with Apache License 2.0 | 5 votes |
public StartReportingMetricsToSLF4J(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); MetricFilter filter = PatternMetricFilter.parse(getConfigs(), config); TimeUnit defaultDurationUnit = getConfigs().getTimeUnit(config, "defaultDurationUnit", TimeUnit.MILLISECONDS); TimeUnit defaultRateUnit = getConfigs().getTimeUnit(config, "defaultRateUnit", TimeUnit.SECONDS); long frequency = getConfigs().getNanoseconds(config, "frequency", 10 * 1000L * 1000 * 1000); // 10 secs, also see https://github.com/typesafehub/config/blob/master/HOCON.md#duration-format this.logger = getConfigs().getString(config, "logger", "metrics"); String marker = getConfigs().getString(config, "marker", null); validateArguments(); MetricRegistry registry = context.getMetricRegistry(); synchronized (REGISTRIES) { Map<String, Slf4jReporter> reporters = REGISTRIES.get(registry); if (reporters == null) { reporters = Maps.newHashMap(); REGISTRIES.put(registry, reporters); } Slf4jReporter reporter = reporters.get(logger); if (reporter == null) { Builder reporterBuilder = Slf4jReporter.forRegistry(registry) .filter(filter) .convertDurationsTo(defaultDurationUnit) .convertRatesTo(defaultRateUnit) .outputTo(LoggerFactory.getLogger(logger)); if (marker != null) { reporterBuilder = reporterBuilder.markWith(new BasicMarkerFactory().getMarker(marker)); } reporter = reporterBuilder.build(); reporter.start(frequency, TimeUnit.NANOSECONDS); reporters.put(logger, reporter); } } }
Example #17
Source File: ReadJsonTestTweetsBuilder.java From kite with Apache License 2.0 | 5 votes |
public ReadJsonTestTweets(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.isLengthDelimited = getConfigs().getBoolean(config, "isLengthDelimited", true); this.idPrefix = getConfigs().getString(config, "idPrefix", null); if ("random".equals(idPrefix)) { idPrefix = String.valueOf(new Random().nextInt()); } else if (idPrefix == null) { idPrefix = ""; } validateArguments(); }
Example #18
Source File: GeoIPBuilder.java From kite with Apache License 2.0 | 5 votes |
public GeoIP(CommandBuilder builder, Config config, Command parent, Command child, final MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, "inputField"); this.databaseFile = new File(getConfigs().getString(config, "database", "GeoLite2-City.mmdb")); try { this.databaseReader = new Reader(databaseFile); } catch (IOException e) { throw new MorphlineCompilationException("Cannot read Maxmind database: " + databaseFile, config, e); } validateArguments(); }
Example #19
Source File: ExtractAvroPathsBuilder.java From kite with Apache License 2.0 | 5 votes |
public ExtractAvroPaths(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); ListMultimap<String, String> stepMultiMap = ArrayListMultimap.create(); this.flatten = getConfigs().getBoolean(config, "flatten", true); Config paths = getConfigs().getConfig(config, "paths"); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(paths)) { String fieldName = entry.getKey(); String path = entry.getValue().toString().trim(); if (path.contains("//")) { throw new MorphlineCompilationException("No support for descendant axis available yet", config); } if (path.startsWith("/")) { path = path.substring(1); } if (path.endsWith("/")) { path = path.substring(0, path.length() - 1); } path = path.trim(); for (String step : path.split("/")) { step = step.trim(); if (step.length() > ARRAY_TOKEN.length() && step.endsWith(ARRAY_TOKEN)) { step = step.substring(0, step.length() - ARRAY_TOKEN.length()); stepMultiMap.put(fieldName, normalize(step)); stepMultiMap.put(fieldName, ARRAY_TOKEN); } else { stepMultiMap.put(fieldName, normalize(step)); } } } this.stepMap = stepMultiMap.asMap(); LOG.debug("stepMap: {}", stepMap); validateArguments(); }
Example #20
Source File: ReadAvroContainerBuilder.java From kite with Apache License 2.0 | 5 votes |
public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); String schemaString = getConfigs().getString(config, "readerSchemaString", null); if (schemaString != null) { this.readerSchema = new Parser().parse(schemaString); } else { String schemaFile = getConfigs().getString(config, "readerSchemaFile", null); if (schemaFile != null) { try { this.readerSchema = new Parser().parse(new File(schemaFile)); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e); } } else { this.readerSchema = null; } } if (getClass() == ReadAvroContainer.class) { resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>( getConfigs().getInt(config, "schemaCacheCapacity", 100)); validateArguments(); } else { resolverCache = null; } }
Example #21
Source File: ToAvroMapBuilder.java From kite with Apache License 2.0 | 5 votes |
public ToAvroMap(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); InputStream in = null; try { in = getClass().getResourceAsStream("morphlineRecord.avsc"); this.schema = new Schema.Parser().parse(in); } catch (IOException e) { throw new MorphlineCompilationException("Cannot parse morphlineRecord schema", config, e, builder); } finally { Closeables.closeQuietly(in); } validateArguments(); }
Example #22
Source File: OpenHdfsFileBuilder.java From kite with Apache License 2.0 | 5 votes |
public OpenHdfsFile(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.conf = new Configuration(); String defaultFileSystemUri = getConfigs().getString(config, "fs", null); if (defaultFileSystemUri != null) { FileSystem.setDefaultUri(conf, defaultFileSystemUri); // see Hadoop's GenericOptionsParser } for (String value : getConfigs().getStringList(config, "conf", Collections.<String>emptyList())) { conf.addResource(new Path(value)); // see Hadoop's GenericOptionsParser } validateArguments(); }
Example #23
Source File: SaxonCommand.java From kite with Apache License 2.0 | 5 votes |
public SaxonCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.isTracing = getConfigs().getBoolean(config, "isTracing", false); boolean isLicensedSaxonEdition = getConfigs().getBoolean(config, "isLicensedSaxonEdition", false); this.processor = new Processor(isLicensedSaxonEdition); this.documentBuilder = processor.newDocumentBuilder(); Config features = getConfigs().getConfig(config, "features", ConfigFactory.empty()); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(features)) { processor.setConfigurationProperty(entry.getKey(), entry.getValue()); } for (String clazz : getConfigs().getStringList(config, "extensionFunctions", Collections.<String>emptyList())) { Object function; try { function = Class.forName(clazz).newInstance(); } catch (Exception e) { throw new MorphlineCompilationException("Cannot instantiate extension function: " + clazz, config); } if (function instanceof ExtensionFunction) { processor.registerExtensionFunction((ExtensionFunction) function); // } // else if (function instanceof ExtensionFunctionDefinition) { // processor.registerExtensionFunction((ExtensionFunctionDefinition) function); } else { throw new MorphlineCompilationException("Extension function has wrong class: " + clazz, config); } } }
Example #24
Source File: RegisterJVMMetricsBuilder.java From kite with Apache License 2.0 | 5 votes |
public RegisterJVMMetrics(CommandBuilder builder, Config config, Command parent, Command child, final MorphlineContext context) { super(builder, config, parent, child, context); validateArguments(); MetricRegistry registry = context.getMetricRegistry(); BufferPoolMetricSet bufferPoolMetrics = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()); registerAll("jvm.buffers", bufferPoolMetrics, registry); registerAll("jvm.gc", new GarbageCollectorMetricSet(), registry); registerAll("jvm.memory", new MemoryUsageGaugeSet(), registry); registerAll("jvm.threads", new ThreadStatesGaugeSet(), registry); register("jvm.fileDescriptorCountRatio", new FileDescriptorRatioGauge(), registry); context.getHealthCheckRegistry().register("deadlocks", new ThreadDeadlockHealthCheck()); }
Example #25
Source File: ReplaceValues.java From kite with Apache License 2.0 | 5 votes |
public ReplaceValues(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context, boolean isRemoving) { super(builder, config, parent, child, context); List<String> nameBlacklist = getConfigs().getStringList(config, "nameBlacklist", Collections.singletonList("*")); List<String> nameWhitelist = getConfigs().getStringList(config, "nameWhitelist", Collections.<String>emptyList()); List<String> valueBlacklist = getConfigs().getStringList(config, "valueBlacklist", Collections.singletonList("*")); List<String> valueWhitelist = getConfigs().getStringList(config, "valueWhitelist", Collections.<String>emptyList()); int nameCacheCapacity = getConfigs().getInt(config, "nameCacheCapacity", 10000); this.nameMatcher = new PatternNameMatcher(nameBlacklist, nameWhitelist, nameCacheCapacity); int valueCacheCapacity = getConfigs().getInt(config, "valueCacheCapacity", 0); this.valueMatcher = new PatternNameMatcher(valueBlacklist, valueWhitelist, valueCacheCapacity); this.replacement = isRemoving ? null : getConfigs().getString(config, "replacement"); validateArguments(); }
Example #26
Source File: ToLowerCaseBuilder.java From sequenceiq-samples with Apache License 2.0 | 5 votes |
public ToLowerCase(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, "field"); this.locale = getConfigs().getLocale(config, "locale", Locale.ROOT); LOG.debug("fieldName: {}", fieldName); validateArguments(); }
Example #27
Source File: HashDigestBuilder.java From kite with Apache License 2.0 | 5 votes |
public HashDigest(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.inputFieldName = getConfigs().getString(config, INPUT_FIELD); this.outputFieldName = getConfigs().getString(config, OUTPUT_FIELD); this.hashType = getConfigs().getString(config, HASH_TYPE); this.preserveExisting = getConfigs().getBoolean(config, PRESERVE_EXISTING_NAME, PRESERVE_EXISTING_DEFAULT); this.charset = getConfigs().getCharset(config, CHARSET_FIELD, Charsets.UTF_8); try { this.digest = MessageDigest.getInstance(hashType); } catch (NoSuchAlgorithmException e) { throw new MorphlineCompilationException("Unable to initialise digest", config, e); } validateArguments(); if (LOG.isTraceEnabled()) { LOG.trace("inputField: {}", inputFieldName); LOG.trace("outputField: {}", outputFieldName); LOG.trace("hashType: {}", hashType); LOG.trace("preserveExisting: {}", preserveExisting ); } }
Example #28
Source File: ExtractURIQueryParametersBuilder.java From kite with Apache License 2.0 | 5 votes |
public ExtractURIQueryParameters(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.parameterName = getConfigs().getString(config, "parameter"); this.inputFieldName = getConfigs().getString(config, "inputField"); this.outputFieldName = getConfigs().getString(config, "outputField"); this.failOnInvalidURI = getConfigs().getBoolean(config, "failOnInvalidURI", false); this.maxParameters = getConfigs().getInt(config, "maxParameters", Integer.MAX_VALUE); this.charset = getConfigs().getString(config, "charset", "UTF-8"); Charset.forName(charset); // fail fast if charset is unsupported validateArguments(); }
Example #29
Source File: LogCommand.java From kite with Apache License 2.0 | 5 votes |
public LogCommand(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.format = getConfigs().getString(config, "format"); List<String> argList = getConfigs().getStringList(config, "args", Collections.<String>emptyList()); this.expressions = new FieldExpression[argList.size()]; for (int i = 0; i < argList.size(); i++) { this.expressions[i] = new FieldExpression(argList.get(i), getConfig()); } validateArguments(); }
Example #30
Source File: TranslateBuilder.java From kite with Apache License 2.0 | 5 votes |
public Translate(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) { super(builder, config, parent, child, context); this.fieldName = getConfigs().getString(config, "field"); Config dict = getConfigs().getConfig(config, "dictionary"); for (Map.Entry<String, Object> entry : new Configs().getEntrySet(dict)) { dictionary.put(entry.getKey(), entry.getValue()); } this.fallback = getConfigs().getString(config, "fallback", null); validateArguments(); }