nl.basjes.parse.core.exceptions.InvalidDissectorException Java Examples

@Test
public void testSetTypeRemapping() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    try{
        new Parser<>(TestRecord.class)
            .setRootType("INPUT")
            .addDissector(new NormalValuesDissector())
            .addTypeRemapping("string", "STRINGXX")
            .addParseTarget("setStringValue", "STRINGXX:string")
            // Should wipe previous
            .setTypeRemappings(Collections.singletonMap("string", Collections.singleton("STRINGYY")))
            .addParseTarget("setStringValue", "STRINGYY:string")
            .parse("Doesn't matter");

        fail("We should get an exception because the mapping to STRINGXX:string wass removed.");
    } catch (MissingDissectorsException mde) {
        assertTrue(mde.getMessage().contains("STRINGXX:string"));
    }
}
 

public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat,
                   final String timestampFormat, final Map<String, String> fieldMapping)
        throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {

  Preconditions.checkArgument(logFormat != null && !logFormat.trim().isEmpty(), "logFormat cannot be null or empty");

  this.record = new HttpdLogRecord(managedBuffer, timestampFormat);
  this.parser = new HttpdLoglineParser<>(HttpdLogRecord.class, logFormat, timestampFormat);

  setupParser(mapWriter, logFormat, fieldMapping);

  if (timestampFormat != null && !timestampFormat.trim().isEmpty()) {
    LOG.info("Custom timestamp format has been specified. This is an informational note only as custom timestamps is rather unusual.");
  }
  if (logFormat.contains("\n")) {
    LOG.info("Specified logformat is a multiline log format: {}", logFormat);
  }
}
 

public void run() throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    HttpdLoglineParser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, logFormat);

    List<String> allPossiblePaths = parser.getPossiblePaths();
    parser.addParseTarget(MyRecord.class.getMethod("setter", String.class, String.class), allPossiblePaths);

    System.out.println("class MyRecord {\n");

    for (String field : parser.getPossiblePaths()) {
        for (Casts cast : parser.getCasts(field)) {
            System.out.println("    @Field{\"" + field + "\"}\n" +
                    "    public void setter(String name, " + castToJavaType(cast) + " value) {\n" +
                    "        System.out.println(\"SETTER CALLED FOR \\\"\" + name + \"\\\" = \\\"\" + value + \"\\\"\");\n" +
                    "    }\n");
        }
    }
    System.out.println("}\n");
}
 

private void assembleDissectorPhases() throws InvalidDissectorException {
    for (final Dissector dissector : allDissectors) {
        final String inputType = dissector.getInputType();
        if (inputType == null) {
            throw new InvalidDissectorException("Dissector returns null on getInputType(): ["+ dissector.getClass().getCanonicalName()+"]");
        }

        final List<String> outputs = dissector.getPossibleOutput();
        if (outputs == null || outputs.isEmpty()) {
            throw new InvalidDissectorException("Dissector cannot create any outputs: ["+ dissector.getClass().getCanonicalName()+"]");
        }

        // Create all dissector phases
        for (final String output: outputs) {
            final int colonPos = output.indexOf(':');
            final String outputType = output.substring(0, colonPos);
            final String name = output.substring(colonPos + 1);
            availableDissectors.add(new DissectorPhase(inputType, outputType, name, dissector));
        }
    }
}
 

private void printAllPossibles(String logformat) throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    // To figure out what values we CAN get from this line we instantiate the parser with a dummy class
    // that does not have ANY @Field annotations.
    Parser<Object> dummyParser= new HttpdLoglineParser<>(Object.class, logformat);

    List<String> possiblePaths;
    possiblePaths = dummyParser.getPossiblePaths();

    // If you want to call 'getCasts' then the actual parser needs to be constructed.
    // Simply calling getPossiblePaths does not build the actual parser.
    // Because we want this for all possibilities yet we are never actually going to use this instance of the parser
    // We simply give it a random method with the right signature and tell it we want all possible paths
    dummyParser.addParseTarget(String.class.getMethod("indexOf", String.class), possiblePaths);

    LOG.info("==================================");
    LOG.info("Possible output:");
    for (String path : possiblePaths) {
        LOG.info("{}     {}", path, dummyParser.getCasts(path));
    }
    LOG.info("==================================");
}
 

private void setupFields() throws MissingDissectorsException, InvalidDissectorException, NoSuchMethodException, IOException {
    if (fieldList == null || fieldList.isEmpty()) {
        return; // Nothing to do here
    }
    String firstField = fieldList.get(0);
    if (fieldList.size() == 1 &&
        firstField.toLowerCase().trim().equals(FIELDS)) {
        outputAllPossibleFields = true;
        allPossiblePaths = getParser().getPossiblePaths();
        allPossiblePathsFieldName = firstField;
        Parser<ParsedRecord> newParser = instantiateParser(logformat)
            .addParseTarget(ParsedRecord.class.getMethod("set", String.class, String.class), allPossiblePaths)
            .addTypeRemappings(typeRemappings);
        allCasts = newParser.getAllCasts();
    }
}
 

@Override
protected void initializeNewInstance(Dissector newInstance) throws InvalidDissectorException {
    if (!(newInstance instanceof UserAgentDissector)) {
        String className = "<<<null>>>";
        if (newInstance != null) {
            className = newInstance.getClass().getCanonicalName();
        }
        throw new InvalidDissectorException("The provided instance of the dissector is a " +
            className + " which is not a UserAgentDissector");
    }
    UserAgentDissector newUserAgentDissector = (UserAgentDissector) newInstance;
    newUserAgentDissector.extraResources = new ArrayList<>(extraResources);
    newUserAgentDissector.allPossibleFieldNames = new ArrayList<>(allPossibleFieldNames);
    newUserAgentDissector.requestedFieldNames = new ArrayList<>(requestedFieldNames);
    allPossibleFieldNames.forEach(newUserAgentDissector::ensureMappingsExistForFieldName);
}
 

/**
 * This record reader is given a batch of records (lines) to read. Next acts upon a batch of records.
 *
 * @return Number of records in this batch.
 */
@Override
public int next() {
  try {
    final Text line = lineReader.createValue();

    writer.allocate();
    writer.reset();

    int recordCount = 0;
    while (recordCount < VECTOR_MEMORY_ALLOCATION && lineReader.next(lineNumber, line)) {
      writer.setPosition(recordCount);
      parser.parse(line.toString());
      recordCount++;
    }
    writer.setValueCount(recordCount);

    return recordCount;
  } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
    throw handleAndGenerate("Failure while parsing log record.", e);
  }
}
 

@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String apacheLogLine = tuple.getStringByField(readFieldName);
    try {
        List<Object> out = new ArrayList<>();
        out.add(parser.parse(apacheLogLine));
        collector.emit(out);
    } catch (MissingDissectorsException
            |InvalidDissectorException
            |DissectionFailure e) {
        e.printStackTrace();
    }
}
 

@Override
protected void initializeNewInstance(Dissector newInstance) throws InvalidDissectorException {
    if (newInstance instanceof SimpleDissector) {
        SimpleDissector dissector = (SimpleDissector) newInstance;
        dissector.inputType     = inputType;
        dissector.outputTypes   = outputTypes;
        dissector.outputCasts   = outputCasts;
    }
}
 

public static void main(String[] args) throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    PojoGenerator generator = new PojoGenerator();
    CmdLineParser parser = new CmdLineParser(generator);
    try {
        parser.parseArgument(args);
        generator.run();
    } catch (CmdLineException e) {
        // handling of wrong arguments
        System.err.println(e.getMessage());
        parser.printUsage(System.err);
    }
}
 

@Test(expected = MissingDissectorsException.class)
public void testFailOnMissingDissectors() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecord.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .addDissector(new FooDissector())
        .addDissector(new BarDissector())
        .failOnMissingDissectors()
        .addParseTarget("setStringValue", "SOMETHING:that.is.not.present")
        .addParseTarget("setStringValue", "STRING:string")
        .parse("Doesn't matter");
}
 

@Test(expected = MissingDissectorsException.class)
public void testFailZeroDissectors() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecord.class)
        .setRootType("INPUT")
        .failOnMissingDissectors()
        .addParseTarget("setStringValue", "SOMETHING:that.is.not.present")
        .addParseTarget("setStringValue", "STRING:string")
        .parse("Doesn't matter");
}
 

/**
 * Parse the value and return a new instance of RECORD.
 * For this method to work the RECORD class may NOT be an inner class.
 */
public RECORD parse(final String value)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();
    final Parsable<RECORD> parsable = createParsable();
    if (parsable == null) {
        return null;
    }
    parsable.setRootDissection(rootType, value);
    return parse(parsable).getRecord();
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectString("ANY:any",       "42")
        .expectString("STRING:string", "FortyTwo")
        .expectString("INT:int",       "42")
        .expectString("LONG:long",     "42")
        .expectString("FLOAT:float",   "42.0")
        .expectString("DOUBLE:double", "42.0");
}
 

/**
 * Parse the value and call all configured setters in the provided instance of RECORD.
 */
public RECORD parse(final RECORD record, final String value)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();
    final Parsable<RECORD> parsable = createParsable(record);
    parsable.setRootDissection(rootType, value);
    return parse(parsable).getRecord();
}
 

Parsable<RECORD> parse(final Parsable<RECORD> parsable)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();

    if (!assembled) {
        return null;
    }

    // Values look like "TYPE:foo.bar"
    Set<ParsedField> toBeParsed = new HashSet<>(parsable.getToBeParsed());

    while (!toBeParsed.isEmpty()) {
        for (ParsedField fieldThatNeedsToBeParsed : toBeParsed) {
            parsable.setAsParsed(fieldThatNeedsToBeParsed);
            Set<DissectorPhase> dissectorSet = compiledDissectors.get(fieldThatNeedsToBeParsed.getId());
            if (dissectorSet != null) {
                for (DissectorPhase dissector : dissectorSet) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Dissect {} with {}", fieldThatNeedsToBeParsed, dissector.instance.getClass().getName());
                    }
                    dissector.instance.dissect(parsable, fieldThatNeedsToBeParsed.getName());
                }
            } else {
                LOG.trace("NO DISSECTORS FOR \"{}\"", fieldThatNeedsToBeParsed);
            }
        }
        toBeParsed.clear();
        toBeParsed.addAll(parsable.getToBeParsed());
    }
    return parsable;
}
 

@Override
public void prepareForRun() throws InvalidDissectorException {
    if (dissectors.isEmpty()) {
        throw new InvalidDissectorException("Cannot run without logformats");
    }

    for (Dissector dissector : dissectors) {
        if (!INPUT_TYPE.equals(dissector.getInputType())) {
            throw new InvalidDissectorException("All dissectors controlled by " + this.getClass().getCanonicalName()
                + " MUST have \"" + INPUT_TYPE + "\" as their inputtype.");
        }
        dissector.prepareForRun();
    }
}
 

@Test(expected=InvalidDissectorException.class)
public void testBrokenDissector() throws Exception {
    Parser<TestRecord> parser = new TestParser<>(TestRecord.class);
    Dissector dissector = new BrokenTestDissector();
    parser.setRootType(dissector.getInputType());
    parser.addParseTarget(TestRecord.class.getMethod("setValue8", String.class, String.class), "FOO:bar");
    parser.addDissector(dissector);
    parser.parse("Something");
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new EmptyValuesDissector())
        .parse("Doesn't matter")

        .noString("ANY:any")
        .noString("STRING:string")
        .noString("INT:int")
        .noString("LONG:long")
        .noString("FLOAT:float")
        .noString("DOUBLE:double");
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new EmptyValuesDissector())
        .parse("Doesn't matter")

        .noLong("ANY:any")
        .noLong("INT:int")
        .noLong("LONG:long");
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new EmptyValuesDissector())
        .parse("Doesn't matter")

        .noDouble("ANY:any")
        .noDouble("FLOAT:float")
        .noDouble("DOUBLE:double");
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectString("ANY:any",       "42")
        .expectString("STRING:string", "FortyTwo")
        .expectString("INT:int",       "42")
        .expectString("LONG:long",     "42")
        .expectString("FLOAT:float",   "42.0")
        .expectString("DOUBLE:double", "42.0");
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectLong("ANY:any",    42L)
        .expectLong("INT:int",    42L)
        .expectLong("LONG:long",  42L);
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectDouble("ANY:any",       42D)
        .expectDouble("FLOAT:float",   42D)
        .expectDouble("DOUBLE:double", 42D);
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noString("ANY:any")
        .noString("STRING:string")
        .noString("INT:int")
        .noString("LONG:long")
        .noString("FLOAT:float")
        .noString("DOUBLE:double");
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noLong("ANY:any")
        .noLong("INT:int")
        .noLong("LONG:long");
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noDouble("ANY:any")
        .noDouble("FLOAT:float")
        .noDouble("DOUBLE:double");
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectDouble("ANY:any",       42D)
        .expectDouble("FLOAT:float",   42D)
        .expectDouble("DOUBLE:double", 42D);
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectLong("ANY:any",    42L)
        .expectLong("INT:int",    42L)
        .expectLong("LONG:long",  42L);
}