nl.basjes.parse.core.exceptions.DissectionFailure Java Examples

/**
 * This record reader is given a batch of records (lines) to read. Next acts upon a batch of records.
 *
 * @return Number of records in this batch.
 */
@Override
public int next() {
  try {
    final Text line = lineReader.createValue();

    writer.allocate();
    writer.reset();

    int recordCount = 0;
    while (recordCount < VECTOR_MEMORY_ALLOCATION && lineReader.next(lineNumber, line)) {
      writer.setPosition(recordCount);
      parser.parse(line.toString());
      recordCount++;
    }
    writer.setValueCount(recordCount);

    return recordCount;
  } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
    throw handleAndGenerate("Failure while parsing log record.", e);
  }
}
 

@Override
public void dissect(Parsable<?> parsable, final String inputname) throws DissectionFailure {
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_set_null", (String) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_set_string", "42");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_set_string_null", (String) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_set_string", "42");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_set_longclass_null", (Long) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_set_longclass", Long.valueOf(42));
    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_set_longprimitive", 42L);

    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_string_null", (String) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_string", "42");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_longclass_null", (Long) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_longclass", Long.valueOf(42));
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_longprimitive", 42L);

    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_doubleclass_null", (Double) null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_doubleclass", Double.valueOf(42));
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_set_doubleprimitive", 42D);

}
 

@Override
public void dissect(final Parsable<?> parsable, final String inputname) throws DissectionFailure {
    final ParsedField field = parsable.getParsableField(INPUT_TYPE, inputname);

    String fieldValue = field.getValue().getString();
    if (fieldValue == null || fieldValue.isEmpty()) {
        return; // Nothing to do here
    }

    InetAddress ipAddress;
    try {
        ipAddress = InetAddress.getByName(fieldValue);
    } catch (UnknownHostException e) {
        return;
    }

    dissect(parsable, inputname, ipAddress);
}
 

@Override
public void dissect(Parsable<?> parsable, final String inputname) throws DissectionFailure {
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_good", "123");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_bad", "Something");
    parsable.addDissection(inputname, "OUTPUT_TYPE", "long_good", "123");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_bad", "Something");
    parsable.addDissection(inputname, "OUTPUT_TYPE", "double_good", "123");

    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_long_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_double_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "multi_null", (String)null);
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_long_good", "123");
    parsable.addDissection(inputname, "OUTPUT_TYPE", "string_double_good", "123");
    parsable.addDissection(inputname, "OUTPUT_TYPE", "multi_good", "123");

}
 

@Test
public void testSetTypeRemapping() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    try{
        new Parser<>(TestRecord.class)
            .setRootType("INPUT")
            .addDissector(new NormalValuesDissector())
            .addTypeRemapping("string", "STRINGXX")
            .addParseTarget("setStringValue", "STRINGXX:string")
            // Should wipe previous
            .setTypeRemappings(Collections.singletonMap("string", Collections.singleton("STRINGYY")))
            .addParseTarget("setStringValue", "STRINGYY:string")
            .parse("Doesn't matter");

        fail("We should get an exception because the mapping to STRINGXX:string wass removed.");
    } catch (MissingDissectorsException mde) {
        assertTrue(mde.getMessage().contains("STRINGXX:string"));
    }
}
 

@Override
public void dissect(final Parsable<?> parsable, final String inputname) throws DissectionFailure {
    final ParsedField field = parsable.getParsableField(INPUT_TYPE, inputname);

    final String fieldValue = field.getValue().getString();
    if (fieldValue == null || fieldValue.isEmpty() || "-".equals(fieldValue)){
        return; // Nothing to do here
    }

    String[] protocol = fieldValue.split("/", 2);

    if (protocol.length == 2) {
        outputDissection(parsable, inputname, "HTTP.PROTOCOL", "", protocol[0]);
        outputDissection(parsable, inputname, "HTTP.PROTOCOL.VERSION", "version", protocol[1]);
        return;
    }

    // In the scenario that the actual URI is too long the last part ("HTTP/1.1") may have been cut off by the
    // Apache HTTPD webserver. To still be able to parse these we try that pattern too

    parsable.addDissection(inputname, "HTTP.PROTOCOL", "", (String) null);
    parsable.addDissection(inputname, "HTTP.PROTOCOL.VERSION", "version", (String) null);
}
 

@Override
public void dissect(Parsable<?> parsable, String inputname, Value value) throws DissectionFailure {
    parsable.addDissection(inputname, "ANY",    "barany",    "42");
    parsable.addDissection(inputname, "STRING", "barstring", "42");
    parsable.addDissection(inputname, "INT",    "barint",    42);
    parsable.addDissection(inputname, "LONG",   "barlong",   42L);
    parsable.addDissection(inputname, "FLOAT",  "barfloat",  42F);
    parsable.addDissection(inputname, "DOUBLE", "bardouble", 42D);
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noString("ANY:any")
        .noString("STRING:string")
        .noString("INT:int")
        .noString("LONG:long")
        .noString("FLOAT:float")
        .noString("DOUBLE:double");
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectDouble("ANY:any",       42D)
        .expectDouble("FLOAT:float",   42D)
        .expectDouble("DOUBLE:double", 42D);
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectLong("ANY:any",    42L)
        .expectLong("INT:int",    42L)
        .expectLong("LONG:long",  42L);
}
 

@Override
public void dissect(Parsable<?> parsable, String inputname, Value value) throws DissectionFailure {
    LOG.info("Outputting \"NULL\" values");
    parsable
        .addDissection(inputname, "ANY",    "any",    (String) null)
        .addDissection(inputname, "STRING", "string", (String) null)
        .addDissection(inputname, "INT",    "int",    (Long)   null)
        .addDissection(inputname, "LONG",   "long",   (Long)   null)
        .addDissection(inputname, "FLOAT",  "float",  (Double) null)
        .addDissection(inputname, "DOUBLE", "double", (Double) null);
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noDouble("ANY:any")
        .noDouble("FLOAT:float")
        .noDouble("DOUBLE:double");
}
 

@Override
public void dissect(Parsable<?> parsable, String inputname, Value value) throws DissectionFailure {
    LOG.info("Outputting \"EMPTY\" values");
    parsable
        .addDissection(inputname, "ANY",    "any",    "")
        .addDissection(inputname, "STRING", "string", "")
        .addDissection(inputname, "INT",    "int",    "")
        .addDissection(inputname, "LONG",   "long",   "")
        .addDissection(inputname, "FLOAT",  "float",  "")
        .addDissection(inputname, "DOUBLE", "double", "");
}
 

@Test
public void testString() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordString.class)
        .setRootType("INPUT")
        .addDissector(new EmptyValuesDissector())
        .parse("Doesn't matter")

        .noString("ANY:any")
        .noString("STRING:string")
        .noString("INT:int")
        .noString("LONG:long")
        .noString("FLOAT:float")
        .noString("DOUBLE:double");
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new EmptyValuesDissector())
        .parse("Doesn't matter")

        .noDouble("ANY:any")
        .noDouble("FLOAT:float")
        .noDouble("DOUBLE:double");
}
 

Parsable<RECORD> parse(final Parsable<RECORD> parsable)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();

    if (!assembled) {
        return null;
    }

    // Values look like "TYPE:foo.bar"
    Set<ParsedField> toBeParsed = new HashSet<>(parsable.getToBeParsed());

    while (!toBeParsed.isEmpty()) {
        for (ParsedField fieldThatNeedsToBeParsed : toBeParsed) {
            parsable.setAsParsed(fieldThatNeedsToBeParsed);
            Set<DissectorPhase> dissectorSet = compiledDissectors.get(fieldThatNeedsToBeParsed.getId());
            if (dissectorSet != null) {
                for (DissectorPhase dissector : dissectorSet) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Dissect {} with {}", fieldThatNeedsToBeParsed, dissector.instance.getClass().getName());
                    }
                    dissector.instance.dissect(parsable, fieldThatNeedsToBeParsed.getName());
                }
            } else {
                LOG.trace("NO DISSECTORS FOR \"{}\"", fieldThatNeedsToBeParsed);
            }
        }
        toBeParsed.clear();
        toBeParsed.addAll(parsable.getToBeParsed());
    }
    return parsable;
}
 

@Test
public void testDouble() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordDouble.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .parse("Doesn't matter")

        .expectDouble("ANY:any",       42D)
        .expectDouble("FLOAT:float",   42D)
        .expectDouble("DOUBLE:double", 42D);
}
 

protected void extractIspFields(final Parsable<?> parsable, final String inputname, IspResponse response) throws DissectionFailure {
    if (wantIspName) {
        parsable.addDissection(inputname, "STRING", "isp.name", response.getIsp());
    }
    if (wantIspOrganization) {
        parsable.addDissection(inputname, "STRING", "isp.organization", response.getOrganization());
    }
}
 

@Test
public void testLong() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecordLong.class)
        .setRootType("INPUT")
        .addDissector(new NullValuesDissector())
        .parse("Doesn't matter")

        .noLong("ANY:any")
        .noLong("INT:int")
        .noLong("LONG:long");
}
 

/**
 * Parse the value and call all configured setters in the provided instance of RECORD.
 */
public RECORD parse(final RECORD record, final String value)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();
    final Parsable<RECORD> parsable = createParsable(record);
    parsable.setRootDissection(rootType, value);
    return parse(parsable).getRecord();
}
 

/**
 * Parse the value and return a new instance of RECORD.
 * For this method to work the RECORD class may NOT be an inner class.
 */
public RECORD parse(final String value)
    throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
    assembleDissectors();
    final Parsable<RECORD> parsable = createParsable();
    if (parsable == null) {
        return null;
    }
    parsable.setRootDissection(rootType, value);
    return parse(parsable).getRecord();
}
 

@Override
public final void dissect(Parsable<?> parsable, String inputname) throws DissectionFailure {
    final ParsedField field = parsable.getParsableField(getInputType(), inputname);
    Value value = field.getValue();
    if (value == null) {
        return; // Nothing to do here
    }
    dissect(parsable, inputname, value);
}
 

@Test
public void testAddTypeRemappings() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecord.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .addTypeRemapping("string", "STRINGXX")
        .addParseTarget("setStringValue", "STRINGXX:string")
        .addTypeRemappings(Collections.singletonMap("string", Collections.singleton("STRINGYY")))
        .addParseTarget("setStringValue", "STRINGYY:string")
        .parse("Doesn't matter")
        .expectString("STRINGXX:string", "FortyTwo")
        .expectString("STRINGYY:string", "FortyTwo");
}
 

/** Store a newly parsed value in the result set */
public Parsable<RECORD> addDissection(final String base, final String type, final String name, final Integer value) throws DissectionFailure {
    LOG.debug("Got new (Integer) dissection: base={}; type={}; name=\"{}\"", base, type, name);
    if (value == null) {
        return addDissection(base, type, name, new Value((Long)null), false);
    }
    return addDissection(base, type, name, new Value((long)value), false);
}
 

private void validateLine2(Parser<TestRecord> parser) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    TestRecord record = new TestRecord();
    parser.parse(record, LINE_2);
    Map<String, String> results = record.getResults();

    assertEquals("127.0.0.2", results.get("IP:connection.client.host"));
    assertEquals("31/Dec/2012:23:49:42 +0100", results.get("TIME.STAMP:request.receive.time"));
    assertEquals("/foo", results.get("HTTP.URI:request.firstline.uri"));
    assertEquals("404", results.get("STRING:request.status.last"));
    assertEquals(null, results.get("BYTESCLF:response.body.bytes"));
    assertEquals(null, results.get("HTTP.URI:request.referer"));
    assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
            results.get("HTTP.USERAGENT:request.user-agent"));
}
 

private void validateLine1(Parser<TestRecord> parser) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    TestRecord record = new TestRecord();
    parser.parse(record, LINE_1);
    Map<String, String> results = record.getResults();

    assertEquals("127.0.0.1", results.get("IP:connection.client.host"));
    assertEquals("31/Dec/2012:23:49:41 +0100", results.get("TIME.STAMP:request.receive.time"));
    assertEquals("/foo", results.get("HTTP.URI:request.firstline.uri"));
    assertEquals("200", results.get("STRING:request.status.last"));
    assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
    assertEquals("http://localhost/index.php?mies=wim", results.get("HTTP.URI:request.referer"));
    assertEquals(null, results.get("HTTP.USERAGENT:request.user-agent"));
}
 

@Override
public void dissect(Parsable<?> parsable, String inputname, Value value) throws DissectionFailure {
    Matcher matcher = binaryIPPattern.matcher(value.getString());
    if (matcher.matches()) {
        String ip =
            String.valueOf(Utils.hexCharsToByte(matcher.group(1))) + '.' +
            String.valueOf(Utils.hexCharsToByte(matcher.group(2))) + '.' +
            String.valueOf(Utils.hexCharsToByte(matcher.group(3))) + '.' +
            String.valueOf(Utils.hexCharsToByte(matcher.group(4)));
        parsable.addDissection(inputname, "IP", "", ip);
    }
}
 

@Test
public void testFieldCleanup() throws NoSuchMethodException, InvalidDissectorException, MissingDissectorsException, DissectionFailure {
    new Parser<>(TestRecord.class)
        .setRootType("INPUT")
        .addDissector(new NormalValuesDissector())
        .addParseTarget("setStringValue", "stRinG:stRinG")
        .parse("Doesn't matter")
        .expectString("STRING:string", "FortyTwo");
}
 

public void dissect(final Parsable<?> parsable, final String inputname, final InetAddress ipAddress) throws DissectionFailure {
    // City is the 'Country' + more details.
    CityResponse response;
    try {
        response = reader.city(ipAddress);
    } catch (IOException | GeoIp2Exception e) {
        return;
    }

    extractCountryFields(parsable, inputname, response);
    extractCityFields(parsable, inputname, response);
}
 

private TestRecord parse(String inputValue) {
    TestRecord testRecord = new TestRecord();
    if (verbose) {
        testRecord.setVerbose();
    }
    try {
        return parser.parse(testRecord, inputValue);
    } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException e) {
        fail(e.toString());
    }
    return testRecord; // This will never happen
}