nl.basjes.parse.httpdlog.HttpdLoglineParser Java Examples
The following examples show how to use
nl.basjes.parse.httpdlog.HttpdLoglineParser.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: HttpdParser.java From Bats with Apache License 2.0 | 6 votes |
|
public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat,
final String timestampFormat, final Map<String, String> fieldMapping)
throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
Preconditions.checkArgument(logFormat != null && !logFormat.trim().isEmpty(), "logFormat cannot be null or empty");
this.record = new HttpdLogRecord(managedBuffer, timestampFormat);
this.parser = new HttpdLoglineParser<>(HttpdLogRecord.class, logFormat, timestampFormat);
setupParser(mapWriter, logFormat, fieldMapping);
if (timestampFormat != null && !timestampFormat.trim().isEmpty()) {
LOG.info("Custom timestamp format has been specified. This is an informational note only as custom timestamps is rather unusual.");
}
if (logFormat.contains("\n")) {
LOG.info("Specified logformat is a multiline log format: {}", logFormat);
}
}
Example #2
| Source File: Main.java From logparser with Apache License 2.0 | 6 votes |
|
private void printAllPossibles(String logformat) throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
// To figure out what values we CAN get from this line we instantiate the parser with a dummy class
// that does not have ANY @Field annotations.
Parser<Object> dummyParser= new HttpdLoglineParser<>(Object.class, logformat);
List<String> possiblePaths;
possiblePaths = dummyParser.getPossiblePaths();
// If you want to call 'getCasts' then the actual parser needs to be constructed.
// Simply calling getPossiblePaths does not build the actual parser.
// Because we want this for all possibilities yet we are never actually going to use this instance of the parser
// We simply give it a random method with the right signature and tell it we want all possible paths
dummyParser.addParseTarget(String.class.getMethod("indexOf", String.class), possiblePaths);
LOG.info("==================================");
LOG.info("Possible output:");
for (String path : possiblePaths) {
LOG.info("{} {}", path, dummyParser.getCasts(path));
}
LOG.info("==================================");
}
Example #3
| Source File: TestCase.java From logparser with Apache License 2.0 | 6 votes |
|
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());
parser.addDissector(new ScreenResolutionDissector());
parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");
parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time");
parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo");
parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s");
parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width");
parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height");
parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla");
parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui");
parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent");
return parser;
}
Example #4
| Source File: NginxUpstreamTest.java From logparser with Apache License 2.0 | 6 votes |
|
@Test
public void testFullLine() {
String logFormat = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" $request_time $upstream_response_time $pipe";
String logLine = "10.77.150.123 - - [15/Dec/2018:19:27:57 -0500] \"GET /25.chunk.js HTTP/1.1\" 200 84210 \"https://api.demo.com/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36\" \"-\" 0.002 0.002 .";
DissectorTester.create()
.verbose()
.withParser(new HttpdLoglineParser<>(TestRecord.class, logFormat))
.withInput(logLine)
.expect("SECOND_MILLIS:nginxmodule.upstream.response.time.0.value", "0.002")
.expect("SECOND_MILLIS:nginxmodule.upstream.response.time.0.redirected", "0.002")
.expect("MICROSECONDS:nginxmodule.upstream.response.time.0.value", "2000")
.expect("MICROSECONDS:nginxmodule.upstream.response.time.0.redirected", "2000")
.checkExpectations()
// .printPossible();
.printAllPossibleValues();
}
Example #5
| Source File: TestDissectUserAgent.java From yauaa with Apache License 2.0 | 5 votes |
|
@Test
public void testExtractUrlFields() {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, "%t \"%{User-agent}i\"");
parser.addDissector(new UserAgentDissector());
String testUri = "https://yauaa.basjes.nl:8080/something.html?aap=noot&mies=wim#zus";
String testUserAgent =
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) " +
"AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/41.0.2272.96 " +
"Mobile Safari/537.36" +
"(" + testUri + ")";
String testLogLine = "[10/Aug/2012:23:55:11 +0200] \""+testUserAgent+"\"";
DissectorTester
.create()
.withParser(parser)
.withInput(testLogLine)
// Did we get the field
.expect("HTTP.USERAGENT:request.user-agent", testUserAgent)
// Basic dissections
.expect("STRING:request.user-agent.device_class", "Phone")
.expect("STRING:request.user-agent.agent_name_version", "Chrome 41.0.2272.96")
.expect("HTTP.URI:request.user-agent.agent_information_url", testUri)
// Further extractions from the URI we found
.expect("HTTP.PROTOCOL:request.user-agent.agent_information_url.protocol", "https")
.expect("HTTP.HOST:request.user-agent.agent_information_url.host", "yauaa.basjes.nl")
.expect("HTTP.PORT:request.user-agent.agent_information_url.port", "8080")
.expect("HTTP.PATH:request.user-agent.agent_information_url.path", "/something.html")
.expect("HTTP.QUERYSTRING:request.user-agent.agent_information_url.query", "&aap=noot&mies=wim")
.expect("STRING:request.user-agent.agent_information_url.query.aap", "noot")
.expect("STRING:request.user-agent.agent_information_url.query.mies", "wim")
.expect("HTTP.REF:request.user-agent.agent_information_url.ref", "zus")
.checkExpectations();
}
Example #6
| Source File: TestCase.java From logparser with Apache License 2.0 | 5 votes |
|
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());
parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector());
parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");
parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time");
parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo");
parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s");
parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width");
parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height");
parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla");
parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui");
parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent");
parser.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB));
parser.addParseTarget("setAsnNumber", "ASN:connection.client.host.asn.number");
parser.addParseTarget("setAsnOrganization", "STRING:connection.client.host.asn.organization");
parser.addParseTarget("setIspName", "STRING:connection.client.host.isp.name");
parser.addParseTarget("setIspOrganization", "STRING:connection.client.host.isp.organization");
parser.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
parser.addParseTarget("setContinentName", "STRING:connection.client.host.continent.name");
parser.addParseTarget("setContinentCode", "STRING:connection.client.host.continent.code");
parser.addParseTarget("setCountryName", "STRING:connection.client.host.country.name");
parser.addParseTarget("setCountryIso", "STRING:connection.client.host.country.iso");
parser.addParseTarget("setSubdivisionName", "STRING:connection.client.host.subdivision.name");
parser.addParseTarget("setSubdivisionIso", "STRING:connection.client.host.subdivision.iso");
parser.addParseTarget("setCityName", "STRING:connection.client.host.city.name");
parser.addParseTarget("setPostalCode", "STRING:connection.client.host.postal.code");
parser.addParseTarget("setLocationLatitude", "STRING:connection.client.host.location.latitude");
parser.addParseTarget("setLocationLongitude", "STRING:connection.client.host.location.longitude");
return parser;
}
Example #7
| Source File: TestParserDoFnAvro.java From logparser with Apache License 2.0 | 5 votes |
|
@Setup
public void setup() {
parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat())
.addDissector(new ScreenResolutionDissector())
.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION")
.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB))
.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
}
Example #8
| Source File: Main.java From logparser with Apache License 2.0 | 5 votes |
|
private void run() throws InvalidDissectorException, MissingDissectorsException, NoSuchMethodException, DissectionFailure {
// This format and logline originate from here:
// https://stackoverflow.com/questions/20349184/java-parse-log-file
String logformat = "%t %u [%D %h %{True-Client-IP}i %{UNIQUE_ID}e %r] %{Cookie}i %s \"%{User-Agent}i\" \"%{host}i\" %l %b %{Referer}i";
String logline = "[02/Dec/2013:14:10:30 -0000] - [52075 10.102.4.254 177.43.52.210 UpyU1gpmBAwAACfd5W0AAAAW GET /SS14-VTam-ny_019.j" +
"pg.rendition.zoomable.jpg HTTP/1.1] hsfirstvisit=http%3A%2F%2Fwww.domain.com%2Fen-us||1372268254000; _opt_vi_3FNG8DZU=F870" +
"DCFD-CBA4-4B6E-BB58-4605A78EE71A; __ptca=145721067.0aDxsZlIuM48.1372279055.1379945057.1379950362.9; __ptv_62vY4e=0aDxsZlIu" +
"M48; __pti_62vY4e=0aDxsZlIuM48; __ptcz=145721067.1372279055.1.0.ptmcsr=(direct)|ptmcmd=(none)|ptmccn=(direct); __hstc=1457" +
"21067.b86362bb7a1d257bfa2d1fb77e128a85.1372268254968.1379934256743.1379939561848.9; hubspotutk=b86362bb7a1d257bfa2d1fb77e1" +
"28a85; USER_GROUP=julinho%3Afalse; has_js=1; WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1385997780893:ss=1385997780893" +
"; dtCookie=1F2E0E1037589799D8D503EB8CFA12A1|_default|1; RM=julinho%3A5248423ad3fe062f06c54915e6cde5cb45147977; wcid=UpyKsQ" +
"pmBAwAABURyNoAAAAS%3A35d8227ba1e8a9a9cebaaf8d019a74777c32b4c8; Carte::KerberosLexicon_getWGSN=82ae3dcd1b956288c3c86bdbed6e" +
"bcc0fd040e1e; UserData=Username%3AJULINHO%3AHomepage%3A1%3AReReg%3A0%3ATrialist%3A0%3ALanguage%3Aen%3ACcode%3Abr%3AForceRe" +
"Reg%3A0; UserID=1356673%3A12345%3A1234567890%3A123%3Accode%3Abr; USER_DATA=1356673%3Ajulinho%3AJulio+Jose%3Ada+Silva%3Ajul" +
"inho%40tecnoblu.com.br%3A0%3A1%3Aen%3Abr%3A%3AWGSN%3A1385990833.81925%3A82ae3dcd1b956288c3c86bdbed6ebcc0fd040e1e; MODE=FON" +
"TIS; SECTION=%2Fcontent%2Fsection%2Fhome.html; edge_auth=ip%3D177.43.52.210~expires%3D1385994522~access%3D%2Fapps%2F%2A%21" +
"%2Fbin%2F%2A%21%2Fcontent%2F%2A%21%2Fetc%2F%2A%21%2Fhome%2F%2A%21%2Flibs%2F%2A%21%2Freport%2F%2A%21%2Fsection%2F%2A%21%2Fw" +
"gsn%2F%2A~md5%3D90e73ee10161c1afacab12c6ea30b4ef; __utma=94539802.1793276213.1372268248.1385572390.1385990581.16; __utmb=9" +
"4539802.52.9.1385991739764; __utmc=94539802; __utmz=94539802.1372268248.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none);" +
" WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1386000374581:ss=1386000374581; dtPC=-; NSC_wtfswfs_xfcgbsn40-41=ffffffff0" +
"96e1a1d45525d5f4f58455e445a4a423660; akamai-edge=5ac6e5b3d0bbe2ea771bb2916d8bab34ea222a6a 200 \"Mozilla/5.0 (Windows NT 6." +
"2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36\" \"www.domain.com\" - 463952 http://ww" +
"w.domain.com/content/report/shows/New_York/KSHK/trip/s_s_14_ny_ww/sheers.html";
printAllPossibles(logformat);
Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, logformat);
MyRecord record = new MyRecord();
LOG.info("==================================================================================");
parser.parse(record, logline);
LOG.info(record.toString());
LOG.info("==================================================================================");
}
Example #9
| Source File: TestCase.java From logparser with Apache License 2.0 | 5 votes |
|
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());
parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector());
parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");
parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time");
parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo");
parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s");
parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width");
parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height");
parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla");
parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui");
parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent");
parser.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB));
parser.addParseTarget("setAsnNumber", "ASN:connection.client.host.asn.number");
parser.addParseTarget("setAsnOrganization", "STRING:connection.client.host.asn.organization");
parser.addParseTarget("setIspName", "STRING:connection.client.host.isp.name");
parser.addParseTarget("setIspOrganization", "STRING:connection.client.host.isp.organization");
parser.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
parser.addParseTarget("setContinentName", "STRING:connection.client.host.continent.name");
parser.addParseTarget("setContinentCode", "STRING:connection.client.host.continent.code");
parser.addParseTarget("setCountryName", "STRING:connection.client.host.country.name");
parser.addParseTarget("setCountryIso", "STRING:connection.client.host.country.iso");
parser.addParseTarget("setSubdivisionName", "STRING:connection.client.host.subdivision.name");
parser.addParseTarget("setSubdivisionIso", "STRING:connection.client.host.subdivision.iso");
parser.addParseTarget("setCityName", "STRING:connection.client.host.city.name");
parser.addParseTarget("setPostalCode", "STRING:connection.client.host.postal.code");
parser.addParseTarget("setLocationLatitude", "STRING:connection.client.host.location.latitude");
parser.addParseTarget("setLocationLongitude", "STRING:connection.client.host.location.longitude");
return parser;
}
Example #10
| Source File: TestParserMapFunctionAvroClass.java From logparser with Apache License 2.0 | 5 votes |
|
@Override
public void open(Configuration parameters) {
parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat())
.addDissector(new ScreenResolutionDissector())
.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION")
.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB))
.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
}
Example #11
| Source File: TestParserMapFunctionAvroInline.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void testInlineDefinitionAvro() throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<String> input = env.fromElements(TestCase.getInputLine());
DataSet<Click> filledTestRecords = input
.map(new RichMapFunction<String, Click>() {
private Parser<ClickSetter> parser;
@Override
public void open(org.apache.flink.configuration.Configuration parameters) {
parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat())
.addDissector(new ScreenResolutionDissector())
.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION")
.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB))
.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
}
@Override
public Click map(String line) throws Exception {
return parser.parse(line).build();
}
}).name("Extract Elements from logline");
filledTestRecords.print();
List<Click> result = filledTestRecords.collect();
assertEquals(1, result.size());
assertEquals(ExpectedClick.create(), result.get(0));
}
Example #12
| Source File: NginxAllFieldsTest.java From logparser with Apache License 2.0 | 5 votes |
|
public void checkVariable(String variableName) {
List<String> allPossible =
DissectorTester.create()
.verbose()
.withParser(new HttpdLoglineParser<>(TestRecord.class, "# " + variableName + " #"))
.getPossible();
allPossible.forEach(p -> assertFalse(p, p.startsWith("UNKNOWN_NGINX_VARIABLE")));
}
Example #13
| Source File: ApacheHttpdLogfileInputFormat.java From logparser with Apache License 2.0 | 4 votes |
|
public static List<String> listPossibleFields(String logformat, Map<String, Set<String>> typeRemappings, List<Dissector> additionalDissectors) {
HttpdLoglineParser<ParsedRecord> parser = new HttpdLoglineParser<>(ParsedRecord.class, logformat);
parser.setTypeRemappings(typeRemappings);
parser.addDissectors(additionalDissectors);
return parser.getPossiblePaths();
}
Example #14
| Source File: ApacheHttpdLogfileRecordReader.java From logparser with Apache License 2.0 | 4 votes |
|
protected Parser<ParsedRecord> instantiateParser(String logFormat) {
return new HttpdLoglineParser<>(ParsedRecord.class, logFormat)
.setTypeRemappings(typeRemappings)
.addDissectors(additionalDissectors);
}
Example #15
| Source File: NginxUpstreamTest.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void testBasicLogFormat() {
// From: http://articles.slicehost.com/2010/8/27/customizing-nginx-web-logs
String logFormat = "\"$upstream_addr\" \"$upstream_bytes_received\"";
String logLine = "\"192.168.1.1:80, 192.168.1.2:80, unix:/tmp/sock : 192.168.10.1:80, 192.168.10.2:80\" \"1, 2, 3 : 4, 5\"";
DissectorTester.create()
.verbose()
.withParser(new HttpdLoglineParser<>(TestRecord.class, logFormat))
.withInput(logLine)
.expect("UPSTREAM_ADDR_LIST:nginxmodule.upstream.addr",
"192.168.1.1:80, 192.168.1.2:80, unix:/tmp/sock : 192.168.10.1:80, 192.168.10.2:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.0.value", "192.168.1.1:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.0.redirected", "192.168.1.1:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.1.value", "192.168.1.2:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.1.redirected", "192.168.1.2:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.2.value", "unix:/tmp/sock")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.2.redirected", "192.168.10.1:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.3.value", "192.168.10.2:80")
.expect("UPSTREAM_ADDR:nginxmodule.upstream.addr.3.redirected", "192.168.10.2:80")
.expectAbsentString("UPSTREAM_ADDR:nginxmodule.upstream.addr.4.value")
.expectAbsentString("UPSTREAM_ADDR:nginxmodule.upstream.addr.4.redirected")
.expect("UPSTREAM_BYTES_LIST:nginxmodule.upstream.bytes.received", "1, 2, 3 : 4, 5")
.expect("BYTES:nginxmodule.upstream.bytes.received.0.value", "1")
.expect("BYTES:nginxmodule.upstream.bytes.received.0.redirected", "1")
.expect("BYTES:nginxmodule.upstream.bytes.received.1.value", "2")
.expect("BYTES:nginxmodule.upstream.bytes.received.1.redirected", "2")
.expect("BYTES:nginxmodule.upstream.bytes.received.2.value", "3")
.expect("BYTES:nginxmodule.upstream.bytes.received.2.redirected", "4")
.expect("BYTES:nginxmodule.upstream.bytes.received.3.value", "5")
.expect("BYTES:nginxmodule.upstream.bytes.received.3.redirected", "5")
.expectAbsentString("BYTES:nginxmodule.upstream.bytes.received.4.value")
.expectAbsentString("BYTES:nginxmodule.upstream.bytes.received.4.redirected")
.checkExpectations()
.printPossible()
.printAllPossibleValues();
}
Example #16
| Source File: TestGeoIPDissectors.java From logparser with Apache License 2.0 | 4 votes |
|
DissectorTester createTester(Dissector dissector) {
return DissectorTester.create()
.withParser(new HttpdLoglineParser<>(TestRecord.class, "%h"))
.withDissector(dissector)
.withPathPrefix("connection.client.host.");
}
Example #17
| Source File: HttpdParser.java From Bats with Apache License 2.0 | 4 votes |
|
private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping)
throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
/**
* If the user has selected fields, then we will use them to configure the parser because this would be the most
* efficient way to parse the log.
*/
final Map<String, String> requestedPaths;
final List<String> allParserPaths = parser.getPossiblePaths();
if (fieldMapping != null && !fieldMapping.isEmpty()) {
LOG.debug("Using fields defined by user.");
requestedPaths = fieldMapping;
} else {
/**
* Use all possible paths that the parser has determined from the specified log format.
*/
LOG.debug("No fields defined by user, defaulting to all possible fields.");
requestedPaths = Maps.newHashMap();
for (final String parserPath : allParserPaths) {
requestedPaths.put(drillFormattedFieldName(parserPath), parserPath);
}
}
/**
* By adding the parse target to the dummy instance we activate it for use. Which we can then use to find out which
* paths cast to which native data types. After we are done figuring this information out, we throw this away
* because this will be the slowest parsing path possible for the specified format.
*/
Parser<Object> dummy = new HttpdLoglineParser<>(Object.class, logFormat);
dummy.addParseTarget(String.class.getMethod("indexOf", String.class), allParserPaths);
for (final Map.Entry<String, String> entry : requestedPaths.entrySet()) {
final EnumSet<Casts> casts;
/**
* Check the field specified by the user to see if it is supposed to be remapped.
*/
if (entry.getValue().startsWith(REMAPPING_FLAG)) {
/**
* Because this field is being remapped we need to replace the field name that the parser uses.
*/
entry.setValue(entry.getValue().substring(REMAPPING_FLAG.length()));
final String[] pieces = entry.getValue().split(":");
addTypeRemapping(parser, pieces[1], pieces[0]);
casts = Casts.STRING_ONLY;
} else {
casts = dummy.getCasts(entry.getValue());
}
LOG.debug("Setting up drill field: {}, parser field: {}, which casts as: {}", entry.getKey(), entry.getValue(), casts);
record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey());
}
}
