org.elasticsearch.ingest.IngestDocument Java Examples

@Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
    String content = ingestDocument.getFieldValue(field, String.class);

    if (Strings.hasLength(content)) {
        String[] values;
        synchronized (parser) {
            values = parser.parseLine(content);
        }
        if (values.length != this.columns.size()) {
            // TODO should be error?
            throw new IllegalArgumentException("field[" + this.field + "] size ["
                + values.length + "] doesn't match header size [" + columns.size() + "].");
        }

        for (int i = 0; i < columns.size(); i++) {
            ingestDocument.setFieldValue(columns.get(i), values[i]);
        }
    } else {
        // TODO should we have ignoreMissing flag?
        throw new IllegalArgumentException("field[" + this.field + "] is empty string.");
    }
    return ingestDocument;
}
 

public void testMismatchLength() throws Exception {
    Map<String, Object> documentShort = new HashMap<>();
    documentShort.put("source_field", "a_value");
    IngestDocument ingestDocumentShort = RandomDocumentPicks.randomIngestDocument(random(), documentShort);

    CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(10), "source_field", defaultColumns, '\"', ',',4096);

    IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocumentShort));
    assertThat(e.getMessage(), equalTo("field[source_field] size [1] doesn't match header size [" + defaultColumns.size() + "]."));

    Map<String, Object> documentLong = new HashMap<>();
    documentLong.put("source_field", "a_value,b_value,c_value");
    IngestDocument ingestDocumentLong = RandomDocumentPicks.randomIngestDocument(random(), documentLong);

    e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocumentLong));
    assertThat(e.getMessage(), equalTo("field[source_field] size [3] doesn't match header size [" + defaultColumns.size() + "]."));
}
 

@SuppressWarnings("unchecked")
private List<String> getGeoShapeFieldsFromDoc(IngestDocument ingestDocument) {
    List<String> fields = new ArrayList<>();

    Map<String, Object> baseMap;
    if (path != null) {
        baseMap = ingestDocument.getFieldValue(this.path, Map.class);
    } else {
        baseMap = ingestDocument.getSourceAndMetadata();
    }

    for (String fieldName : baseMap.keySet()) {
        if (Regex.simpleMatch(field, fieldName)) {
            if (path != null) {
                fieldName = path + "." + fieldName;
            }
            fields.add(fieldName);
        }
    }

    return fields;
}
 

public void testManyTimes() throws Exception {
    CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(10), "source_field", defaultColumns, '\"', ',',4096);
    int times = 50000;

    logger.info("start");
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < times; i++) {
        Map<String, Object> document = new HashMap<>();
        document.put("source_field", "a_value, b_value");
        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);


        Map<String, Object> data = processor.execute(ingestDocument).getSourceAndMetadata();

        assertThat(data, hasKey("a"));
        assertThat(data.get("a"), is("a_value"));
        assertThat(data, hasKey("b"));
        assertThat(data.get("b"), is("b_value"));
    }
    logger.info("end. Loop is " + times + " times. Process Time is " + String.valueOf(System.currentTimeMillis() - startTime) + " ms");
}
 

public void testAnnotatedText() throws Exception {
    Map<String, Object> config = new HashMap<>();
    config.put("field", "source_field");
    config.put("annotated_text_field", "my_annotated_text_field");

    OpenNlpProcessor.Factory factory = new OpenNlpProcessor.Factory(service);
    Map<String, Processor.Factory> registry = Collections.emptyMap();
    OpenNlpProcessor processor = factory.create(registry, randomAlphaOfLength(10), config);

    IngestDocument ingestDocument = processor.execute(getIngestDocument());
    String content = ingestDocument.getFieldValue("my_annotated_text_field", String.class);
    assertThat(content, is("[Kobe Bryant](Person_Kobe Bryant) was one of the best basketball players of all times. Not even" +
            " [Michael Jordan](Person_Michael Jordan) has ever scored 81 points in one game. [Munich](Location_Munich) is really" +
            " an awesome city, but [New York](Location_New York) is as well. [Yesterday](Date_Yesterday) has been the hottest" +
            " day of the year."));
}
 

public void testThatExistingValuesAreMergedWithoutDuplicates() throws Exception {
    OpenNlpProcessor processor = new OpenNlpProcessor(service, randomAlphaOfLength(10), "source_field", "target_field",
            null, new HashSet<>(Arrays.asList("names", "dates", "locations")));

    IngestDocument ingestDocument = getIngestDocument();

    Map<String, Object> entityData = new HashMap<>();
    entityData.put("names", Arrays.asList("Magic Johnson", "Kobe Bryant"));
    entityData.put("locations", Arrays.asList("Paris", "Munich"));
    entityData.put("dates", Arrays.asList("Today", "Yesterday"));

    ingestDocument.setFieldValue("target_field", entityData);

    ingestDocument = processor.execute(ingestDocument);

    entityData = getIngestDocumentData(ingestDocument);

    assertThatHasElements(entityData, "names", "Magic Johnson", "Kobe Bryant", "Michael Jordan");
    assertThatHasElements(entityData, "dates", "Today", "Yesterday");
    assertThatHasElements(entityData, "locations", "Paris", "Munich", "New York");
}
 

@Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
    Detector detector = DetectorFactory.create();
    detector.setMaxTextLength(maxLength.bytesAsInt());

    String content;
    try {
        content = ingestDocument.getFieldValue(field, String.class);
    } catch (IllegalArgumentException e) {
        if (ignoreMissing) {
            return ingestDocument;
        }
        throw e;
    }
    if (Strings.isEmpty(content)) {
        return ingestDocument;
    }

    detector.append(content);
    String language = detector.detect();

    ingestDocument.setFieldValue(targetField, language);

    return ingestDocument;
}
 

public void testSimple() throws Exception {
    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "a_value, b_value");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);

    CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(10), "source_field", defaultColumns, '\"', ',',4096);
    Map<String, Object> data = processor.execute(ingestDocument).getSourceAndMetadata();

    assertThat(data, hasKey("a"));
    assertThat(data.get("a"), is("a_value"));
    assertThat(data, hasKey("b"));
    assertThat(data.get("b"), is("b_value"));

}
 

public void testToXContent() throws Exception {
    OpenNlpProcessor processor = new OpenNlpProcessor(service, randomAlphaOfLength(10), "source_field", "target_field",
            null, new HashSet<>(Arrays.asList("names", "dates", "locations")));

    IngestDocument ingestDocument = getIngestDocument();
    processor.execute(ingestDocument);

    SimulateProcessorResult result = new SimulateProcessorResult("tag", ingestDocument);

    try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
    }
}
 

private static void mergeExisting(Map<String, Set<String>> entities, IngestDocument ingestDocument, String targetField) {
    if (ingestDocument.hasField(targetField)) {
        @SuppressWarnings("unchecked")
        Map<String, Set<String>> existing = ingestDocument.getFieldValue(targetField, Map.class);
        entities.putAll(existing);
    } else {
        ingestDocument.setFieldValue(targetField, entities);
    }
}
 

@Override
public IngestDocument execute(IngestDocument ingestDocument) {
    String content = ingestDocument.getFieldValue(sourceField, String.class);

    if (Strings.hasLength(content)) {
        Map<String, Set<String>> entities = new HashMap<>();
        mergeExisting(entities, ingestDocument, targetField);

        List<ExtractedEntities> extractedEntities = new ArrayList<>();
        for (String field : fields) {
            ExtractedEntities data = openNlpService.find(content, field);
            extractedEntities.add(data);
            merge(entities, field, data.getEntityValues());
        }

        // convert set to list, otherwise toXContent serialization in simulate pipeline fails
        Map<String, List<String>> entitiesToStore = new HashMap<>();
        Iterator<Map.Entry<String, Set<String>>> iterator = entities.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry<String, Set<String>> entry = iterator.next();
            entitiesToStore.put(entry.getKey(), new ArrayList<>(entry.getValue()));
        }

        ingestDocument.setFieldValue(targetField, entitiesToStore);

        if (Strings.hasLength(annotatedTextField) && extractedEntities.isEmpty() == false) {
            String annotatedText = OpenNlpService.createAnnotatedText(content, extractedEntities);
            ingestDocument.setFieldValue(annotatedTextField, annotatedText);
        }
    }

    return ingestDocument;
}
 

@Test
public void testThatProcessorWorks() {
    Map<String, Object> document = new HashMap<>();
    document.put(SOURCE_FIELD,
        "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
            "AppleWebKit/537.36 (KHTML, like Gecko) " +
            "Chrome/53.0.2785.124 Mobile Safari/537.36");
    IngestDocument ingestDocument = new IngestDocument("index", "type", "id", null, 42L, VersionType.EXTERNAL, document);

    YauaaProcessor      processor = new YauaaProcessor("tag", SOURCE_FIELD, TARGET_FIELD, null, -1, -1, null);
    Map<String, Object> data      = processor.execute(ingestDocument).getSourceAndMetadata();

    MatcherAssert.assertThat(data, hasKey(TARGET_FIELD));

    Map<String, String> results = (Map<String, String>) data.get(TARGET_FIELD);

    assertHasKValue(results, "DeviceClass",                      "Phone");
    assertHasKValue(results, "DeviceBrand",                      "Google");
    assertHasKValue(results, "DeviceName",                       "Google Nexus 6");
    assertHasKValue(results, "OperatingSystemClass",             "Mobile");
    assertHasKValue(results, "OperatingSystemName",              "Android");
    assertHasKValue(results, "OperatingSystemNameVersion",       "Android 7.0");
    assertHasKValue(results, "OperatingSystemNameVersionMajor",  "Android 7");
    assertHasKValue(results, "OperatingSystemVersion",           "7.0");
    assertHasKValue(results, "OperatingSystemVersionBuild",      "NBD90Z");
    assertHasKValue(results, "OperatingSystemVersionMajor",      "7");
    assertHasKValue(results, "LayoutEngineClass",                "Browser");
    assertHasKValue(results, "LayoutEngineName",                 "Blink");
    assertHasKValue(results, "LayoutEngineNameVersion",          "Blink 53.0");
    assertHasKValue(results, "LayoutEngineNameVersionMajor",     "Blink 53");
    assertHasKValue(results, "LayoutEngineVersion",              "53.0");
    assertHasKValue(results, "LayoutEngineVersionMajor",         "53");
    assertHasKValue(results, "AgentClass",                       "Browser");
    assertHasKValue(results, "AgentName",                        "Chrome");
    assertHasKValue(results, "AgentNameVersion",                 "Chrome 53.0.2785.124");
    assertHasKValue(results, "AgentNameVersionMajor",            "Chrome 53");
    assertHasKValue(results, "AgentVersion",                     "53.0.2785.124");
    assertHasKValue(results, "AgentVersionMajor",                "53");
}
 

@Override
public IngestDocument execute(IngestDocument ingestDocument) {
    String content = ingestDocument.getFieldValue(field, String.class);

    UserAgent userAgent = uaa.parse(content);

    Map<String, String> resultMap = userAgent.toMap();
    resultMap.remove(USERAGENT_FIELDNAME);
    ingestDocument.setFieldValue(targetField, resultMap);
    return ingestDocument;
}
 

public void testEmptyField() throws Exception {
    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);

    CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(10), "source_field", defaultColumns, '\"', ',',4096);

    IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
    assertThat(e.getMessage(), equalTo("field[source_field] is empty string."));
}
 

@Test
public void testIngestPlugin() throws Exception {
    IngestYauaaPlugin plugin = new IngestYauaaPlugin();

    Map<String, Processor.Factory> processors = plugin.getProcessors(null);

    Processor.Factory yauaaFactory = processors.get("yauaa");

    Map<String, Object>  configuration = new HashMap<>();

    configuration.put("field",        SOURCE_FIELD);
    configuration.put("target_field", TARGET_FIELD);
    configuration.put("fieldNames",   Arrays.asList("DeviceClass", "DeviceBrand", "DeviceName", "AgentNameVersionMajor", "FirstProductName"));
    configuration.put("cacheSize",    10);
    configuration.put("preheat",      10);
    configuration.put("extraRules",   "config:\n- matcher:\n    extract:\n      - 'FirstProductName     : 1 :agent.(1)product.(1)name'\n");

    Processor processor = yauaaFactory.create(processors, "tag", configuration);

    assertEquals("yauaa", processor.getType());

    Map<String, Object> document = new HashMap<>();
    document.put(SOURCE_FIELD,
        "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
            "AppleWebKit/537.36 (KHTML, like Gecko) " +
            "Chrome/53.0.2785.124 Mobile Safari/537.36");
    IngestDocument ingestDocument = new IngestDocument("index", "type", "id", null, 42L, VersionType.EXTERNAL, document);

    Map<String, Object> data      = processor.execute(ingestDocument).getSourceAndMetadata();

    MatcherAssert.assertThat(data, hasKey(TARGET_FIELD));

    Map<String, String> results    = (Map<String, String>) data.get(TARGET_FIELD);

    // The EXPLICITLY requested fields
    assertHasKValue(results, "FirstProductName",        "Mozilla");
    assertHasKValue(results, "DeviceClass",             "Phone");
    assertHasKValue(results, "DeviceBrand",             "Google");
    assertHasKValue(results, "DeviceName",              "Google Nexus 6");
    assertHasKValue(results, "AgentNameVersionMajor",   "Chrome 53");

    // The IMPLICITLY requested fields (i.e. partials of the actually requested ones)
    assertHasKValue(results, "AgentName",               "Chrome");
    assertHasKValue(results, "AgentVersion",            "53.0.2785.124");
    assertHasKValue(results, "AgentVersionMajor",       "53");

    // The NOT requested fields
    assertHasNotKey(results, "OperatingSystemClass");
    assertHasNotKey(results, "OperatingSystemName");
    assertHasNotKey(results, "OperatingSystemNameVersion");
    assertHasNotKey(results, "OperatingSystemNameVersionMajor");
    assertHasNotKey(results, "OperatingSystemVersion");
    assertHasNotKey(results, "OperatingSystemVersionBuild");
    assertHasNotKey(results, "OperatingSystemVersionMajor");
    assertHasNotKey(results, "LayoutEngineClass");
    assertHasNotKey(results, "LayoutEngineName");
    assertHasNotKey(results, "LayoutEngineNameVersion");
    assertHasNotKey(results, "LayoutEngineNameVersionMajor");
    assertHasNotKey(results, "LayoutEngineVersion");
    assertHasNotKey(results, "LayoutEngineVersionMajor");
    assertHasNotKey(results, "AgentClass");
    assertHasNotKey(results, "AgentNameVersion");

    LoggerFactory.getLogger("TestYauaaProcessor").info("Complete set of returned results:{}", results);
}
 

@Test
public void testExtraRules() {
    Map<String, Object> document = new HashMap<>();
    document.put(SOURCE_FIELD,
        "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
            "AppleWebKit/537.36 (KHTML, like Gecko) " +
            "Chrome/53.0.2785.124 Mobile Safari/537.36");
    IngestDocument ingestDocument = new IngestDocument("index", "type", "id", null, 42L, VersionType.EXTERNAL, document);

    List<String> fieldNames = Arrays.asList("DeviceClass", "DeviceBrand", "DeviceName", "AgentNameVersionMajor", "FirstProductName");
    Integer      cacheSize  = 10;
    Integer      preheat    = 10;
    String       extraRules = "config:\n- matcher:\n    extract:\n      - 'FirstProductName     : 1 :agent.(1)product.(1)name'\n";

    YauaaProcessor      processor = new YauaaProcessor("tag", SOURCE_FIELD, TARGET_FIELD, fieldNames, cacheSize, preheat, extraRules);
    Map<String, Object> data      = processor.execute(ingestDocument).getSourceAndMetadata();

    MatcherAssert.assertThat(data, hasKey(TARGET_FIELD));

    Map<String, String> results = (Map<String, String>) data.get(TARGET_FIELD);

    // The EXPLICITLY requested fields
    assertHasKValue(results, "FirstProductName",        "Mozilla");
    assertHasKValue(results, "DeviceClass",             "Phone");
    assertHasKValue(results, "DeviceBrand",             "Google");
    assertHasKValue(results, "DeviceName",              "Google Nexus 6");
    assertHasKValue(results, "AgentNameVersionMajor",   "Chrome 53");

    // The IMPLICITLY requested fields (i.e. partials of the actually requested ones)
    assertHasKValue(results, "AgentName",               "Chrome");
    assertHasKValue(results, "AgentVersion",            "53.0.2785.124");
    assertHasKValue(results, "AgentVersionMajor",       "53");

    // The NOT requested fields
    assertHasNotKey(results, "OperatingSystemClass");
    assertHasNotKey(results, "OperatingSystemName");
    assertHasNotKey(results, "OperatingSystemNameVersion");
    assertHasNotKey(results, "OperatingSystemNameVersionMajor");
    assertHasNotKey(results, "OperatingSystemVersion");
    assertHasNotKey(results, "OperatingSystemVersionBuild");
    assertHasNotKey(results, "OperatingSystemVersionMajor");
    assertHasNotKey(results, "LayoutEngineClass");
    assertHasNotKey(results, "LayoutEngineName");
    assertHasNotKey(results, "LayoutEngineNameVersion");
    assertHasNotKey(results, "LayoutEngineNameVersionMajor");
    assertHasNotKey(results, "LayoutEngineVersion");
    assertHasNotKey(results, "LayoutEngineVersionMajor");
    assertHasNotKey(results, "AgentClass");
    assertHasNotKey(results, "AgentNameVersion");
}
 

private Map<String, Object> getIngestDocumentData(OpenNlpProcessor processor) throws Exception {
    IngestDocument ingestDocument = getIngestDocument();
    return getIngestDocumentData(processor.execute(ingestDocument));
}
 

private IngestDocument getIngestDocument() throws Exception {
    return getIngestDocument("Kobe Bryant was one of the best basketball players of all times. Not even Michael Jordan has ever " +
            "scored 81 points in one game. Munich is really an awesome city, but New York is as well. Yesterday has been the " +
            "hottest day of the year.");
}
 

private IngestDocument getIngestDocument(String content) throws Exception {
    Map<String, Object> document = new HashMap<>();
    document.put("source_field", content);
    return RandomDocumentPicks.randomIngestDocument(random(), document);
}
 

private Map<String, Object> getIngestDocumentData(IngestDocument ingestDocument) throws Exception {
    @SuppressWarnings("unchecked")
    Map<String, Object> data = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
    return data;
}
 

@SuppressWarnings("unchecked")
@Override
public IngestDocument execute(IngestDocument ingestDocument) throws IOException, ParseException {
    List<String> geo_objects_list = getGeoShapeFieldsFromDoc(ingestDocument);
    for (String geoShapeField : geo_objects_list) {

        Object geoShapeObject = ingestDocument.getFieldValue(geoShapeField, Object.class);

        if (geoShapeObject == null) {
            continue;
        }

        ShapeBuilder<?,?, ?> shapeBuilder = getShapeBuilderFromObject(geoShapeObject);

        Shape shape = null;
        try {
            shape = shapeBuilder.buildS4J();
        }
        catch (InvalidShapeException ignored) {}

        if (shape == null && fixedField == null) {
            throw new IllegalArgumentException("unable to parse shape [" + shapeBuilder.toWKT() + "]");
        }

        Geometry geom = new WKTReader().read(shapeBuilder.toWKT());

        // fix shapes if needed
        if (shape == null && fixedField != null) {
            geom = GeoUtils.removeDuplicateCoordinates(geom);
        }

        ingestDocument.removeField(geoShapeField);

        if (keepShape) {
            ingestDocument.setFieldValue(geoShapeField + "." + shapeField, geoShapeObject);
        }

        if (fixedField != null) {
            ingestDocument.setFieldValue(geoShapeField + "." + fixedField, new WKTWriter().write(geom));
        }

        // compute and add extra geo sub-fields
        byte[] wkb = new WKBWriter().write(geom);  // elastic will auto-encode this as b64

        if (hashField != null) ingestDocument.setFieldValue(
                geoShapeField + ".hash", String.valueOf(GeoUtils.getHashFromWKB(new BytesRef(wkb))));
        if (wkbField != null) ingestDocument.setFieldValue(
                geoShapeField + "." + wkbField, wkb);
        if (typeField != null) ingestDocument.setFieldValue(
                geoShapeField + "." + typeField, geom.getGeometryType());
        if (areaField != null) ingestDocument.setFieldValue(
                geoShapeField + "." + areaField, geom.getArea());
        if (centroidField != null) ingestDocument.setFieldValue(
                geoShapeField + "." + centroidField, GeoUtils.getCentroidFromGeom(geom));
        if (bboxField != null) {
            Coordinate[] coords = geom.getEnvelope().getCoordinates();
            if (coords.length >= 4) ingestDocument.setFieldValue(
                    geoShapeField + "." + bboxField,
                    GeoUtils.getBboxFromCoords(coords));
        }
    }
    return ingestDocument;
}