org.elasticsearch.search.SearchHit Java Examples

@Override
public Event getEvent(final long snifferId, final String eventId) {
	return clientTpl.executeWithClient(new ClientCallback<Event>() {
		@Override
		public Event execute(final Client client) {
			try {
				final SearchResponse r = client.prepareSearch(indexNamingStrategy.getRetrievalNames(snifferId))
						.setIndicesOptions(IndicesOptions.lenientExpandOpen())
						.setQuery(QueryBuilders.idsQuery().ids(eventId)).execute().get();
				if (r != null && r.getHits().hits().length > 0) {
					final SearchHit hit = r.getHits().hits()[0];
					final Event event = jsonMapper.readValue(hit.getSourceAsString(), Event.class);
					event.setId(hit.getId());
					return event;
				} else {
					return null;
				}
			} catch (final Exception e) {
				throw new DataAccessException("Failed to load for sniffer=" + snifferId + " the event: " + eventId,
						e);
			}
		}
	});
}
 

private boolean hasData(int expect) {
    SearchHits searchHits = getData();

    if (searchHits == null) {
        LOG.debug("There are not search hit to return");

        return false;
    }

    SearchHit[] hits = searchHits.getHits();
    if (hits == null) {
        LOG.debug("Empty data set");

        return false;
    }

    int count = hits.length;

    if (count != expect) {
        LOG.debug("Not enough records: {} available, but {} expected", count, expect);

        return false;
    }

    return true;
}
 

void assertSearch(int expectedHits, SearchResponse response, Event... events) {
  SearchHits hitResponse = response.getHits();
  assertEquals(expectedHits, hitResponse.getTotalHits());

  SearchHit[] hits = hitResponse.getHits();
  Arrays.sort(hits, new Comparator<SearchHit>() {
    @Override
    public int compare(SearchHit o1, SearchHit o2) {
      return o1.getSourceAsString().compareTo(o2.getSourceAsString());
    }
  });

  for (int i = 0; i < events.length; i++) {
    Event event = events[i];
    SearchHit hit = hits[i];
    Map<String, Object> source = hit.getSource();
    assertEquals(new String(event.getBody()), source.get("@message"));
  }
}
 

private void leftJoinWithAllFromSecondTable(boolean useNestedLoops) throws SQLFeatureNotSupportedException, IOException, SqlParseException {
    String query = String.format("select c.name.firstname , d.* from %s/gotCharacters c " +
            "LEFT JOIN %s/gotCharacters d on d.name = c.house " +
            "where d.sigil <> 'direwolf'"
            ,  TEST_INDEX_GAME_OF_THRONES, TEST_INDEX_GAME_OF_THRONES);
    if(useNestedLoops) query = query.replace("select","select /*! USE_NL*/ ");
    SearchHit[] hits = joinAndGetHits(query);
    Assert.assertEquals(10, hits.length);
    for (SearchHit hit : hits) {
        if(hit.getId().endsWith("0")){
            Assert.assertEquals(1,hit.getSourceAsMap().size());
        }
        else {
            Assert.assertEquals(5,hit.getSourceAsMap().size());
        }
    }

}
 

private ProfileTask parseTask(SearchHit data) {
    return ProfileTask.builder()
                      .id(data.getId())
                      .serviceId((String) data.getSourceAsMap().get(ProfileTaskRecord.SERVICE_ID))
                      .endpointName((String) data.getSourceAsMap().get(ProfileTaskRecord.ENDPOINT_NAME))
                      .startTime(((Number) data.getSourceAsMap().get(ProfileTaskRecord.START_TIME)).longValue())
                      .createTime(((Number) data.getSourceAsMap()
                                                .get(ProfileTaskRecord.CREATE_TIME)).longValue())
                      .duration(((Number) data.getSourceAsMap().get(ProfileTaskRecord.DURATION)).intValue())
                      .minDurationThreshold(((Number) data.getSourceAsMap()
                                                          .get(
                                                              ProfileTaskRecord.MIN_DURATION_THRESHOLD)).intValue())
                      .dumpPeriod(((Number) data.getSourceAsMap()
                                                .get(ProfileTaskRecord.DUMP_PERIOD)).intValue())
                      .maxSamplingCount(((Number) data.getSourceAsMap()
                                                      .get(ProfileTaskRecord.MAX_SAMPLING_COUNT)).intValue())
                      .build();
}
 

@Override
public void onResponse(SearchResponse response) {
    SearchHits hits = response.getHits();
    LOG.info("{} ES query returned {} hits in {} msec", logIdprefix,
            hits.getHits().length, response.getTook().getMillis());
    hasFinished = hits.getHits().length == 0;
    synchronized (this.queue) {
        // Unlike standard spouts, the scroll queries should never return
        // the same
        // document twice -> no need to look in the buffer or cache
        for (SearchHit hit : hits) {
            Map<String, Object> keyValues = hit.getSourceAsMap();
            String url = (String) keyValues.get("url");
            String status = (String) keyValues.get("status");
            String nextFetchDate = (String) keyValues.get("nextFetchDate");
            Metadata metadata = fromKeyValues(keyValues);
            metadata.setValue(
                    AbstractStatusUpdaterBolt.AS_IS_NEXTFETCHDATE_METADATA,
                    nextFetchDate);
            this.queue.add(new Values(url, metadata, Status.valueOf(status)));
        }
    }
    scrollId = response.getScrollId();
    // remove lock
    markQueryReceivedNow();
}
 

/**
 * @return  All modules registered in elastic search
 */
public List<Module> getAll() {

    SearchResponse response = elasticSearchClient
            .getClient()
            .prepareSearch( DAO_INDEX_KEY )
            .setTypes( DAO_TYPE_KEY )
            .setSize( MAX_RESULT_SIZE )
            .execute()
            .actionGet();

    LOG.debug( "response: {}", response );

    ArrayList<Module> modules = new ArrayList<Module>();

    for ( SearchHit hit : response.getHits().hits() ) {
        modules.add( toModule( hit ) );
    }

    return modules;
}
 

private static void parseAgentIds(SearchHit hit, Occurrence occ) {
  Function<Map<String, Object>, AgentIdentifier> mapFn = m -> {
    AgentIdentifier ai = new AgentIdentifier();
    extractValue(m, "type", AgentIdentifierType::valueOf).ifPresent(ai::setType);
    extractStringValue(m, "value").ifPresent(ai::setValue);
    return ai;
  };

  getObjectsListValue(hit, RECORDED_BY_ID)
    .map(i -> i.stream().map(mapFn).collect(Collectors.toList()))
    .ifPresent(occ::setRecordedByIds);

  getObjectsListValue(hit, IDENTIFIED_BY_ID)
    .map(i -> i.stream().map(mapFn).collect(Collectors.toList()))
    .ifPresent(occ::setIdentifiedByIds);
}
 

@Override
public List<NetworkAddressAlias> loadLastUpdate(long timeBucketInMinute) {
    List<NetworkAddressAlias> networkAddressAliases = new ArrayList<>();

    try {
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.rangeQuery(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET)
                                               .gte(timeBucketInMinute));
        searchSourceBuilder.size(resultWindowMaxSize);

        SearchResponse response = getClient().search(NetworkAddressAlias.INDEX_NAME, searchSourceBuilder);

        final NetworkAddressAlias.Builder builder = new NetworkAddressAlias.Builder();
        for (SearchHit searchHit : response.getHits().getHits()) {
            networkAddressAliases.add(builder.map2Data(searchHit.getSourceAsMap()));
        }
    } catch (Throwable t) {
        log.error(t.getMessage(), t);
    }

    return networkAddressAliases;
}
 

/**
 * 
 * @param indices
 * @param field
 * @param queryString
 */
private static void matchQuery(String indices,String field,String queryString){
	Client client = createTransportClient();
	SearchResponse searchResponse = client.prepareSearch(indices)
	        .setQuery(QueryBuilders.matchQuery(field, queryString))
	        .execute()
	        .actionGet();
	SearchHits searchHits = searchResponse.getHits();
	System.out.println("---------------matchquery--在["+field+"]中搜索关键字["+queryString+"]---------------------");
	System.out.println("共匹配到:"+searchHits.getTotalHits()+"条记录!");
	SearchHit[] hits = searchHits.getHits();
	for (SearchHit searchHit : hits) {
		Map<String, Object> sourceAsMap = searchHit.sourceAsMap();
		Set<String> keySet = sourceAsMap.keySet();
		for (String string : keySet) {
			System.out.println(string+":"+sourceAsMap.get(string));
		}
		System.out.println();
	}
}
 

/**
 * 
 * @param hit
 * @param seach_fileds
 * @return
 */
public static Map<String, Object> getHighlight(SearchHit hit, List<String> seach_fileds) {
    Map<String, Object> result = new HashMap<String, Object>();
    Map<String, HighlightField> highlights = hit.highlightFields();
    for (String filed : seach_fileds) {
        HighlightField highlight = highlights.get(filed);
        if (null == highlight) {
            continue;
        }
        StringBuffer sb = new StringBuffer();
        Text[] fragments = highlight.fragments();
        for (Text fragment : fragments) {
            sb.append(fragment);
        }
        result.put(filed + "_HIGH", sb.toString());
    }
    return result;

}
 

private List<DetectorDocument> getDetectorsFromElasticSearch(SearchRequest searchRequest) {
    SearchResponse response;
    try {
        response = legacyElasticSearchClient.search(searchRequest, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error("Error ES lookup", e);
        throw new RuntimeException(e);
    }

    SearchHit[] hits = response.getHits().getHits();
    List<DetectorDocument> detectors = new ArrayList<>();
    for (val hit : hits) {

        val detector = (DetectorDocument) objectMapperUtil.convertToObject(hit.getSourceAsString(), new TypeReference<DetectorDocument>() {
        });
        val newElasticsearchDetector = getElasticSearchDetector(detector);
        detectors.add(newElasticsearchDetector);
    }
    return detectors;
}
 

public void search() throws Exception {
	try {
		QueryBuilder qb = QueryBuilders.termQuery("title", "大师");
		SearchResponse scrollResp = client.prepareSearch("app").setSearchType(SearchType.SCAN).setScroll(
				new TimeValue(60000)).setQuery(qb).setSize(100).execute().actionGet();

		while (true) {
			scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
			for (SearchHit hit : scrollResp.getHits()) {
				Map<String, Object> source = hit.getSource();
				if (!source.isEmpty()) {
					for (Iterator<Map.Entry<String, Object>> it = source.entrySet().iterator(); it.hasNext();) {
						Map.Entry<String, Object> entry = it.next();
						System.out.println(entry.getKey() + "=======" + entry.getValue());

					}
				}

			}
			if (scrollResp.getHits().getTotalHits() == 0) {
				break;
			}

		}
	}
	catch (Exception e) {
		e.printStackTrace();
	}
	finally {
		client.close();
	}

}
 

/**
 * 高亮结果集 特殊处理
 * @param searchResponse 搜索的结果集
 * @param highlightField 高亮字段
 */
private static List<Map<String, Object>> setSearchResponse(SearchResponse searchResponse, String highlightField) {
    List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
    for (SearchHit searchHit : searchResponse.getHits().getHits()) {
        Map<String, Object> resultMap = getResultMap(searchHit, highlightField);
        sourceList.add(resultMap);
    }
    return sourceList;
}
 

private void fillComparableSetFromHits(String[] fieldsOrder, SearchHit[] hits, Set<ComperableHitResult> setToFill) {
    if (Objects.isNull(hits)) {
        return;
    }

    for (SearchHit hit : hits) {
        ComperableHitResult comperableHitResult = new ComperableHitResult(hit, fieldsOrder, this.separator);
        if (!comperableHitResult.isAllNull()) {
            setToFill.add(comperableHitResult);
        }
    }
}
 

public static void search(Map<String, Object> m) throws Exception{

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.from(0);
        searchSourceBuilder.size(5);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.query(QueryBuilders.termQuery("message", "视频"));

        SearchRequest searchRequest = new SearchRequest();
        searchRequest.indices("torrent");
        searchRequest.source(searchSourceBuilder);


        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        RestStatus status = searchResponse.status();
        TimeValue took = searchResponse.getTook();
        Boolean terminatedEarly = searchResponse.isTerminatedEarly();
        boolean timedOut = searchResponse.isTimedOut();
        SearchHits hits = searchResponse.getHits();
        SearchHit[] searchHits = hits.getHits();
        for (SearchHit hit : searchHits) {
            // do something with the SearchHit
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            System.out.print(sourceAsMap+"===");
        }


    }
 

private static void setDatasetFields(SearchHit hit, Occurrence occ) {
  getValue(hit, PUBLISHING_COUNTRY, v -> Country.fromIsoCode(v.toUpperCase()))
      .ifPresent(occ::setPublishingCountry);
  getValue(hit, DATASET_KEY, UUID::fromString).ifPresent(occ::setDatasetKey);
  getValue(hit, INSTALLATION_KEY, UUID::fromString).ifPresent(occ::setInstallationKey);
  getValue(hit, PUBLISHING_ORGANIZATION_KEY, UUID::fromString)
      .ifPresent(occ::setPublishingOrgKey);
  getValue(hit, LICENSE, v -> License.fromString(v).orElse(null)).ifPresent(occ::setLicense);
  getValue(hit, PROTOCOL, EndpointType::fromString).ifPresent(occ::setProtocol);

  getListValue(hit, NETWORK_KEY)
      .ifPresent(
          v -> occ.setNetworkKeys(v.stream().map(UUID::fromString).collect(Collectors.toList())));
}
 

private void testTopCustomerQueriesWithHiveAndES() {
	hiveSearchClicksService.loadSearchCustomerQueryTable();

	hiveSearchClicksService
			.loadTopSearchCustomerQueryToElasticSearchIndex();

	String indexName = "topqueries";
	Client client = searchClientService.getClient();
	boolean exists = client.admin().indices().prepareExists(indexName)
			.get().isExists();
	long totalCount = 0;
	if (exists) {
		totalCount = client.prepareCount(indexName).get().getCount();
	}
	System.out.println("Total topqueries count:" + totalCount);

	for (SearchHit searchHit : client.prepareSearch(indexName)
			.setSize(searchEventsCount).get().getHits()) {
		System.out.println(searchHit.getSource());
	}

	long countTotalRecords = customerTopQueryService
			.countCustomerTopQueryTotalRecords();
	assertTrue(countTotalRecords > 0);
	assertEquals(totalCount, countTotalRecords);
	customerTopQueryService.deleteAllCustomerTopQueryRecords();
}
 

@Test
public void idsQueryMultipleId() throws SqlParseException, SQLFeatureNotSupportedException {
    String query = String.format("select * from %s/dog where _id = IDS_QUERY(dog,1,2,3)",TEST_INDEX_DOG);
    SearchHit[] hits = query(query).getHits();
    Assert.assertEquals(1,hits.length);
    Map<String, Object> hitAsMap = hits[0].getSourceAsMap();
    Assert.assertEquals("rex",hitAsMap.get("dog_name"));
    Assert.assertEquals("Daenerys",hitAsMap.get("holdersName"));
    Assert.assertEquals(2, hitAsMap.get("age"));

}
 

@Override
public SegmentRecord getProfiledSegment(String segmentId) throws IOException {
    SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
    sourceBuilder.query(QueryBuilders.termQuery(SegmentRecord.SEGMENT_ID, segmentId));
    sourceBuilder.size(1);

    SearchResponse response = getClient().search(SegmentRecord.INDEX_NAME, sourceBuilder);

    if (response.getHits().getHits().length == 0) {
        return null;
    }
    SearchHit searchHit = response.getHits().getHits()[0];
    SegmentRecord segmentRecord = new SegmentRecord();
    segmentRecord.setSegmentId((String) searchHit.getSourceAsMap().get(SegmentRecord.SEGMENT_ID));
    segmentRecord.setTraceId((String) searchHit.getSourceAsMap().get(SegmentRecord.TRACE_ID));
    segmentRecord.setServiceId((String) searchHit.getSourceAsMap().get(SegmentRecord.SERVICE_ID));
    segmentRecord.setEndpointName((String) searchHit.getSourceAsMap().get(SegmentRecord.ENDPOINT_NAME));
    segmentRecord.setStartTime(((Number) searchHit.getSourceAsMap().get(SegmentRecord.START_TIME)).longValue());
    segmentRecord.setEndTime(((Number) searchHit.getSourceAsMap().get(SegmentRecord.END_TIME)).longValue());
    segmentRecord.setLatency(((Number) searchHit.getSourceAsMap().get(SegmentRecord.LATENCY)).intValue());
    segmentRecord.setIsError(((Number) searchHit.getSourceAsMap().get(SegmentRecord.IS_ERROR)).intValue());
    String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(SegmentRecord.DATA_BINARY);
    if (!Strings.isNullOrEmpty(dataBinaryBase64)) {
        segmentRecord.setDataBinary(Base64.getDecoder().decode(dataBinaryBase64));
    }
    segmentRecord.setVersion(((Number) searchHit.getSourceAsMap().get(SegmentRecord.VERSION)).intValue());
    return segmentRecord;
}
 

@Override
public void decode(SearchHit hit, Supplier<Object> getter, BlockBuilder output)
{
    Object value = getter.get();
    if (value == null) {
        output.appendNull();
    }
    else if (value instanceof Number) {
        REAL.writeLong(output, Float.floatToRawIntBits(((Number) value).floatValue()));
    }
    else {
        throw new PrestoException(TYPE_MISMATCH, format("Expected a numeric value for field %s of type REAL: %s [%s]", path, value, value.getClass().getSimpleName()));
    }
}
 

@Override
public void decode(SearchHit hit, Supplier<Object> getter, BlockBuilder output)
{
    Object value = getter.get();
    if (value == null) {
        output.appendNull();
    }
    else if (value instanceof Number) {
        BIGINT.writeLong(output, ((Number) value).longValue());
    }
    else {
        throw new PrestoException(TYPE_MISMATCH, format("Expected a numeric value for field '%s' of type BIGINT: %s [%s]", path, value, value.getClass().getSimpleName()));
    }
}
 

protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForCursor).setSize(sizeForCursor);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            if (!handler.apply(searchHits)) {
                break;
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
}
 

/**
 * Gets a User object containing the information for given username
 *
 * @param username  queried User's username
 * @return          User object or null if no user with <code>username</code> exists
 */
public User get( String username ) {

    SearchResponse response = elasticSearchClient.getClient()
            .prepareSearch( DAO_INDEX_KEY )
            .setTypes( DAO_TYPE_KEY )
            .setQuery( termQuery( "_id", username ) )
            .execute()
            .actionGet();

    SearchHit hits[] = response.getHits().hits();

    return hits.length > 0 ? toUser( hits[ 0 ] ) : null;
}
 

private static List<String> getIds(final SearchResponse response) {
    final List<String> ret = Lists.newArrayList();
    for (SearchHit hit : response.getHits().getHits()) {
        ret.add(hit.getId());
    }
    return ret;
}
 

@Test
public void dateSearch() throws IOException, SqlParseException, SQLFeatureNotSupportedException, ParseException {
	DateTimeFormatter formatter = DateTimeFormat.forPattern(DATE_FORMAT);
	DateTime dateToCompare = new DateTime(2014, 8, 18, 0, 0, 0);

	SearchHits response = query(String.format("SELECT insert_time FROM %s/online WHERE insert_time < '2014-08-18'", TEST_INDEX_ONLINE));
	SearchHit[] hits = response.getHits();
	for(SearchHit hit : hits) {
		Map<String, Object> source = hit.getSourceAsMap();
		DateTime insertTime = formatter.parseDateTime((String) source.get("insert_time"));

		String errorMessage = String.format("insert_time must be smaller then 2014-08-18. found: %s", insertTime);
		Assert.assertTrue(errorMessage, insertTime.isBefore(dateToCompare));
	}
}
 

@Override
public Object read(SearchHit hit) {
    SearchHitField field = hit.getFields().get(name);
    if (field != null && !field.values().isEmpty()) {
        if (field.values().size() == 1) {
            return field.values().get(0);
        } else {
            return field.values();
        }
    }
    return null;
}
 

@Override
public void decode(SearchHit hit, Supplier<Object> getter, BlockBuilder output)
{
    Object value = getter.get();
    if (value == null) {
        output.appendNull();
    }
    else if (value instanceof String || value instanceof Number) {
        VARCHAR.writeSlice(output, Slices.utf8Slice(value.toString()));
    }
    else {
        throw new PrestoException(TYPE_MISMATCH, format("Expected a string or numeric value for field '%s' of type VARCHAR: %s [%s]", path, value, value.getClass().getSimpleName()));
    }
}
 

private void innerMinus_CMinusTNoExistsOneField(String hint) throws SqlParseException, SQLFeatureNotSupportedException, IOException {
    String query = String.format("SELECT "+hint+" letter FROM %s/systems WHERE system_name = 'C' " +
            "minus " +
            "SELECT letter  FROM %s/systems WHERE system_name = 'T' ",TEST_INDEX_SYSTEM,TEST_INDEX_SYSTEM);
    SearchHit[] searchHits = executeAndGetHits(query);
    Assert.assertEquals("all hits should be returned", 3, searchHits.length);
}
 

private List<EventExecution> mapEventExecutionsResponse(SearchResponse response) throws IOException {
    SearchHit[] hits = response.getHits().getHits();
    List<EventExecution> executions = new ArrayList<>(hits.length);
    for (SearchHit hit : hits) {
        String source = hit.getSourceAsString();
        EventExecution tel = objectMapper.readValue(source, EventExecution.class);
        executions.add(tel);
    }
    return executions;
}