Java Code Examples for org.elasticsearch.index.fielddata.IndexFieldData

The following examples show how to use org.elasticsearch.index.fielddata.IndexFieldData. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: DocValuesIndexFieldData.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore Circuit Breaker
    final Names fieldNames = fieldType.names();
    final Settings fdSettings = fieldType.fieldDataType().getSettings();
    final Map<String, Settings> filter = fdSettings.getGroups("filter");
    if (filter != null && !filter.isEmpty()) {
        throw new IllegalArgumentException("Doc values field data doesn't support filters [" + fieldNames.fullName() + "]");
    }

    if (BINARY_INDEX_FIELD_NAMES.contains(fieldNames.indexName())) {
        assert numericType == null;
        return new BinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
    } else if (numericType != null) {
        if (TimestampFieldMapper.NAME.equals(fieldNames.indexName())
                || Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1)) {
            return new SortedNumericDVIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        } else {
            // prior to ES 1.4: multi-valued numerics were boxed inside a byte[] as BINARY
            return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType());
        }
    } else {
        return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, fieldType.fieldDataType());
    }
}
 
Example 2
/**
 * Instantiates a new reusable {@link BytesRefTermStream} based on the field type.
 */
public static BytesRefTermStream get(IndexReader reader, IndexFieldData indexFieldData) {
  if (indexFieldData instanceof IndexNumericFieldData) {
    IndexNumericFieldData numFieldData = (IndexNumericFieldData) indexFieldData;
    switch (numFieldData.getNumericType()) {

      case INT:
        return new IntegerBytesRefTermStream(reader, numFieldData);

      case LONG:
        return new LongBytesRefTermStream(reader, numFieldData);

      default:
        throw new UnsupportedOperationException("Streaming numeric type '" + numFieldData.getNumericType().name() + "' is unsupported");

    }
  }
  else {
    return new BytesBytesRefTermStream(reader, indexFieldData);
  }
}
 
Example 3
Source Project: Elasticsearch   Source File: SortSymbolVisitor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * generate a SortField from a Reference symbol.
 *
 * the implementation is similar to what {@link org.elasticsearch.search.sort.SortParseElement}
 * does.
 */
@Override
public SortField visitReference(final Reference symbol, final SortSymbolContext context) {
    // can't use the SortField(fieldName, type) constructor
    // because values are saved using docValues and therefore they're indexed in lucene as binary and not
    // with the reference valueType.
    // this is why we use a custom comparator source with the same logic as ES

    ColumnIdent columnIdent = symbol.info().ident().columnIdent();

    if (columnIdent.isColumn()) {
        if (SortParseElement.SCORE_FIELD_NAME.equals(columnIdent.name())) {
            return !context.reverseFlag ? SORT_SCORE_REVERSE : SortParseElement.SORT_SCORE;
        } else if (DocSysColumns.RAW.equals(columnIdent) || DocSysColumns.ID.equals(columnIdent)) {
            return customSortField(DocSysColumns.nameForLucene(columnIdent), symbol, context,
                    LUCENE_TYPE_MAP.get(symbol.valueType()), false);
        }
    }

    MultiValueMode sortMode = context.reverseFlag ? MultiValueMode.MAX : MultiValueMode.MIN;

    String indexName;
    IndexFieldData.XFieldComparatorSource fieldComparatorSource;
    MappedFieldType fieldType = context.context.mapperService().smartNameFieldType(columnIdent.fqn());
    if (fieldType == null){
        indexName = columnIdent.fqn();
        fieldComparatorSource = new NullFieldComparatorSource(LUCENE_TYPE_MAP.get(symbol.valueType()), context.reverseFlag, context.nullFirst);
    } else {
        indexName = fieldType.names().indexName();
        fieldComparatorSource = context.context.fieldData()
                .getForField(fieldType)
                .comparatorSource(SortOrder.missing(context.reverseFlag, context.nullFirst), sortMode, null);
    }
    return new SortField(
            indexName,
            fieldComparatorSource,
            context.reverseFlag
    );
}
 
Example 4
Source Project: Elasticsearch   Source File: AggregationContext.java    License: Apache License 2.0 5 votes vote down vote up
private ValuesSource bytesField(ValuesSourceConfig<?> config) throws IOException {
    final IndexFieldData<?> indexFieldData = config.fieldContext.indexFieldData();
    ValuesSource dataSource;
    if (indexFieldData instanceof ParentChildIndexFieldData) {
        dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData);
    } else if (indexFieldData instanceof IndexOrdinalsFieldData) {
        dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData);
    } else {
        dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);
    }
    if (config.script != null) {
        dataSource = new ValuesSource.WithScript(dataSource, config.script);
    }
    return dataSource;
}
 
Example 5
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    return new GeoPointDVIndexFieldData(index, fieldType.names(), fieldType.fieldDataType(),
            Version.indexCreated(indexSettings).before(Version.V_2_2_0));
}
 
Example 6
Source Project: Elasticsearch   Source File: BytesBinaryDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    final Names fieldNames = fieldType.names();
    return new BytesBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
}
 
Example 7
Source Project: Elasticsearch   Source File: RandomScoreFunction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a RandomScoreFunction.
 *
 * @param seed A seed for randomness
 * @param salt A value to salt the seed with, ideally unique to the running node/index
 * @param uidFieldData The field data for _uid to use for generating consistent random values for the same id
 */
public RandomScoreFunction(int seed, int salt, IndexFieldData<?> uidFieldData) {
    super(CombineFunction.MULT);
    this.originalSeed = seed;
    this.saltedSeed = seed ^ salt;
    this.uidFieldData = uidFieldData;
    if (uidFieldData == null) throw new NullPointerException("uid missing");
}
 
Example 8
private final Query toFieldDataTermsQuery(MappedFieldType fieldType, IndexFieldData fieldData,
                                          byte[] encodedTerms, long cacheKey) {
  Query query = null;

  if (fieldType instanceof NumberFieldMapper.NumberFieldType) {
    query = FieldDataTermsQuery.newLongs(encodedTerms, (IndexNumericFieldData) fieldData, cacheKey);
  } else if (fieldType instanceof StringFieldMapper.StringFieldType) {
    query = FieldDataTermsQuery.newBytes(encodedTerms, fieldData, cacheKey);
  } else {
    throw new ElasticsearchParseException("[fielddata_terms] query does not support field data type " + fieldType.fieldDataType().getType());
  }

  return query;
}
 
Example 9
private TermsCollector getTermsCollector(TermsByQueryRequest.TermsEncoding termsEncoding,
                                         IndexFieldData indexFieldData, SearchContext context) {
  switch (termsEncoding) {
    case LONG:
      return new LongTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case INTEGER:
      return new IntegerTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case BLOOM:
      return new BloomFilterTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    case BYTES:
      return new BytesRefTermsCollector(indexFieldData, context, breakerService.getBreaker(CircuitBreaker.REQUEST));
    default:
      throw new IllegalArgumentException("[termsByQuery] Invalid terms encoding: " + termsEncoding.name());
  }
}
 
Example 10
/**
 * Instantiates a new reusable {@link NumericTermStream} based on the field type.
 */
public static NumericTermStream get(IndexReader reader, IndexFieldData indexFieldData) {
  if (indexFieldData instanceof IndexNumericFieldData) {
    IndexNumericFieldData numFieldData = (IndexNumericFieldData) indexFieldData;
    if (!numFieldData.getNumericType().isFloatingPoint()) {
      return new LongTermStream(reader, numFieldData);
    }
    else {
      throw new UnsupportedOperationException("Streaming floating points is unsupported");
    }
  }
  else {
    return new HashTermStream(reader, indexFieldData);
  }
}
 
Example 11
Source Project: Elasticsearch   Source File: FieldContext.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * @return The index field datas in this context
 */
public IndexFieldData<?> indexFieldData() {
    return indexFieldData;
}
 
Example 12
Source Project: Elasticsearch   Source File: ValuesSource.java    License: Apache License 2.0 4 votes vote down vote up
public FieldData(IndexFieldData<?> indexFieldData) {
    this.indexFieldData = indexFieldData;
}
 
Example 13
Source Project: Elasticsearch   Source File: SortParseElement.java    License: Apache License 2.0 4 votes vote down vote up
private void addSortField(SearchContext context, List<SortField> sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException {
    if (SCORE_FIELD_NAME.equals(fieldName)) {
        if (reverse) {
            sortFields.add(SORT_SCORE_REVERSE);
        } else {
            sortFields.add(SORT_SCORE);
        }
    } else if (DOC_FIELD_NAME.equals(fieldName)) {
        if (reverse) {
            sortFields.add(SORT_DOC_REVERSE);
        } else {
            sortFields.add(SORT_DOC);
        }
    } else {
        MappedFieldType fieldType = context.smartNameFieldType(fieldName);
        if (fieldType == null) {
            if (unmappedType != null) {
                fieldType = context.mapperService().unmappedFieldType(unmappedType);
            } else {
                throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on", null);
            }
        }

        if (!fieldType.isSortable()) {
            throw new SearchParseException(context, "Sorting not supported for field[" + fieldName + "]", null);
        }

        // Enable when we also know how to detect fields that do tokenize, but only emit one token
        /*if (fieldMapper instanceof StringFieldMapper) {
            StringFieldMapper stringFieldMapper = (StringFieldMapper) fieldMapper;
            if (stringFieldMapper.fieldType().tokenized()) {
                // Fail early
                throw new SearchParseException(context, "Can't sort on tokenized string field[" + fieldName + "]");
            }
        }*/

        // We only support AVG and SUM on number based fields
        if (fieldType.isNumeric() == false && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) {
            sortMode = null;
        }
        if (sortMode == null) {
            sortMode = resolveDefaultSortMode(reverse);
        }

        final Nested nested;
        if (nestedHelper != null && nestedHelper.getPath() != null) {
            BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter());
            Query innerDocumentsFilter;
            if (nestedHelper.filterFound()) {
                // TODO: use queries instead
                innerDocumentsFilter = nestedHelper.getInnerFilter();
            } else {
                innerDocumentsFilter = nestedHelper.getNestedObjectMapper().nestedTypeFilter();
            }
            nested = new Nested(rootDocumentsFilter,  context.searcher().createNormalizedWeight(innerDocumentsFilter, false));
        } else {
            nested = null;
        }

        IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldType)
                .comparatorSource(missing, sortMode, nested);
        sortFields.add(new SortField(fieldType.names().indexName(), fieldComparatorSource, reverse));
    }
}
 
Example 14
Source Project: Elasticsearch   Source File: RandomScoreFunctionParser.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ScoreFunction parse(QueryParseContext parseContext, XContentParser parser) throws IOException, QueryParsingException {

    int seed = -1;

    String currentFieldName = null;
    XContentParser.Token token;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            currentFieldName = parser.currentName();
        } else if (token.isValue()) {
            if ("seed".equals(currentFieldName)) {
                if (token == XContentParser.Token.VALUE_NUMBER) {
                    if (parser.numberType() == XContentParser.NumberType.INT) {
                        seed = parser.intValue();
                    } else if (parser.numberType() == XContentParser.NumberType.LONG) {
                        seed = Longs.hashCode(parser.longValue());
                    } else {
                        throw new QueryParsingException(parseContext, "random_score seed must be an int, long or string, not '"
                                + token.toString() + "'");
                    }
                } else if (token == XContentParser.Token.VALUE_STRING) {
                    seed = parser.text().hashCode();
                } else {
                    throw new QueryParsingException(parseContext, "random_score seed must be an int/long or string, not '"
                            + token.toString() + "'");
                }
            } else {
                throw new QueryParsingException(parseContext, NAMES[0] + " query does not support [" + currentFieldName + "]");
            }
        }
    }

    final MappedFieldType fieldType = SearchContext.current().mapperService().smartNameFieldType("_uid");
    if (fieldType == null) {
        // mapper could be null if we are on a shard with no docs yet, so this won't actually be used
        return new RandomScoreFunction();
    }

    if (seed == -1) {
        seed = Longs.hashCode(parseContext.nowInMillis());
    }
    final ShardId shardId = SearchContext.current().indexShard().shardId();
    final int salt = (shardId.index().name().hashCode() << 10) | shardId.id();
    final IndexFieldData<?> uidFieldData = SearchContext.current().fieldData().getForField(fieldType);

    return new RandomScoreFunction(seed, salt, uidFieldData);
}
 
Example 15
Source Project: Elasticsearch   Source File: QueryParseContext.java    License: Apache License 2.0 4 votes vote down vote up
public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType mapper) {
    return indexQueryParser.fieldDataService.getForField(mapper);
}
 
Example 16
public BytesRefFieldComparatorSource(IndexFieldData<?> indexFieldData, Object missingValue, MultiValueMode sortMode, Nested nested) {
    this.indexFieldData = indexFieldData;
    this.sortMode = sortMode;
    this.missingValue = missingValue;
    this.nested = nested;
}
 
Example 17
Source Project: Elasticsearch   Source File: IndexIndexFieldData.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
        CircuitBreakerService breakerService, MapperService mapperService) {
    return new IndexIndexFieldData(index, fieldType.names());
}
 
Example 18
Source Project: Elasticsearch   Source File: BinaryDVIndexFieldData.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
    return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
}
 
Example 19
Source Project: Elasticsearch   Source File: GeoPointArrayIndexFieldData.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    return new GeoPointArrayIndexFieldData(index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache,
            breakerService);
}
 
Example 20
/**
 * Creates a new {@link FieldDataTermsQuery} from the given field data.
 */
public FieldDataTermsQuery(final byte[] encodedTerms, final IndexFieldData fieldData, final long cacheKey) {
  this.encodedTerms = encodedTerms;
  this.fieldData = fieldData;
  this.cacheKey = cacheKey;
}
 
Example 21
@Override
public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
  XContentParser parser = parseContext.parser();

  XContentParser.Token token = parser.nextToken();
  if (token != XContentParser.Token.FIELD_NAME) {
      throw new QueryParsingException(parseContext, "[fielddata_terms] a field name is required");
  }
  String fieldName = parser.currentName();

  String queryName = null;
  byte[] value = null;
  Long cacheKey = null;

  token = parser.nextToken();
  if (token == XContentParser.Token.START_OBJECT) {
    String currentFieldName = null;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
      if (token == XContentParser.Token.FIELD_NAME) {
        currentFieldName = parser.currentName();
      } else {
        if ("value".equals(currentFieldName)) {
          value = parser.binaryValue();
        } else if ("_name".equals(currentFieldName)) {
          queryName = parser.text();
        } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) {
          cacheKey = parser.longValue();
        } else {
          throw new QueryParsingException(parseContext, "[fielddata_terms] filter does not support [" + currentFieldName + "]");
        }
      }
    }
    parser.nextToken();
  } else {
    value = parser.binaryValue();
    // move to the next token
    parser.nextToken();
  }

  if (value == null) {
    throw new QueryParsingException(parseContext, "[fielddata_terms] a binary value is required");
  }
  if (cacheKey == null) { // cache key is mandatory - see #170
    throw new QueryParsingException(parseContext, "[fielddata_terms] a cache key is required");
  }

  if (fieldName == null) {
    throw new QueryParsingException(parseContext, "[fielddata_terms] a field name is required");
  }

  MappedFieldType fieldType = parseContext.fieldMapper(fieldName);
  if (fieldType == null) {
    return new MatchNoDocsQuery();
  }

  IndexFieldData fieldData = parseContext.getForField(fieldType);
  Query query = this.toFieldDataTermsQuery(fieldType, fieldData, value, cacheKey);

  if (queryName != null) {
    parseContext.addNamedQuery(queryName, query);
  }

  return query;
}
 
Example 22
/**
 * The operation that executes the query and generates a {@link TermsByQueryShardResponse} for each shard.
 */
@Override
protected TermsByQueryShardResponse shardOperation(TermsByQueryShardRequest shardRequest) throws ElasticsearchException {
  IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId().getIndex());
  IndexShard indexShard = indexService.shardSafe(shardRequest.shardId().id());
  TermsByQueryRequest request = shardRequest.request();
  OrderByShardOperation orderByOperation = OrderByShardOperation.get(request.getOrderBy(), request.maxTermsPerShard());

  SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(),
                                                        shardRequest.shardId().getIndex(),
                                                        shardRequest.shardId().id());

  ShardSearchRequest shardSearchRequest = new ShardSearchLocalRequest(request.types(), request.nowInMillis(),
                                                                      shardRequest.filteringAliases());

  SearchContext context = new DefaultSearchContext(0, shardSearchRequest, shardTarget,
    indexShard.acquireSearcher("termsByQuery"), indexService, indexShard, scriptService,
    pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
    SearchService.NO_TIMEOUT);
  SearchContext.setCurrent(context);

  try {
    MappedFieldType fieldType = context.smartNameFieldType(request.field());
    if (fieldType == null) {
      throw new SearchContextException(context, "[termsByQuery] field '" + request.field() +
              "' not found for types " + Arrays.toString(request.types()));
    }

    IndexFieldData indexFieldData = context.fieldData().getForField(fieldType);

    BytesReference querySource = request.querySource();
    if (querySource != null && querySource.length() > 0) {
      XContentParser queryParser = null;
      try {
        queryParser = XContentFactory.xContent(querySource).createParser(querySource);
        QueryParseContext.setTypes(request.types());
        ParsedQuery parsedQuery = orderByOperation.getParsedQuery(queryParser, indexService);
        if (parsedQuery != null) {
          context.parsedQuery(parsedQuery);
        }
      }
      finally {
        QueryParseContext.removeTypes();
        if (queryParser != null) {
          queryParser.close();
        }
      }
    }

    context.preProcess();

    // execute the search only gathering the hit count and bitset for each segment
    logger.debug("{}: Executes search for collecting terms {}", Thread.currentThread().getName(),
      shardRequest.shardId());

    TermsCollector termsCollector = this.getTermsCollector(request.termsEncoding(), indexFieldData, context);
    if (request.expectedTerms() != null) termsCollector.setExpectedTerms(request.expectedTerms());
    if (request.maxTermsPerShard() != null) termsCollector.setMaxTerms(request.maxTermsPerShard());
    HitStream hitStream = orderByOperation.getHitStream(context);
    TermsSet terms = termsCollector.collect(hitStream);

    logger.debug("{}: Returns terms response with {} terms for shard {}", Thread.currentThread().getName(),
      terms.size(), shardRequest.shardId());

    return new TermsByQueryShardResponse(shardRequest.shardId(), terms);
  }
  catch (Throwable e) {
    logger.error("[termsByQuery] Error executing shard operation", e);
    throw new QueryPhaseExecutionException(context, "[termsByQuery] Failed to execute query", e);
  }
  finally {
    // this will also release the index searcher
    context.close();
    SearchContext.removeCurrent();
  }
}
 
Example 23
public BloomFilterTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                                 final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 
Example 24
public IntegerTermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                             final CircuitBreaker breaker) {
  super(indexFieldData, context, breaker);
}
 
Example 25
public TermsCollector(final IndexFieldData indexFieldData, final SearchContext context,
                      final CircuitBreaker breaker) {
  this.indexFieldData = indexFieldData;
  this.context = context;
  this.breaker = breaker;
}
 
Example 26
protected BytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader);
  this.fieldData = indexFieldData;
}
 
Example 27
protected BytesBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 
Example 28
protected NumericBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 
Example 29
protected IntegerBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}
 
Example 30
protected LongBytesRefTermStream(IndexReader reader, IndexFieldData indexFieldData) {
  super(reader, indexFieldData);
}