org.apache.lucene.util.AttributeImpl Java Examples

The following examples show how to use org.apache.lucene.util.AttributeImpl. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexTimeSynonymTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public TokenStreamComponents createComponents(String fieldName) {
  Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {
    final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
    int p = 0;
    
    @Override
    public boolean incrementToken() {
      if( p >= tokens.length ) return false;
      clearAttributes();
      tokens[p++].copyTo(reusableToken);
      return true;
    }

    @Override
    public void reset() throws IOException {
      super.reset();
      this.p = 0;
    }
  };
  return new TokenStreamComponents(ts);
}
 
Example #2
Source File: Retriever.java    From lucene4ir with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the list of tokens extracted from the query string using the specified analyzer.
 *
 * @param field document field.
 *
 * @param queryTerms query string.
 *
 * @param distinctTokens if true, return the distinct tokens in the query string.
 *
 * @return the list of tokens extracted from the given query.
 *
 * @throws IOException
 */
List<String> getTokens(String field, String queryTerms, boolean distinctTokens) throws IOException {

    List<String> tokens = new ArrayList<String>();

    StringReader topicTitleReader = new StringReader(queryTerms);

    Set<String> seenTokens = new TreeSet<String>();

    TokenStream tok;
    tok = analyzer.tokenStream(field, topicTitleReader);
    tok.reset();
    while (tok.incrementToken()) {
        Iterator<AttributeImpl> atts = tok.getAttributeImplsIterator();
        AttributeImpl token = atts.next();
        String text = "" + token;
        if (seenTokens.contains(text) && distinctTokens) {
            continue;
        }
        seenTokens.add(text);
        tokens.add(text);
    }
    tok.close();

    return tokens;
}
 
Example #3
Source File: PackedTokenAttributeImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  if (target instanceof PackedTokenAttributeImpl) {
    final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target;
    to.copyBuffer(buffer(), 0, length());
    to.positionIncrement = positionIncrement;
    to.positionLength = positionLength;
    to.startOffset = startOffset;
    to.endOffset = endOffset;
    to.type = type;
    to.termFrequency = termFrequency;
  } else {
    super.copyTo(target);
    ((OffsetAttribute) target).setOffset(startOffset, endOffset);
    ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
    ((PositionLengthAttribute) target).setPositionLength(positionLength);
    ((TypeAttribute) target).setType(type);
    ((TermFrequencyAttribute) target).setTermFrequency(termFrequency);
  }
}
 
Example #4
Source File: UniqueFieldAttributeImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {

  if (!(target instanceof UniqueFieldAttributeImpl)) {
    throw new IllegalArgumentException(
        "cannot copy the values from attribute UniqueFieldAttribute to an instance of "
            + target.getClass().getName());
  }

  UniqueFieldAttributeImpl uniqueFieldAttr = (UniqueFieldAttributeImpl) target;
  uniqueFieldAttr.uniqueField = uniqueField.toString();

}
 
Example #5
Source File: TestBytesRefAttImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static <T extends AttributeImpl> T assertCopyIsEqual(T att) throws Exception {
  @SuppressWarnings("unchecked")
  T copy = (T) att.getClass().getConstructor().newInstance();
  att.copyTo(copy);
  assertEquals("Copied instance must be equal", att, copy);
  assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode());
  return copy;
}
 
Example #6
Source File: TestCharTermAttributeImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static <T extends AttributeImpl> T assertCopyIsEqual(T att) throws Exception {
  @SuppressWarnings("unchecked")
  T copy = (T) att.getClass().getConstructor().newInstance();
  att.copyTo(copy);
  assertEquals("Copied instance must be equal", att, copy);
  assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode());
  return copy;
}
 
Example #7
Source File: TestCharTermAttributeImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static <T extends AttributeImpl> T assertCloneIsEqual(T att) {
  @SuppressWarnings("unchecked")
  T clone = (T) att.clone();
  assertEquals("Clone must be equal", att, clone);
  assertEquals("Clone's hashcode must be equal", att.hashCode(), clone.hashCode());
  return clone;
}
 
Example #8
Source File: AnalysisRequestHandlerBase.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected void addAttributes(AttributeSource attributeSource) {
  // note: ideally we wouldn't call addAttributeImpl which is marked internal. But nonetheless it's possible
  //  this method is used by some custom attributes, especially since Solr doesn't provide a way to customize the
  //  AttributeFactory which is the recommended way to choose which classes implement which attributes.
  Iterator<AttributeImpl> atts = attributeSource.getAttributeImplsIterator();
  while (atts.hasNext()) {
    addAttributeImpl(atts.next()); // adds both impl & interfaces
  }
}
 
Example #9
Source File: Test2BTerms.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
  if (attClass == TermToBytesRefAttribute.class)
    return new MyTermAttributeImpl();
  if (CharTermAttribute.class.isAssignableFrom(attClass))
    throw new IllegalArgumentException("no");
  return delegate.createAttributeInstance(attClass);
}
 
Example #10
Source File: TestNumericTokenStream.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static <T extends AttributeImpl> T assertCloneIsEqual(T att) {
  @SuppressWarnings("unchecked")
  T clone = (T) att.clone();
  assertEquals("Clone must be equal", att, clone);
  assertEquals("Clone's hashcode must be equal", att.hashCode(), clone.hashCode());
  return clone;
}
 
Example #11
Source File: TestNumericTokenStream.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static <T extends AttributeImpl> T assertCopyIsEqual(T att) throws Exception {
  @SuppressWarnings("unchecked")
  T copy = (T) att.getClass().getConstructor().newInstance();
  att.copyTo(copy);
  assertEquals("Copied instance must be equal", att, copy);
  assertEquals("Copied instance's hashcode must be equal", att.hashCode(), copy.hashCode());
  return copy;
}
 
Example #12
Source File: MorphosyntacticTagsAttributeImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  List<StringBuilder> cloned = null;
  if (tags != null) {
    cloned = new ArrayList<>(tags.size());
    for (StringBuilder b : tags) {
      cloned.add(new StringBuilder(b));
    }
  }
  ((MorphosyntacticTagsAttribute) target).setTags(cloned);
}
 
Example #13
Source File: AutoPhrasingTokenFilter.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
private CharTermAttribute getTermAttribute() {
    Iterator<AttributeImpl> attrIt = getAttributeImplsIterator();
    while (attrIt != null && attrIt.hasNext()) {
        AttributeImpl attrImp = attrIt.next();
        if (attrImp instanceof CharTermAttribute) {
            return (CharTermAttribute) attrImp;
        }
    }
    return null;
}
 
Example #14
Source File: AutoPhrasingTokenFilter.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
private OffsetAttribute getOffsetAttribute() {
    Iterator<AttributeImpl> attrIt = getAttributeImplsIterator();
    while (attrIt != null && attrIt.hasNext()) {
        AttributeImpl attrImp = attrIt.next();
        if (attrImp instanceof OffsetAttribute) {
            return (OffsetAttribute) attrImp;
        }
    }
    return null;
}
 
Example #15
Source File: AutoPhrasingTokenFilter.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
private PositionIncrementAttribute getPositionIncrementAttribute() {
    Iterator<AttributeImpl> attrIt = getAttributeImplsIterator();
    while (attrIt != null && attrIt.hasNext()) {
        AttributeImpl attrImp = attrIt.next();
        if (attrImp instanceof PositionIncrementAttribute) {
            return (PositionIncrementAttribute) attrImp;
        }
    }
    return null;
}
 
Example #16
Source File: AnalysisImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private List<TokenAttribute> copyAttributes(TokenStream tokenStream, CharTermAttribute charAtt) {
  List<TokenAttribute> attributes = new ArrayList<>();
  Iterator<AttributeImpl> itr = tokenStream.getAttributeImplsIterator();
  while(itr.hasNext()) {
    AttributeImpl att = itr.next();
    Map<String, String> attValues = new LinkedHashMap<>();
    att.reflectWith((attClass, key, value) -> {
      if (value != null)
        attValues.put(key, value.toString());
    });
    attributes.add(new TokenAttribute(att.getClass().getSimpleName(), attValues));
  }
  return attributes;
}
 
Example #17
Source File: NumericTokenizer.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/** Make this tokenizer get attributes from the delegate token stream. */
private static final AttributeFactory delegatingAttributeFactory(final AttributeSource source) {
    return new AttributeFactory() {
        @Override
        public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
            return (AttributeImpl) source.addAttribute(attClass);
        }
    };
}
 
Example #18
Source File: OffsetAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  OffsetAttribute t = (OffsetAttribute) target;
  t.setOffset(startOffset, endOffset);
}
 
Example #19
Source File: Test2BTerms.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  throw new UnsupportedOperationException();
}
 
Example #20
Source File: TypeAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  TypeAttribute t = (TypeAttribute) target;
  t.setType(type);
}
 
Example #21
Source File: PositionIncrementAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  PositionIncrementAttribute t = (PositionIncrementAttribute) target;
  t.setPositionIncrement(positionIncrement);
}
 
Example #22
Source File: CharTermAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  CharTermAttribute t = (CharTermAttribute) target;
  t.copyBuffer(termBuffer, 0, termLength);
}
 
Example #23
Source File: AnalysisRequestHandlerBase.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  final TokenTrackingAttribute t = (TokenTrackingAttribute) target;
  t.reset(basePositions, position);
}
 
Example #24
Source File: TestToken.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {}
 
Example #25
Source File: LegacyNumericTokenStream.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
  if (CharTermAttribute.class.isAssignableFrom(attClass))
    throw new IllegalArgumentException("LegacyNumericTokenStream does not support CharTermAttribute.");
  return delegate.createAttributeInstance(attClass);
}
 
Example #26
Source File: LegacyNumericTokenStream.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  final LegacyNumericTermAttribute a = (LegacyNumericTermAttribute) target;
  a.init(value, valueSize, precisionStep, shift);
}
 
Example #27
Source File: TaggingAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  ((TaggingAttribute) target).setTaggable(taggable);
}
 
Example #28
Source File: BytesTermAttributeImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  BytesTermAttributeImpl other = (BytesTermAttributeImpl) target;
  other.bytes = bytes == null ? null : BytesRef.deepCopyOf(bytes);
}
 
Example #29
Source File: Token.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  super.copyTo(target);
  ((FlagsAttribute) target).setFlags(flags);
  ((PayloadAttribute) target).setPayload((payload == null) ? null : BytesRef.deepCopyOf(payload));
}
 
Example #30
Source File: POSAttributeImpl.java    From fnlp with GNU Lesser General Public License v3.0 4 votes vote down vote up
@Override
public void copyTo(AttributeImpl target) {
  ((POSAttribute) target).setPartOfSpeech(pos);
}