Java Code Examples for org.apache.lucene.util.fst.FST#getBytesReader()

The following examples show how to use org.apache.lucene.util.fst.FST#getBytesReader() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSTTermsReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
static<T> void walk(FST<T> fst) throws IOException {
  final ArrayList<FST.Arc<T>> queue = new ArrayList<>();
  final BitSet seen = new BitSet();
  final FST.BytesReader reader = fst.getBytesReader();
  final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
  queue.add(startArc);
  while (!queue.isEmpty()) {
    final FST.Arc<T> arc = queue.remove(0);
    final long node = arc.target();
    //System.out.println(arc);
    if (FST.targetHasArcs(arc) && !seen.get((int) node)) {
      seen.set((int) node);
      fst.readFirstRealTargetArc(node, arc, reader);
      while (true) {
        queue.add(new FST.Arc<T>().copyFrom(arc));
        if (arc.isLast()) {
          break;
        } else {
          fst.readNextRealArc(arc, reader);
        }
      }
    }
  }
}
Example 2
Source File: FSTCompletion.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up 
/**
 * Cache the root node's output arcs starting with completions with the
 * highest weights.
 */
@SuppressWarnings({"unchecked","rawtypes"})
private static Arc<Object>[] cacheRootArcs(FST<Object> automaton) {
  try {
    List<Arc<Object>> rootArcs = new ArrayList<>();
    Arc<Object> arc = automaton.getFirstArc(new Arc<>());
    FST.BytesReader fstReader = automaton.getBytesReader();
    automaton.readFirstTargetArc(arc, arc, fstReader);
    while (true) {
      rootArcs.add(new Arc<>().copyFrom(arc));
      if (arc.isLast()) break;
      automaton.readNextArc(arc, fstReader);
    }
    
    Collections.reverse(rootArcs); // we want highest weights first.
    return rootArcs.toArray(new Arc[rootArcs.size()]);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
Example 3
Source File: FSTCursor.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up 
public FSTCursor(FST<Long> fst, int characterStart) {
  this.fst = fst;
  arc = fst.getFirstArc(new FST.Arc<Long>());
  output = fst.outputs.getNoOutput();
  fstReader = fst.getBytesReader();
  this.characterStart = characterStart;
}
Example 4
Source File: Dictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
IntsRef lookup(FST<IntsRef> fst, char word[], int offset, int length) {
  if (fst == null) {
    return null;
  }
  final FST.BytesReader bytesReader = fst.getBytesReader();
  final FST.Arc<IntsRef> arc = fst.getFirstArc(new FST.Arc<IntsRef>());
  // Accumulate output as we go
  final IntsRef NO_OUTPUT = fst.outputs.getNoOutput();
  IntsRef output = NO_OUTPUT;
  
  int l = offset + length;
  try {
    for (int i = offset, cp = 0; i < l; i += Character.charCount(cp)) {
      cp = Character.codePointAt(word, i, l);
      if (fst.findTargetArc(cp, arc, arc, bytesReader) == null) {
        return null;
      } else if (arc.output() != NO_OUTPUT) {
        output = fst.outputs.add(output, arc.output());
      }
    }
    if (fst.findTargetArc(FST.END_LABEL, arc, arc, bytesReader) == null) {
      return null;
    } else if (arc.output() != NO_OUTPUT) {
      return fst.outputs.add(output, arc.output());
    } else {
      return output;
    }
  } catch (IOException bogus) {
    throw new RuntimeException(bogus);
  }
}
Example 5
Source File: Dictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up 
static void applyMappings(FST<CharsRef> fst, StringBuilder sb) throws IOException {
  final FST.BytesReader bytesReader = fst.getBytesReader();
  final FST.Arc<CharsRef> firstArc = fst.getFirstArc(new FST.Arc<CharsRef>());
  final CharsRef NO_OUTPUT = fst.outputs.getNoOutput();
  
  // temporary stuff
  final FST.Arc<CharsRef> arc = new FST.Arc<>();
  int longestMatch;
  CharsRef longestOutput;
  
  for (int i = 0; i < sb.length(); i++) {
    arc.copyFrom(firstArc);
    CharsRef output = NO_OUTPUT;
    longestMatch = -1;
    longestOutput = null;
    
    for (int j = i; j < sb.length(); j++) {
      char ch = sb.charAt(j);
      if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) {
        break;
      } else {
        output = fst.outputs.add(output, arc.output());
      }
      if (arc.isFinal()) {
        longestOutput = fst.outputs.add(output, arc.nextFinalOutput());
        longestMatch = j;
      }
    }
    
    if (longestMatch >= 0) {
      sb.delete(i, longestMatch+1);
      sb.insert(i, longestOutput);
      i += (longestOutput.length - 1);
    }
  }
}