org.apache.lucene.store.InputStreamDataInput Java Examples

The following examples show how to use org.apache.lucene.store.InputStreamDataInput. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFSTDirectAddressing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void recompileAndWalk(String fstFilePath) throws IOException {
  try (InputStreamDataInput in = new InputStreamDataInput(newInputStream(Paths.get(fstFilePath)))) {

    System.out.println("Reading FST");
    long startTimeMs = System.currentTimeMillis();
    FST<CharsRef> originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
    long endTimeMs = System.currentTimeMillis();
    System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");

    for (float oversizingFactor : List.of(0f, 0f, 0f, 1f, 1f, 1f)) {
      System.out.println("\nFST construction (oversizingFactor=" + oversizingFactor + ")");
      startTimeMs = System.currentTimeMillis();
      FST<CharsRef> fst = recompile(originalFst, oversizingFactor);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
      System.out.println("FST RAM = " + fst.ramBytesUsed() + " B");

      System.out.println("FST enum");
      startTimeMs = System.currentTimeMillis();
      walk(fst);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
    }
  }
}
 
Example #2
Source File: XAnalyzingSuggester.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
    maxAnalyzedPathsForOneInput = dataIn.readVInt();
    hasPayloads = dataIn.readByte() == 1;
  } finally {
    IOUtils.close(input);
  }
  return true;
}
 
Example #3
Source File: Completion090PostingsFormat.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException {
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(payload.bytes, payload.offset, payload.length);
    InputStreamDataInput input = new InputStreamDataInput(byteArrayInputStream);
    ref.weight = input.readVLong() - 1;
    int len = input.readVInt();
    ref.surfaceForm.grow(len);
    ref.surfaceForm.setLength(len);
    input.readBytes(ref.surfaceForm.bytes(), 0, ref.surfaceForm.length());
    len = input.readVInt();
    ref.payload.grow(len);
    ref.payload.setLength(len);
    input.readBytes(ref.payload.bytes(), 0, ref.payload.length());
    input.close();
}
 
Example #4
Source File: Connection.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public Connection(int tcpPort) throws IOException {
  this.destTCPPort = tcpPort;
  this.s = new Socket(InetAddress.getLoopbackAddress(), tcpPort);
  this.sockIn = s.getInputStream();
  this.in = new InputStreamDataInput(sockIn);
  this.bos = new BufferedOutputStream(s.getOutputStream());
  this.out = new OutputStreamDataOutput(bos);
  if (Node.VERBOSE_CONNECTIONS) {
    System.out.println("make new client Connection socket=" + this.s + " destPort=" + tcpPort);
  }
}
 
Example #5
Source File: Lookup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Calls {@link #load(DataInput)} after converting
 * {@link InputStream} to {@link DataInput}
 */
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    return load(dataIn);
  } finally {
    IOUtils.close(input);
  }
}
 
Example #6
Source File: TokenInfoDictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  // TODO: some way to configure?
  this.fst = new TokenInfoFST(fst, true);
}
 
Example #7
Source File: TokenInfoDictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  this.fst = new TokenInfoFST(fst);
}
 
Example #8
Source File: FST.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Reads an automaton from a file. 
 */
public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
  try (InputStream is = Files.newInputStream(path)) {
    DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
    return new FST<>(in, in, outputs);
  }
}
 
Example #9
Source File: FstDecompounder.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
public FstDecompounder(InputStream inputStream, List<String> glue) throws IOException {
    try {
        this.surfaceForms = new FST<>(new InputStreamDataInput(inputStream), NoOutputs.getSingleton());
        // set up glue morphemes
        this.glueMorphemes = createGlueMorphemes(glue != null && glue.size() > 0 ? glue :morphemes);
    } finally {
        inputStream.close();
    }
}
 
Example #10
Source File: Checkpoint.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public static Checkpoint read(Path path) throws IOException {
    try (InputStream in = Files.newInputStream(path)) {
        return new Checkpoint(new InputStreamDataInput(in));
    }
}
 
Example #11
Source File: SimpleServer.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  boolean success = false;
  try {
    //node.message("using stream buffer size=" + bufferSize);
    InputStream is = new BufferedInputStream(socket.getInputStream(), bufferSize);
    DataInput in = new InputStreamDataInput(is);
    BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize);
    DataOutput out = new OutputStreamDataOutput(bos);

    if (node instanceof SimplePrimaryNode) {
      ((SimplePrimaryNode) node).handleOneConnection(random(), ss, stop, is, socket, in, out, bos);
    } else {
      ((SimpleReplicaNode) node).handleOneConnection(ss, stop, is, socket, in, out, bos);
    }

    bos.flush();
    if (Node.VERBOSE_CONNECTIONS) {
      node.message("bos.flush done");
    }

    success = true;
  } catch (Throwable t) {
    if (t instanceof SocketException == false && t instanceof NodeCommunicationException == false) {
      node.message("unexpected exception handling client connection; now failing test:");
      t.printStackTrace(System.out);
      IOUtils.closeWhileHandlingException(ss);
      // Test should fail with this:
      throw new RuntimeException(t);
    } else {
      node.message("exception handling client connection; ignoring:");
      t.printStackTrace(System.out);
    }
  } finally {
    if (success) {
      try {
        IOUtils.close(socket);
      } catch (IOException ioe) {
        throw new RuntimeException(ioe);
      }
    } else {
      IOUtils.closeWhileHandlingException(socket);
    }
  }
  if (Node.VERBOSE_CONNECTIONS) {
    node.message("socket.close done");
  }
}