Java Code Examples for org.apache.lucene.store.InputStreamDataInput

The following examples show how to use org.apache.lucene.store.InputStreamDataInput. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestFSTDirectAddressing.java    License: Apache License 2.0 6 votes vote down vote up
private static void recompileAndWalk(String fstFilePath) throws IOException {
  try (InputStreamDataInput in = new InputStreamDataInput(newInputStream(Paths.get(fstFilePath)))) {

    System.out.println("Reading FST");
    long startTimeMs = System.currentTimeMillis();
    FST<CharsRef> originalFst = new FST<>(in, in, CharSequenceOutputs.getSingleton());
    long endTimeMs = System.currentTimeMillis();
    System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");

    for (float oversizingFactor : List.of(0f, 0f, 0f, 1f, 1f, 1f)) {
      System.out.println("\nFST construction (oversizingFactor=" + oversizingFactor + ")");
      startTimeMs = System.currentTimeMillis();
      FST<CharsRef> fst = recompile(originalFst, oversizingFactor);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
      System.out.println("FST RAM = " + fst.ramBytesUsed() + " B");

      System.out.println("FST enum");
      startTimeMs = System.currentTimeMillis();
      walk(fst);
      endTimeMs = System.currentTimeMillis();
      System.out.println("time = " + (endTimeMs - startTimeMs) + " ms");
    }
  }
}
 
Example 2
Source Project: Elasticsearch   Source File: XAnalyzingSuggester.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    this.fst = new FST<>(dataIn, new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
    maxAnalyzedPathsForOneInput = dataIn.readVInt();
    hasPayloads = dataIn.readByte() == 1;
  } finally {
    IOUtils.close(input);
  }
  return true;
}
 
Example 3
Source Project: Elasticsearch   Source File: Completion090PostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void parsePayload(BytesRef payload, SuggestPayload ref) throws IOException {
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(payload.bytes, payload.offset, payload.length);
    InputStreamDataInput input = new InputStreamDataInput(byteArrayInputStream);
    ref.weight = input.readVLong() - 1;
    int len = input.readVInt();
    ref.surfaceForm.grow(len);
    ref.surfaceForm.setLength(len);
    input.readBytes(ref.surfaceForm.bytes(), 0, ref.surfaceForm.length());
    len = input.readVInt();
    ref.payload.grow(len);
    ref.payload.setLength(len);
    input.readBytes(ref.payload.bytes(), 0, ref.payload.length());
    input.close();
}
 
Example 4
Source Project: lucene-solr   Source File: Connection.java    License: Apache License 2.0 5 votes vote down vote up
public Connection(int tcpPort) throws IOException {
  this.destTCPPort = tcpPort;
  this.s = new Socket(InetAddress.getLoopbackAddress(), tcpPort);
  this.sockIn = s.getInputStream();
  this.in = new InputStreamDataInput(sockIn);
  this.bos = new BufferedOutputStream(s.getOutputStream());
  this.out = new OutputStreamDataOutput(bos);
  if (Node.VERBOSE_CONNECTIONS) {
    System.out.println("make new client Connection socket=" + this.s + " destPort=" + tcpPort);
  }
}
 
Example 5
Source Project: lucene-solr   Source File: Lookup.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calls {@link #load(DataInput)} after converting
 * {@link InputStream} to {@link DataInput}
 */
public boolean load(InputStream input) throws IOException {
  DataInput dataIn = new InputStreamDataInput(input);
  try {
    return load(dataIn);
  } finally {
    IOUtils.close(input);
  }
}
 
Example 6
Source Project: lucene-solr   Source File: TokenInfoDictionary.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  // TODO: some way to configure?
  this.fst = new TokenInfoFST(fst, true);
}
 
Example 7
Source Project: lucene-solr   Source File: TokenInfoDictionary.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param resourceScheme - scheme for loading resources (FILE or CLASSPATH).
 * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
 * this class's name as the path.
 */
public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
  super(resourceScheme, resourcePath);
  FST<Long> fst;
  try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {
    DataInput in = new InputStreamDataInput(is);
    fst = new FST<>(in, in, PositiveIntOutputs.getSingleton());
  }
  this.fst = new TokenInfoFST(fst);
}
 
Example 8
Source Project: lucene-solr   Source File: FST.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads an automaton from a file. 
 */
public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
  try (InputStream is = Files.newInputStream(path)) {
    DataInput in = new InputStreamDataInput(new BufferedInputStream(is));
    return new FST<>(in, in, outputs);
  }
}
 
Example 9
public FstDecompounder(InputStream inputStream, List<String> glue) throws IOException {
    try {
        this.surfaceForms = new FST<>(new InputStreamDataInput(inputStream), NoOutputs.getSingleton());
        // set up glue morphemes
        this.glueMorphemes = createGlueMorphemes(glue != null && glue.size() > 0 ? glue :morphemes);
    } finally {
        inputStream.close();
    }
}
 
Example 10
Source Project: Elasticsearch   Source File: Checkpoint.java    License: Apache License 2.0 4 votes vote down vote up
public static Checkpoint read(Path path) throws IOException {
    try (InputStream in = Files.newInputStream(path)) {
        return new Checkpoint(new InputStreamDataInput(in));
    }
}
 
Example 11
Source Project: lucene-solr   Source File: SimpleServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  boolean success = false;
  try {
    //node.message("using stream buffer size=" + bufferSize);
    InputStream is = new BufferedInputStream(socket.getInputStream(), bufferSize);
    DataInput in = new InputStreamDataInput(is);
    BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize);
    DataOutput out = new OutputStreamDataOutput(bos);

    if (node instanceof SimplePrimaryNode) {
      ((SimplePrimaryNode) node).handleOneConnection(random(), ss, stop, is, socket, in, out, bos);
    } else {
      ((SimpleReplicaNode) node).handleOneConnection(ss, stop, is, socket, in, out, bos);
    }

    bos.flush();
    if (Node.VERBOSE_CONNECTIONS) {
      node.message("bos.flush done");
    }

    success = true;
  } catch (Throwable t) {
    if (t instanceof SocketException == false && t instanceof NodeCommunicationException == false) {
      node.message("unexpected exception handling client connection; now failing test:");
      t.printStackTrace(System.out);
      IOUtils.closeWhileHandlingException(ss);
      // Test should fail with this:
      throw new RuntimeException(t);
    } else {
      node.message("exception handling client connection; ignoring:");
      t.printStackTrace(System.out);
    }
  } finally {
    if (success) {
      try {
        IOUtils.close(socket);
      } catch (IOException ioe) {
        throw new RuntimeException(ioe);
      }
    } else {
      IOUtils.closeWhileHandlingException(socket);
    }
  }
  if (Node.VERBOSE_CONNECTIONS) {
    node.message("socket.close done");
  }
}