org.apache.lucene.store.OutputStreamDataOutput Java Examples

The following examples show how to use org.apache.lucene.store.OutputStreamDataOutput. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: XAnalyzingSuggester.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    if (fst == null) {
      return false;
    }

    fst.save(dataOut);
    dataOut.writeVInt(maxAnalyzedPathsForOneInput);
    dataOut.writeByte((byte) (hasPayloads ? 1 : 0));
  } finally {
    IOUtils.close(output);
  }
  return true;
}
 
Example #2
Source File: Completion090PostingsFormat.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public BytesRef buildPayload(BytesRef surfaceForm, long weight, BytesRef payload) throws IOException {
    if (weight < -1 || weight > Integer.MAX_VALUE) {
        throw new IllegalArgumentException("weight must be >= -1 && <= Integer.MAX_VALUE");
    }
    for (int i = 0; i < surfaceForm.length; i++) {
        if (surfaceForm.bytes[i] == UNIT_SEPARATOR) {
            throw new IllegalArgumentException(
                    "surface form cannot contain unit separator character U+001F; this character is reserved");
        }
    }
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream);
    output.writeVLong(weight + 1);
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(payload.length);
    output.writeBytes(payload.bytes, 0, payload.length);

    output.close();
    return new BytesRef(byteArrayOutputStream.toByteArray());
}
 
Example #3
Source File: TranslogWriter.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static TranslogWriter create(Type type, ShardId shardId, String translogUUID, long fileGeneration, Path file, Callback<ChannelReference> onClose, int bufferSize, ChannelFactory channelFactory) throws IOException {
    final BytesRef ref = new BytesRef(translogUUID);
    final int headerLength = getHeaderLength(ref.length);
    final FileChannel channel = channelFactory.open(file);
    try {
        // This OutputStreamDataOutput is intentionally not closed because
        // closing it will close the FileChannel
        final OutputStreamDataOutput out = new OutputStreamDataOutput(java.nio.channels.Channels.newOutputStream(channel));
        CodecUtil.writeHeader(out, TRANSLOG_CODEC, VERSION);
        out.writeInt(ref.length);
        out.writeBytes(ref.bytes, ref.offset, ref.length);
        channel.force(true);
        writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE);
        final TranslogWriter writer = type.create(shardId, fileGeneration, new ChannelReference(file, fileGeneration, channel, onClose), bufferSize);
        return writer;
    } catch (Throwable throwable){
        // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
        // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition
        IOUtils.closeWhileHandlingException(channel);
        throw throwable;
    }
}
 
Example #4
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example #5
Source File: CharacterDefinitionWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example #6
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 2) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 2 characters");
        }
        out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
      }
    }
  }
}
 
Example #7
Source File: TestSuggestField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testTokenStream() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  SuggestField suggestField = new SuggestField("field", "input", 1);
  BytesRef surfaceForm = new BytesRef("input");
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(1 + 1);
    output.writeByte(SuggestField.TYPE);
  }
  BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
  TokenStream stream = new PayloadAttrToTypeAttrFilter(suggestField.tokenStream(analyzer, null));
  assertTokenStreamContents(stream, new String[] {"input"}, null, null, new String[]{payload.utf8ToString()}, new int[]{1}, null, null);

  CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
  stream = new PayloadAttrToTypeAttrFilter(suggestField.tokenStream(completionAnalyzer, null));
  assertTokenStreamContents(stream, new String[] {"input"}, null, null, new String[]{payload.utf8ToString()}, new int[]{1}, null, null);
}
 
Example #8
Source File: ConnectionCostsWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Files.createDirectories(baseDir);
  String fileName = ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
  try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
    out.writeVInt(forwardSize);
    out.writeVInt(backwardSize);
    int last = 0;
    for (int i = 0; i < costs.limit() / 2; i++) {
      short cost = costs.getShort(i * 2);
      int delta = (int) cost - last;
      out.writeZInt(delta);
      last = cost;
    }
  }
}
 
Example #9
Source File: BinaryDictionaryWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void writePosDict(Path path) throws IOException {
  Files.createDirectories(path.getParent());
  try (OutputStream os = Files.newOutputStream(path);
       OutputStream bos = new BufferedOutputStream(os)) {
    final DataOutput out = new OutputStreamDataOutput(bos);
    CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
    out.writeVInt(posDict.size());
    for (String s : posDict) {
      if (s == null) {
        out.writeByte((byte)0);
        out.writeByte((byte)0);
        out.writeByte((byte)0);
      } else {
        String[] data = CSVUtil.parse(s);
        if (data.length != 3) {
          throw new IllegalArgumentException("Malformed pos/inflection: " + s + "; expected 3 characters");
        }
        out.writeString(data[0]);
        out.writeString(data[1]);
        out.writeString(data[2]);
      }
    }
  }
}
 
Example #10
Source File: CharacterDefinitionWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void write(Path baseDir) throws IOException {
  Path path = baseDir.resolve(CharacterDefinition.class.getName().replace('.', '/') + CharacterDefinition.FILENAME_SUFFIX);
  Files.createDirectories(path.getParent());
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))){
    final DataOutput out = new OutputStreamDataOutput(os);
    CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
    out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
    for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
      final byte b = (byte) (
        (invokeMap[i] ? 0x01 : 0x00) | 
        (groupMap[i] ? 0x02 : 0x00)
      );
      out.writeByte(b);
    }
  }
}
 
Example #11
Source File: TestContextSuggestField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testTokenStream() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random());
  ContextSuggestField field = new ContextSuggestField("field", "input", 1, "context1", "context2");
  BytesRef surfaceForm = new BytesRef("input");
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(1 + 1);
    output.writeByte(ContextSuggestField.TYPE);
  }
  BytesRef payload = new BytesRef(byteArrayOutputStream.toByteArray());
  String[] expectedOutputs = new String[2];
  CharsRefBuilder builder = new CharsRefBuilder();
  builder.append("context1");
  builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
  builder.append((char) ConcatenateGraphFilter.SEP_LABEL);
  builder.append("input");
  expectedOutputs[0] = builder.toCharsRef().toString();
  builder.clear();
  builder.append("context2");
  builder.append(((char) ContextSuggestField.CONTEXT_SEPARATOR));
  builder.append((char) ConcatenateGraphFilter.SEP_LABEL);
  builder.append("input");
  expectedOutputs[1] = builder.toCharsRef().toString();
  TokenStream stream = new TestSuggestField.PayloadAttrToTypeAttrFilter(field.tokenStream(analyzer, null));
  assertTokenStreamContents(stream, expectedOutputs, null, null, new String[]{payload.utf8ToString(), payload.utf8ToString()}, new int[]{1, 0}, null, null);

  CompletionAnalyzer completionAnalyzer = new CompletionAnalyzer(analyzer);
  stream = new TestSuggestField.PayloadAttrToTypeAttrFilter(field.tokenStream(completionAnalyzer, null));
  assertTokenStreamContents(stream, expectedOutputs, null, null, new String[]{payload.utf8ToString(), payload.utf8ToString()}, new int[]{1, 0}, null, null);
}
 
Example #12
Source File: FST.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Writes an automaton to a file. 
 */
public void save(final Path path) throws IOException {
  try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {
    DataOutput out = new OutputStreamDataOutput(os);
    save(out, out);
  }
}
 
Example #13
Source File: PushdownLargeFieldedListsVisitor.java    From datawave with Apache License 2.0 5 votes vote down vote up
protected URI createFst(SortedSet<String> values) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException {
    FST fst = DatawaveFieldIndexListIteratorJexl.getFST(values);
    
    // now serialize to our file system
    CompressionCodec codec = null;
    String extension = "";
    if (config.getHdfsFileCompressionCodec() != null) {
        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
        if (classLoader == null) {
            classLoader = this.getClass().getClassLoader();
        }
        Class<? extends CompressionCodec> clazz = Class.forName(config.getHdfsFileCompressionCodec(), true, classLoader).asSubclass(CompressionCodec.class);
        codec = clazz.newInstance();
        extension = codec.getDefaultExtension();
    }
    int fstCount = config.getFstCount().incrementAndGet();
    Path fstFile = new Path(fstHdfsUri, "PushdownLargeFileFst." + fstCount + ".fst" + extension);
    
    OutputStream fstFileOut = new BufferedOutputStream(fs.create(fstFile, false));
    if (codec != null) {
        fstFileOut = codec.createOutputStream(fstFileOut);
    }
    
    OutputStreamDataOutput outStream = new OutputStreamDataOutput(fstFileOut);
    fst.save(outStream);
    outStream.close();
    
    return fstFile.toUri();
}
 
Example #14
Source File: SuggestField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private BytesRef buildSuggestPayload() {
  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
  try (OutputStreamDataOutput output = new OutputStreamDataOutput(byteArrayOutputStream)) {
    output.writeVInt(surfaceForm.length);
    output.writeBytes(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length);
    output.writeVInt(weight + 1);
    output.writeByte(type());
  } catch (IOException e) {
    throw new RuntimeException(e); // not possible, it's a ByteArrayOutputStream!
  }
  return new BytesRef(byteArrayOutputStream.toByteArray());
}
 
Example #15
Source File: Lookup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Calls {@link #store(DataOutput)} after converting
 * {@link OutputStream} to {@link DataOutput}
 */
public boolean store(OutputStream output) throws IOException {
  DataOutput dataOut = new OutputStreamDataOutput(output);
  try {
    return store(dataOut);
  } finally {
    IOUtils.close(output);
  }
}
 
Example #16
Source File: Connection.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public Connection(int tcpPort) throws IOException {
  this.destTCPPort = tcpPort;
  this.s = new Socket(InetAddress.getLoopbackAddress(), tcpPort);
  this.sockIn = s.getInputStream();
  this.in = new InputStreamDataInput(sockIn);
  this.bos = new BufferedOutputStream(s.getOutputStream());
  this.out = new OutputStreamDataOutput(bos);
  if (Node.VERBOSE_CONNECTIONS) {
    System.out.println("make new client Connection socket=" + this.s + " destPort=" + tcpPort);
  }
}
 
Example #17
Source File: SimpleServer.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  boolean success = false;
  try {
    //node.message("using stream buffer size=" + bufferSize);
    InputStream is = new BufferedInputStream(socket.getInputStream(), bufferSize);
    DataInput in = new InputStreamDataInput(is);
    BufferedOutputStream bos = new BufferedOutputStream(socket.getOutputStream(), bufferSize);
    DataOutput out = new OutputStreamDataOutput(bos);

    if (node instanceof SimplePrimaryNode) {
      ((SimplePrimaryNode) node).handleOneConnection(random(), ss, stop, is, socket, in, out, bos);
    } else {
      ((SimpleReplicaNode) node).handleOneConnection(ss, stop, is, socket, in, out, bos);
    }

    bos.flush();
    if (Node.VERBOSE_CONNECTIONS) {
      node.message("bos.flush done");
    }

    success = true;
  } catch (Throwable t) {
    if (t instanceof SocketException == false && t instanceof NodeCommunicationException == false) {
      node.message("unexpected exception handling client connection; now failing test:");
      t.printStackTrace(System.out);
      IOUtils.closeWhileHandlingException(ss);
      // Test should fail with this:
      throw new RuntimeException(t);
    } else {
      node.message("exception handling client connection; ignoring:");
      t.printStackTrace(System.out);
    }
  } finally {
    if (success) {
      try {
        IOUtils.close(socket);
      } catch (IOException ioe) {
        throw new RuntimeException(ioe);
      }
    } else {
      IOUtils.closeWhileHandlingException(socket);
    }
  }
  if (Node.VERBOSE_CONNECTIONS) {
    node.message("socket.close done");
  }
}
 
Example #18
Source File: FstCompiler.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 4 votes vote down vote up
/**
 *
 * @param inputStream the input stream
 * @param outputStream the output stream
 * @throws IOException if compilation fails
 */
public void compile(InputStream inputStream, OutputStream outputStream) throws IOException {
    final HashSet<BytesRef> words = new HashSet<>();
    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
    String line;
    String last = null;
    StringBuilder stringBuilder = new StringBuilder();
    while ((line = reader.readLine()) != null) {
        if (line.indexOf('#') >= 0) {
            continue;
        }
        line = pattern.split(line)[0].trim();
        line = line.toLowerCase(Locale.ROOT);
        if (line.equals(last)) {
            continue;
        }
        last = line;
        /*
         * Add the word to the hash set in left-to-right characters order and reversed
         * for easier matching later on.
         */
        stringBuilder.setLength(0);
        stringBuilder.append(line);
        final int len = stringBuilder.length();
        stringBuilder.append('>');
        words.add(new BytesRef(stringBuilder));
        stringBuilder.setLength(len);
        stringBuilder.reverse().append('<');
        words.add(new BytesRef(stringBuilder));
    }
    reader.close();
    final BytesRef [] all = new BytesRef[words.size()];
    words.toArray(all);
    Arrays.sort(all, BytesRef::compareTo);
    final Object nothing = NoOutputs.getSingleton().getNoOutput();
    final Builder<Object> builder = new Builder<>(INPUT_TYPE.BYTE4, NoOutputs.getSingleton());
    final IntsRefBuilder intsRef = new IntsRefBuilder();
    for (BytesRef bytesRef : all) {
        intsRef.clear();
        intsRef.copyUTF8Bytes(bytesRef);
        builder.add(intsRef.get(), nothing);
    }
    final FST<Object> fst = builder.finish();
    try (OutputStreamDataOutput out = new OutputStreamDataOutput(outputStream)) {
        fst.save(out);
    }
}