Java Code Examples for org.elasticsearch.common.bytes.BytesReference

The following examples show how to use org.elasticsearch.common.bytes.BytesReference. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public void addEvent(Event event, IndexNameBuilder indexNameBuilder, String indexType, long ttlMs) throws Exception {
    BytesReference content = serializer.getContentBuilder(event).bytes();
    Map<String, Map<String, String>> parameters = new HashMap<String, Map<String, String>>();
    Map<String, String> indexParameters = new HashMap<String, String>();
    indexParameters.put(INDEX_PARAM, indexNameBuilder.getIndexName(event));
    indexParameters.put(TYPE_PARAM, indexType);
    if (ttlMs > 0) {
        indexParameters.put(TTL_PARAM, Long.toString(ttlMs));
    }
    parameters.put(INDEX_OPERATION_NAME, indexParameters);

    Gson gson = new Gson();
    synchronized (bulkBuilder) {
        bulkBuilder.append(gson.toJson(parameters));
        bulkBuilder.append("\n");
        bulkBuilder.append(content.toBytesArray().toUtf8());
        bulkBuilder.append("\n");
    }
}
 
Example 2
Source Project: crate   Source File: Store.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception
 * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>.
 */
public void markStoreCorrupted(IOException exception) throws IOException {
    ensureOpen();
    if (!isMarkedCorrupted()) {
        String uuid = CORRUPTED + UUIDs.randomBase64UUID();
        try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(output, CODEC, VERSION);
            BytesStreamOutput out = new BytesStreamOutput();
            out.writeException(exception);
            BytesReference bytes = out.bytes();
            output.writeVInt(bytes.length());
            BytesRef ref = bytes.toBytesRef();
            output.writeBytes(ref.bytes, ref.offset, ref.length);
            CodecUtil.writeFooter(output);
        } catch (IOException ex) {
            logger.warn("Can't mark store as corrupted", ex);
        }
        directory().sync(Collections.singleton(uuid));
    }
}
 
Example 3
@Override
protected RefreshResponse createResponse(HttpInvocationContext<RefreshRequest,RefreshResponse> httpInvocationContext) {
    if (httpInvocationContext == null) {
        throw new IllegalStateException("no http context");
    }
    HttpResponse httpResponse = httpInvocationContext.getHttpResponse();
    try {
        BytesReference ref = new ChannelBufferBytesReference(httpResponse.getContent());
        Map<String,Object> map = JsonXContent.jsonXContent.createParser(ref).map();
        logger.info("{}", map);
        //  RefreshResponse(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures) {
        return new RefreshResponse();
    } catch (IOException e) {
        //
    }
    return null;
}
 
Example 4
Source Project: Elasticsearch   Source File: RestSearchScrollAction.java    License: Apache License 2.0 6 votes vote down vote up
public static void buildFromContent(BytesReference content, SearchScrollRequest searchScrollRequest) {
    try (XContentParser parser = XContentHelper.createParser(content)) {
        if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
            throw new IllegalArgumentException("Malforrmed content, must start with an object");
        } else {
            XContentParser.Token token;
            String currentFieldName = null;
            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                if (token == XContentParser.Token.FIELD_NAME) {
                    currentFieldName = parser.currentName();
                } else if ("scroll_id".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                    searchScrollRequest.scrollId(parser.text());
                } else if ("scroll".equals(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                    searchScrollRequest.scroll(new Scroll(TimeValue.parseTimeValue(parser.text(), null, "scroll")));
                } else {
                    throw new IllegalArgumentException("Unknown parameter [" + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] ");
                }
            }
        }
    } catch (IOException e) {
        throw new IllegalArgumentException("Failed to parse request body", e);
    }
}
 
Example 5
Source Project: elasticsearch-xml   Source File: XmlXContentFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Guesses the content type based on the provided bytes.
 */
public static XmlXContentType xContentType(BytesReference bytes) {
    int length = bytes.length() < GUESS_HEADER_LENGTH ? bytes.length() : GUESS_HEADER_LENGTH;
    if (length == 0) {
        return null;
    }
    byte first = bytes.get(0);
    if (first == '{') {
        return XmlXContentType.JSON;
    }
    if (length > 2 && first == SmileConstants.HEADER_BYTE_1 && bytes.get(1) == SmileConstants.HEADER_BYTE_2 && bytes.get(2) == SmileConstants.HEADER_BYTE_3) {
        return XmlXContentType.SMILE;
    }
    if (length > 2 && first == '-' && bytes.get(1) == '-' && bytes.get(2) == '-') {
        return XmlXContentType.YAML;
    }
    if (length > 2 && first == '<' && bytes.get(1) == '?' && bytes.get(2) == 'x') {
        return XmlXContentType.XML;
    }
    for (int i = 0; i < length; i++) {
        if (bytes.get(i) == '{') {
            return XmlXContentType.JSON;
        }
    }
    return null;
}
 
Example 6
Source Project: anomaly-detection   Source File: TestHelpers.java    License: Apache License 2.0 6 votes vote down vote up
public static SearchResponse createSearchResponse(ToXContentObject o) throws IOException {
    XContentBuilder content = o.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS);

    SearchHit[] hits = new SearchHit[1];
    hits[0] = new SearchHit(0).sourceRef(BytesReference.bytes(content));

    return new SearchResponse(
        new InternalSearchResponse(
            new SearchHits(hits, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f),
            new InternalAggregations(Collections.emptyList()),
            new Suggest(Collections.emptyList()),
            new SearchProfileShardResults(Collections.emptyMap()),
            false,
            false,
            1
        ),
        "",
        5,
        5,
        0,
        100,
        ShardSearchFailure.EMPTY_ARRAY,
        SearchResponse.Clusters.EMPTY
    );
}
 
Example 7
Source Project: elasticsearch-helper   Source File: HttpSearchAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected SearchResponse createResponse(HttpInvocationContext<SearchRequest,SearchResponse> httpInvocationContext) throws IOException {
    if (httpInvocationContext == null) {
        throw new IllegalStateException("no http context");
    }
    HttpResponse httpResponse = httpInvocationContext.getHttpResponse();
    logger.info("{}", httpResponse.getContent().toString(CharsetUtil.UTF_8));
    BytesReference ref = new ChannelBufferBytesReference(httpResponse.getContent());
    Map<String,Object> map = JsonXContent.jsonXContent.createParser(ref).map();

    logger.info("{}", map);

    InternalSearchResponse internalSearchResponse = parseInternalSearchResponse(map);
    String scrollId = (String)map.get(SCROLL_ID);
    int totalShards = 0;
    int successfulShards = 0;
    if (map.containsKey(SHARDS)) {
        Map<String,?> shards = (Map<String,?>)map.get(SHARDS);
        totalShards =  shards.containsKey(TOTAL) ? (Integer)shards.get(TOTAL) : -1;
        successfulShards =  shards.containsKey(SUCCESSFUL) ? (Integer)shards.get(SUCCESSFUL) : -1;
    }
    int tookInMillis = map.containsKey(TOOK) ? (Integer)map.get(TOOK) : -1;
    ShardSearchFailure[] shardFailures = parseShardFailures(map);
    return new SearchResponse(internalSearchResponse, scrollId, totalShards, successfulShards, tookInMillis, shardFailures);
}
 
Example 8
Source Project: crate   Source File: SQLRequestParser.java    License: Apache License 2.0 6 votes vote down vote up
public static SQLRequestParseContext parseSource(BytesReference source) throws IOException {
    if (source.length() == 0) {
        throw new SQLParseException("Missing request body");
    }
    XContentParser parser = null;
    try {
        SQLRequestParseContext parseContext = new SQLRequestParseContext();
        parser = XContentFactory.xContent(XContentType.JSON).createParser(
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, BytesReference.toBytes(source));
        parse(parseContext, parser);
        validate(parseContext);
        return parseContext;
    } catch (Exception e) {
        String sSource = "_na_";
        try {
            sSource = XContentHelper.convertToJson(source, XContentType.JSON);
        } catch (Throwable e1) {
            // ignore
        }
        throw new SQLParseException("Failed to parse source [" + sSource + "]", e);
    } finally {
        if (parser != null) {
            parser.close();
        }
    }
}
 
Example 9
Source Project: crate   Source File: Netty4MessageChannelHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
    Transports.assertTransportThread();
    if (!(msg instanceof ByteBuf)) {
        ctx.fireChannelRead(msg);
        return;
    }
    final ByteBuf buffer = (ByteBuf) msg;
    final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE);
    final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
    try {
        Channel channel = ctx.channel();
        InetSocketAddress remoteAddress = (InetSocketAddress) channel.remoteAddress();
        // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
        // buffer, or in the cumulative buffer, which is cleaned each time so it could be bigger than the actual size
        BytesReference reference = Netty4Utils.toBytesReference(buffer, remainingMessageSize);
        Attribute<NettyTcpChannel> channelAttribute = channel.attr(Netty4Transport.CHANNEL_KEY);
        transport.messageReceived(reference, channelAttribute.get(), profileName, remoteAddress, remainingMessageSize);
    } finally {
        // Set the expected position of the buffer, no matter what happened
        buffer.readerIndex(expectedReaderIndex);
    }
}
 
Example 10
Source Project: Elasticsearch   Source File: DigestBlob.java    License: Apache License 2.0 6 votes vote down vote up
public void addToHead(BytesReference content) throws IOException {
    if (content == null) {
        return;
    }

    int written = 0;
    ChannelBuffer channelBuffer = content.toChannelBuffer();
    int readableBytes = channelBuffer.readableBytes();
    assert readableBytes + headSize.get() <= headLength : "Got too many bytes in addToHead()";

    ByteBuffer byteBuffer = channelBuffer.toByteBuffer();
    while (written < readableBytes) {
        updateDigest(byteBuffer);
        written += headFileChannel.write(byteBuffer);
    }
    headSize.addAndGet(written);
    if (headSize.get() == headLength) {
        headCatchedUpLatch.countDown();
    }
}
 
Example 11
Source Project: crate   Source File: AlterTableRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
private static String mapToJson(Map<String, Object> mapping) throws IOException {
    if (mapping.isEmpty()) {
        return null;
    }
    XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
    builder.map(mapping);
    return XContentHelper.convertToJson(BytesReference.bytes(builder), XContentType.JSON);
}
 
Example 12
Source Project: crate   Source File: PublicationTransportHandler.java    License: Apache License 2.0 5 votes vote down vote up
private static void buildDiffAndSerializeStates(ClusterState clusterState, ClusterState previousState, DiscoveryNodes discoveryNodes,
                                                boolean sendFullVersion, Map<Version, BytesReference> serializedStates,
                                                Map<Version, BytesReference> serializedDiffs) {
    Diff<ClusterState> diff = null;
    for (DiscoveryNode node : discoveryNodes) {
        if (node.equals(discoveryNodes.getLocalNode())) {
            // ignore, see newPublicationContext
            continue;
        }
        try {
            if (sendFullVersion || !previousState.nodes().nodeExists(node)) {
                if (serializedStates.containsKey(node.getVersion()) == false) {
                    serializedStates.put(node.getVersion(), serializeFullClusterState(clusterState, node.getVersion()));
                }
            } else {
                // will send a diff
                if (diff == null) {
                    diff = clusterState.diff(previousState);
                }
                if (serializedDiffs.containsKey(node.getVersion()) == false) {
                    serializedDiffs.put(node.getVersion(), serializeDiffClusterState(diff, node.getVersion()));
                }
            }
        } catch (IOException e) {
            throw new ElasticsearchException("failed to serialize cluster state for publishing to node {}", e, node);
        }
    }
}
 
Example 13
Source Project: crate   Source File: ByteBufStreamInput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BytesReference readBytesReference(int length) throws IOException {
    // NOTE: It is unsafe to share a reference of the internal structure, so we
    // use the default implementation which will copy the bytes. It is unsafe because
    // a netty ByteBuf might be pooled which requires a manual release to prevent
    // memory leaks.
    return super.readBytesReference(length);
}
 
Example 14
public void testSimpleMapping() throws Exception {
    IndexService indexService = createIndex("some_index", Settings.EMPTY,
            "someType", getMapping("simple-mapping.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("someType");
    String sampleText = copyToStringFromClasspath("english.txt");
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParsedDocument doc = docMapper.parse(sourceToParse);
    assertEquals(1, doc.rootDoc().getFields("someField").length);
    assertEquals("en", doc.rootDoc().getFields("someField")[0].stringValue());
}
 
Example 15
Source Project: Elasticsearch   Source File: CreateIndexRequest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets the aliases that will be associated with the index when it gets created
 */
public CreateIndexRequest aliases(BytesReference source) {
    try {
        XContentParser parser = XContentHelper.createParser(source);
        //move to the first alias
        parser.nextToken();
        while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
            alias(Alias.fromXContent(parser));
        }
        return this;
    } catch(IOException e) {
        throw new ElasticsearchParseException("Failed to parse aliases", e);
    }
}
 
Example 16
Source Project: crate   Source File: StreamOutput.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Notice: when serialization a map, the stream out map with the stream in map maybe have the
 * different key-value orders, they will maybe have different stream order.
 * If want to keep stream out map and stream in map have the same stream order when stream,
 * can use {@code writeMapWithConsistentOrder}
 */
public void writeGenericValue(@Nullable Object value) throws IOException {
    if (value == null) {
        writeByte((byte) -1);
        return;
    }
    final Class type;
    if (value instanceof List) {
        type = List.class;
    } else if (value instanceof Object[]) {
        type = Object[].class;
    } else if (value instanceof Map) {
        type = Map.class;
    } else if (value instanceof ReadableInstant) {
        type = ReadableInstant.class;
    } else if (value instanceof BytesReference) {
        type = BytesReference.class;
    } else {
        type = value.getClass();
    }
    final Writer writer = WRITERS.get(type);
    if (writer != null) {
        writer.write(this, value);
    } else {
        throw new IOException("can not write type [" + type + "]");
    }
}
 
Example 17
Source Project: Elasticsearch   Source File: XContentFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static XContent xContent(BytesReference bytes) {
    XContentType type = xContentType(bytes);
    if (type == null) {
        throw new ElasticsearchParseException("Failed to derive xcontent");
    }
    return xContent(type);
}
 
Example 18
Source Project: Elasticsearch   Source File: DeflateCompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean isCompressed(BytesReference bytes) {
    if (bytes.length() < HEADER.length) {
        return false;
    }
    for (int i = 0; i < HEADER.length; ++i) {
        if (bytes.get(i) != HEADER[i]) {
            return false;
        }
    }
    return true;
}
 
Example 19
Source Project: crate   Source File: StreamInput.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads an optional bytes reference from this stream. It might hold an actual reference to the underlying bytes of the stream. Use this
 * only if you must differentiate null from empty. Use {@link StreamInput#readBytesReference()} and
 * {@link StreamOutput#writeBytesReference(BytesReference)} if you do not.
 */
@Nullable
public BytesReference readOptionalBytesReference() throws IOException {
    int length = readVInt() - 1;
    if (length < 0) {
        return null;
    }
    return readBytesReference(length);
}
 
Example 20
@Override
protected UpdateSettingsResponse createResponse(HttpInvocationContext<UpdateSettingsRequest,UpdateSettingsResponse> httpInvocationContext) throws IOException {
    if (httpInvocationContext == null) {
        throw new IllegalStateException("no http context");
    }
    HttpResponse httpResponse = httpInvocationContext.getHttpResponse();
    BytesReference ref = new ChannelBufferBytesReference(httpResponse.getContent());
    Map<String,Object> map = JsonXContent.jsonXContent.createParser(ref).map();
    return new UpdateSettingsResponse();
}
 
Example 21
Source Project: Elasticsearch   Source File: BulkRequest.java    License: Apache License 2.0 5 votes vote down vote up
private int findNextMarker(byte marker, int from, BytesReference data, int length) {
    for (int i = from; i < length; i++) {
        if (data.get(i) == marker) {
            return i;
        }
    }
    return -1;
}
 
Example 22
Source Project: crate   Source File: DeflateCompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean isCompressed(BytesReference bytes) {
    if (bytes.length() < HEADER.length) {
        return false;
    }
    for (int i = 0; i < HEADER.length; ++i) {
        if (bytes.get(i) != HEADER[i]) {
            return false;
        }
    }
    return true;
}
 
Example 23
private SearchResponse mockSearchResponse(String searchIndex) {
    SearchResponse searchResponse = mock(SearchResponse.class);
    Map<String, DocumentField> fields = new HashMap<>();
    SearchHit searchHit = new SearchHit(101, "xxx", null, fields);
    BytesReference source = new BytesArray("{}");
    searchHit.sourceRef(source);
    SearchHit[] bunchOfSearchHits = new SearchHit[1];
    bunchOfSearchHits[0] = searchHit;
    SearchHits searchHits = new SearchHits(bunchOfSearchHits, 1, 1);
    when(searchResponse.getHits()).thenReturn(searchHits);
    return searchResponse;
}
 
Example 24
@Override
protected DeployNodeResponse nodeOperation(DeployNodeRequest request) throws ElasticSearchException {
    String name = request.getRequest().getName();
    if (name == null) {
        throw new ElasticSearchException("no name given");
    }
    String path = request.getRequest().getPath();
    if (path == null) {
        throw new ElasticSearchException("no path given");
    }
    BytesReference ref = request.getRequest().getBytes();
    if (ref == null || ref.length() == 0) {
        throw new ElasticSearchException("no bytes in request");
    }
    // place all deployments under gatherer to avoid overwriting of other plugins
    File dir = new File(environment.pluginsFile(), GathererPlugin.NAME + "/" + name);
    if (dir.exists()) {
        throw new ElasticSearchException("refusing cowardly to overwrite existing path: " + dir.getAbsolutePath());
    }
    try {
        dir.mkdirs();
        File f = new File(path); // just to get file name
        File target = new File(dir, f.getName());
        logger.info("deploying to {}", target.getAbsolutePath());
        FileOutputStream out = new FileOutputStream(target);
        InputStream in = new ByteArrayInputStream(ref.array());
        StreamUtil.copy(in, out);
        in.close();
        out.close();
        // deploy service knows how to unpack archive and add jars to class path
        deployService.add(name, target.getAbsolutePath());
        // TODO set success result in DeployNodeResponse
    } catch (IOException e) {
        throw new ElasticSearchException(e.getMessage());
    }
    DeployNodeResponse response = new DeployNodeResponse();
    return response;
}
 
Example 25
Source Project: crate   Source File: MockTcpTransport.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void sendMessage(BytesReference reference, ActionListener<Void> listener) {
    try {
        synchronized (this) {
            OutputStream outputStream = new BufferedOutputStream(activeChannel.getOutputStream());
            reference.writeTo(outputStream);
            outputStream.flush();
        }
        listener.onResponse(null);
    } catch (IOException e) {
        listener.onFailure(e);
        onException(this, e);
    }
}
 
Example 26
Source Project: crate   Source File: RecoverySourceHandlerTests.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeFileChunk(StoreFileMetaData fileMetaData,
                           long position,
                           BytesReference content,
                           boolean lastChunk,
                           int totalTranslogOps,
                           ActionListener<Void> listener) {
}
 
Example 27
Source Project: Elasticsearch   Source File: JsonXContent.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public XContentParser createParser(BytesReference bytes) throws IOException {
    if (bytes.hasArray()) {
        return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
    }
    return createParser(bytes.streamInput());
}
 
Example 28
public void testToFields() throws Exception {
    IndexService indexService = createIndex("some_index", Settings.EMPTY,
            "someType", getMapping("mapping-to-fields.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("someType");
    String sampleText = copyToStringFromClasspath("english.txt");
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParsedDocument doc = docMapper.parse(sourceToParse);
    assertEquals(1, doc.rootDoc().getFields("someField").length);
    assertEquals("en", doc.rootDoc().getFields("someField")[0].stringValue());
    assertEquals(1, doc.rootDoc().getFields("english_field").length);
    assertEquals("This is a very small example of a text", doc.rootDoc().getFields("english_field")[0].stringValue());
}
 
Example 29
Source Project: Elasticsearch   Source File: BlobRecoveryDeleteRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    digests = new BytesReference[in.readVInt()];
    for (int i = 0; i < digests.length; i++) {
        digests[i] = in.readBytesReference();
    }
}
 
Example 30
Source Project: Elasticsearch   Source File: StreamInput.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads a bytes reference from this stream, might hold an actual reference to the underlying
 * bytes of the stream.
 */
public BytesReference readBytesReference(int length) throws IOException {
    if (length == 0) {
        return BytesArray.EMPTY;
    }
    byte[] bytes = new byte[length];
    readBytes(bytes, 0, length);
    return new BytesArray(bytes, 0, length);
}