org.komamitsu.fluency.Fluency Java Examples

@Test
void buildWithSslAndCustomHostAndPort()
        throws IOException
{
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(true);
    try (Fluency fluency = builder.build("192.168.0.99", 54321)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "192.168.0.99",
                54321,
                SSLSender.class);
    }
}
 

public static void main(String[] args) {
    try {
        Fluency fluency = Fluency.defaultFluency(
                new Fluency.Config().setSenderErrorHandler(Throwable::printStackTrace)
        );
        String tag = "foo_db.bar_tbl";
        Map<String, Object> event = new HashMap<>();
        event.put("name", "komamitsu");
        event.put("age", 42);
        event.put("rate", 3.14);
        fluency.emit(tag, event);
        fluency.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

@Test
void testWithRealFluentd()
        throws Exception
{
    WithRealFluentd.Config config = getConfig();
    assumeTrue(config != null);

    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(config.sslEnabled);

    try (Fluency fluency = builder.build(config.host, config.port)) {
        Map<String, Object> data = new HashMap<>();
        data.put("name", "komamitsu");
        data.put("age", 42);
        data.put("comment", "hello, world");
        ExecutorService executorService = Executors.newCachedThreadPool();
        List<Future<Void>> futures = new ArrayList<>();
        for (int i = 0; i < config.concurrency; i++) {
            futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests)));
        }
        for (Future<Void> future : futures) {
            future.get(config.waitSeconds, TimeUnit.SECONDS);
        }
    }
}
 

@Test
void testSenderErrorHandler()
        throws IOException, InterruptedException
{
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    final AtomicReference<Throwable> errorContainer = new AtomicReference<>();

    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSenderMaxRetryCount(1);
    builder.setErrorHandler(e -> {
        errorContainer.set(e);
        countDownLatch.countDown();
    });

    try (Fluency fluency = builder.build(Integer.MAX_VALUE)) {
        HashMap<String, Object> event = new HashMap<>();
        event.put("name", "foo");
        fluency.emit("tag", event);

        if (!countDownLatch.await(10, TimeUnit.SECONDS)) {
            throw new AssertionError("Timeout");
        }

        assertThat(errorContainer.get(), is(instanceOf(RetryableSender.RetryOverException.class)));
    }
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithoutAckResponse(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(1, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertNull(exception);
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testBufferWithJacksonModule()
        throws IOException
{
    AtomicBoolean serialized = new AtomicBoolean();
    SimpleModule simpleModule = new SimpleModule();
    simpleModule.addSerializer(Foo.class, new FooSerializer(serialized));

    FluentdRecordFormatter.Config recordFormatterConfig = new FluentdRecordFormatter.Config();
    recordFormatterConfig.setJacksonModules(Collections.singletonList(simpleModule));

    Fluency fluency = new TestableFluencyBuilder()
            .buildFromIngester(new FluentdRecordFormatter(recordFormatterConfig), ingester);

    Map<String, Object> event = new HashMap<>();
    Foo foo = new Foo();
    foo.s = "Hello";
    event.put("foo", foo);
    fluency.emit("tag", event);

    assertThat(serialized.get(), is(true));
}
 

@Test
void buildWithSsl()
        throws IOException
{
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(true);
    try (Fluency fluency = builder.build()) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "127.0.0.1",
                24224,
                SSLSender.class);
    }
}
 

@Test
void buildWithSslAndCustomPort()
        throws IOException
{
    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(true);
    try (Fluency fluency = builder.build(54321)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "127.0.0.1",
                54321,
                SSLSender.class);
    }
}
 

@Test
void build()
        throws IOException
{
    try (Fluency fluency = new FluencyBuilderForFluentd().build()) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "127.0.0.1",
                24224,
                TCPSender.class);
    }
}
 

@Test
void buildWithCustomPort()
        throws IOException
{
    try (Fluency fluency = new FluencyBuilderForFluentd().build(54321)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "127.0.0.1",
                54321,
                TCPSender.class);
    }
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
public void testWithAckResponseWithProperToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString();
                assertNotNull(ackResponseToken);

                MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
                packer.packMapHeader(1)
                        .packString("ack").packString(ackResponseToken)
                        .close();

                // Close the input stream after closing the output stream to avoid closing a socket too early
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertNull(exception);
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString());

                MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream());
                packer.packMapHeader(1)
                        .packString("ack").packString(UUID.randomUUID().toString())
                        .close();

                // Close the input stream after closing the output stream to avoid closing a socket too early
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertEquals(exception.getClass(), TimeoutException.class);
}
 

@ParameterizedTest
@MethodSource("sslFlagsProvider")
void testWithAckResponseButNotReceiveToken(final boolean sslEnabled)
        throws Throwable
{
    Exception exception = new ConfigurableTestServer(sslEnabled).run(
            clientSocket -> {
                MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream());
                assertEquals(3, unpacker.unpackArrayHeader());
                assertEquals("foo.bar", unpacker.unpackString());
                ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue();
                Map<Value, Value> map = unpacker.unpackValue().asMapValue().map();
                assertEquals(2, map.size());
                assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt());
                assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString());
                unpacker.close();
            },
            serverPort -> {
                FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
                builder.setSslEnabled(sslEnabled);
                builder.setAckResponseMode(true);

                try (Fluency fluency = builder.build(serverPort)) {
                    fluency.emit("foo.bar", new HashMap<>());
                }
            }, 5000);
    assertEquals(exception.getClass(), TimeoutException.class);
}
 

@Test
void buildWithCustomHostAndPort()
        throws IOException
{
    try (Fluency fluency = new FluencyBuilderForFluentd().build("192.168.0.99", 54321)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (FluentdSender) fluency.getFlusher().getIngester().getSender(),
                "192.168.0.99",
                54321,
                TCPSender.class);
    }
}
 

private EmitTask(Fluency fluency, String tag, Map<String, Object> data, int count)
{
    this.fluency = fluency;
    this.tag = tag;
    this.data = data;
    this.count = count;
}
 

@Test
void testWithRealFluentdWithFileBackup()
        throws ExecutionException, TimeoutException, IOException, InterruptedException
{
    WithRealFluentd.Config config = getConfig();
    assumeTrue(config != null);

    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(config.sslEnabled);
    // Fluency might use a lot of buffer for loaded backup files.
    // So it'd better increase max buffer size
    builder.setMaxBufferSize(512 * 1024 * 1024L);
    builder.setFileBackupDir(System.getProperty("java.io.tmpdir"));

    try (Fluency fluency = new FluencyBuilderForFluentd().build(config.host, config.port)) {
        Map<String, Object> data = new HashMap<>();
        data.put("name", "komamitsu");
        data.put("age", 42);
        data.put("comment", "hello, world");
        ExecutorService executorService = Executors.newCachedThreadPool();
        List<Future<Void>> futures = new ArrayList<>();
        for (int i = 0; i < config.concurrency; i++) {
            futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests)));
        }
        for (Future<Void> future : futures) {
            future.get(config.waitSeconds, TimeUnit.SECONDS);
        }
    }
}
 

@Test
void testWithRealMultipleFluentd()
        throws IOException, InterruptedException, TimeoutException, ExecutionException
{
    WithRealFluentd.Config config = getConfig();
    assumeTrue(config != null);
    assumeTrue(config.anotherPort != null);

    FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd();
    builder.setSslEnabled(config.sslEnabled);
    builder.setAckResponseMode(true);

    try (Fluency fluency = builder.build(
            Arrays.asList(
                    new InetSocketAddress(config.host, config.port),
                    new InetSocketAddress(config.host, config.anotherPort)))) {

        Map<String, Object> data = new HashMap<>();
        data.put("name", "komamitsu");
        data.put("age", 42);
        data.put("comment", "hello, world");
        ExecutorService executorService = Executors.newCachedThreadPool();
        List<Future<Void>> futures = new ArrayList<>();
        for (int i = 0; i < config.concurrency; i++) {
            futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests)));
        }
        for (Future<Void> future : futures) {
            future.get(config.waitSeconds, TimeUnit.SECONDS);
        }
    }
}
 

public Fluency build(List<InetSocketAddress> servers)
{
    List<FluentdSender> senders = new ArrayList<>();
    for (InetSocketAddress server : servers) {
        senders.add(createBaseSender(server.getHostName(), server.getPort(), true));
    }
    return buildFromIngester(
            buildRecordFormatter(),
            buildIngester(new MultiSender(senders)));
}
 

@Nullable
public static Fluency buildStaticFluency(YanagishimaConfig config) {
    if (config.getFluentdExecutedTag().isPresent() || config.getFluentdFaliedTag().isPresent()) {
        try {
            return Fluency.defaultFluency(config.getFluentdHost(), config.getFluentdPort());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    return null;
}
 

@Test
void build()
        throws IOException, NoSuchFieldException, IllegalAccessException
{
    try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY)) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (TreasureDataSender) fluency.getFlusher().getIngester().getSender(),
                "api-import.treasuredata.com", true);
    }
}
 

@Test
void buildWithCustomHttpsEndpoint()
        throws IOException, NoSuchFieldException, IllegalAccessException
{
    try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "https://custom.endpoint.org")) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (TreasureDataSender) fluency.getFlusher().getIngester().getSender(),
                "custom.endpoint.org", true);
    }
}
 

@Test
void buildWithCustomHttpsEndpointWithoutScheme()
        throws IOException, NoSuchFieldException, IllegalAccessException
{
    try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "custom.endpoint.org")) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (TreasureDataSender) fluency.getFlusher().getIngester().getSender(),
                "custom.endpoint.org", true);
    }
}
 

@Test
void buildWithCustomHttpEndpoint()
        throws IOException, NoSuchFieldException, IllegalAccessException
{
    try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "http://custom.endpoint.org")) {
        assertBuffer(fluency.getBuffer());
        assertFlusher(fluency.getFlusher());
        assertDefaultFluentdSender(
                (TreasureDataSender) fluency.getFlusher().getIngester().getSender(),
                "custom.endpoint.org", false);
    }
}
 

public Fluency build(int port)
{
    return buildFromIngester(
            buildRecordFormatter(),
            buildIngester(createBaseSender(null, port)));
}
 

public Fluency build(String apikey, String endpoint)
{
    return buildFromIngester(
            buildRecordFormatter(),
            buildIngester(createSenderConfig(endpoint, apikey)));
}
 

public Fluency build(String apikey)
{
    return buildFromIngester(
            buildRecordFormatter(),
            buildIngester(createSenderConfig(null, apikey)));
}
 

@Test
void buildWithAllCustomConfig()
        throws IOException
{
    String tmpdir = System.getProperty("java.io.tmpdir");
    assertNotNull(tmpdir);

    FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData();
    builder.setFlushAttemptIntervalMillis(200);
    builder.setMaxBufferSize(Long.MAX_VALUE);
    builder.setBufferChunkInitialSize(7 * 1024 * 1024);
    builder.setBufferChunkRetentionSize(13 * 1024 * 1024);
    builder.setBufferChunkRetentionTimeMillis(19 * 1000);
    builder.setJvmHeapBufferMode(true);
    builder.setWaitUntilBufferFlushed(42);
    builder.setWaitUntilFlusherTerminated(24);
    builder.setFileBackupDir(tmpdir);
    builder.setSenderRetryIntervalMillis(1234);
    builder.setSenderMaxRetryIntervalMillis(345678);
    builder.setSenderRetryFactor(3.14f);
    builder.setSenderRetryMax(17);
    builder.setSenderWorkBufSize(123456);
    ;

    try (Fluency fluency = builder.build(APIKEY)) {
        assertEquals(Buffer.class, fluency.getBuffer().getClass());
        Buffer buffer = fluency.getBuffer();
        assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize());
        assertEquals(tmpdir, buffer.getFileBackupDir());
        assertEquals("packed_forward", buffer.bufferFormatType());
        assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis());
        assertEquals(2f, buffer.getChunkExpandRatio());
        assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize());
        assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize());
        assertTrue(buffer.getJvmHeapBufferMode());

        Flusher flusher = fluency.getFlusher();
        assertFalse(flusher.isTerminated());
        assertEquals(200, flusher.getFlushAttemptIntervalMillis());
        assertEquals(42, flusher.getWaitUntilBufferFlushed());
        assertEquals(24, flusher.getWaitUntilTerminated());

        assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass());
        TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender();
        assertEquals(1234, sender.getRetryInternalMs());
        assertEquals(345678, sender.getMaxRetryInternalMs());
        assertEquals(3.14f, sender.getRetryFactor());
        assertEquals(17, sender.getRetryMax());
        assertEquals(123456, sender.getWorkBufSize());
    }
}
 

public Fluency build(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig)
{
    return buildFromIngester(recordFormatter, createIngester(recordFormatter, senderConfig));
}
 

public Fluency build()
{
    return build(createRecordFormatter());
}
 

@BeforeEach
void setUp()
{
    fluency = mock(Fluency.class);

    builderWithDefaultConfig = spy(new FluencyBuilderForAwsS3());
    builderWithDefaultConfig.setFormatCsvColumnNames(ImmutableList.of("dummy"));
    doReturn(fluency).when(builderWithDefaultConfig)
            .createFluency(
                    any(RecordFormatter.class),
                    any(Ingester.class),
                    any(Buffer.Config.class),
                    any(Flusher.Config.class));

    {
        FluencyBuilderForAwsS3 builder = new FluencyBuilderForAwsS3();
        builder.setAwsEndpoint("https://foo.bar.org");
        builder.setAwsRegion("us-east-1");
        builder.setCompressionEnabled(false);
        builder.setAwsAccessKeyId("ACCESSKEYID");
        builder.setAwsSecretAccessKey("SECRETACCESSKEY");
        builder.setBufferChunkInitialSize(2 * 1024 * 1024);
        builder.setBufferChunkRetentionSize(9 * 1024 * 1024);
        builder.setBufferChunkRetentionTimeMillis(1234);
        builder.setMaxBufferSize(42 * 1024 * 1024L);
        builder.setS3KeyPrefix("mydata");
        builder.setS3KeySuffix(".zzz");
        builder.setS3KeyTimeZoneId(ZoneOffset.of("JST", SHORT_IDS));
        builder.setSenderRetryMax(4);
        builder.setSenderMaxRetryIntervalMillis(65432);
        builder.setSenderRetryIntervalMillis(543);
        builder.setSenderRetryFactor(1.234f);
        builder.setSenderWorkBufSize(99 * 1024);
        builderWithCustomConfig = spy(builder);
    }
    doReturn(fluency).when(builderWithCustomConfig)
            .createFluency(
                    any(RecordFormatter.class),
                    any(Ingester.class),
                    any(Buffer.Config.class),
                    any(Flusher.Config.class));
}