org.influxdb.dto.BatchPoints Java Examples

The following examples show how to use org.influxdb.dto.BatchPoints. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchProcessorTest.java    From influxdb-java with MIT License 6 votes vote down vote up
@Test
public void testSchedulerExceptionHandling() throws InterruptedException, IOException {
    InfluxDB mockInfluxDB = mock(InfluxDBImpl.class);
    BatchProcessor batchProcessor = BatchProcessor.builder(mockInfluxDB).actions(Integer.MAX_VALUE)
        .interval(1, TimeUnit.NANOSECONDS).build();

    doThrow(new RuntimeException()).when(mockInfluxDB).write(any(BatchPoints.class));

    Point point = Point.measurement("cpu").field("6", "").build();
    BatchProcessor.HttpBatchEntry batchEntry1 = new BatchProcessor.HttpBatchEntry(point, "db1", "");
    BatchProcessor.HttpBatchEntry batchEntry2 = new BatchProcessor.HttpBatchEntry(point, "db2", "");

    batchProcessor.put(batchEntry1);
    Thread.sleep(200); // wait for scheduler

    // first try throws an exception
    verify(mockInfluxDB, times(1)).write(any(BatchPoints.class));

    batchProcessor.put(batchEntry2);
    Thread.sleep(200); // wait for scheduler
    // without try catch the 2nd time does not occur
    verify(mockInfluxDB, times(2)).write(any(BatchPoints.class));
}
 
Example #2
Source File: InfluxHistoryTest.java    From monsoon with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void add() throws Exception {
    final GroupName group = GroupName.valueOf(SimpleGroupPath.valueOf("foo", "bar"), Tags.valueOf(singletonMap("x", MetricValue.fromIntValue(17))));
    final ImmutableTimeSeriesValue tsv0 = new ImmutableTimeSeriesValue(group, singletonMap(MetricName.valueOf("met", "ric"), MetricValue.fromStrValue("value")));
    final TimeSeriesCollection tsdata = new SimpleTimeSeriesCollection(DateTime.parse("2017-09-17T10:00:00.000Z"), singleton(tsv0));

    boolean historyAddResult = history.add(tsdata);

    assertTrue(historyAddResult);

    verify(influxDB, times(1)).write(Mockito.<BatchPoints>argThat(
            Matchers.hasProperty(
                    "points",
                    Matchers.contains(
                            Point.measurement("foo.bar")
                            .tag("x", "17")
                            .time(DateTime.parse("2017-09-17T10:00:00.000Z").getMillis(), TimeUnit.MILLISECONDS)
                            .addField("met.ric", "value")
                            .build()
                    )
            )
    ));
    verifyNoMoreInteractions(influxDB);
}
 
Example #3
Source File: StatsCollectorTest.java    From cloudstack with Apache License 2.0 6 votes vote down vote up
@Test
public void writeBatchesTest() {
    InfluxDB influxDbConnection = Mockito.mock(InfluxDB.class);
    Mockito.doNothing().when(influxDbConnection).write(Mockito.any(Point.class));
    Builder builder = Mockito.mock(Builder.class);
    BatchPoints batchPoints = Mockito.mock(BatchPoints.class);
    PowerMockito.mockStatic(BatchPoints.class);
    PowerMockito.when(BatchPoints.database(DEFAULT_DATABASE_NAME)).thenReturn(builder);
    Mockito.when(builder.build()).thenReturn(batchPoints);
    Map<String, String> tagsToAdd = new HashMap<>();
    tagsToAdd.put("hostId", "1");
    Map<String, Object> fieldsToAdd = new HashMap<>();
    fieldsToAdd.put("total_memory_kbs", 10000000);
    Point point = Point.measurement("measure").tag(tagsToAdd).time(System.currentTimeMillis(), TimeUnit.MILLISECONDS).fields(fieldsToAdd).build();
    List<Point> points = new ArrayList<>();
    points.add(point);
    Mockito.when(batchPoints.point(point)).thenReturn(batchPoints);

    statsCollector.writeBatches(influxDbConnection, DEFAULT_DATABASE_NAME, points);

    Mockito.verify(influxDbConnection).write(batchPoints);
}
 
Example #4
Source File: InfluxDBReporterTest.java    From statsd-jvm-profiler with MIT License 6 votes vote down vote up
@Override
protected void testCase(Object[] args) {
    assertEquals(1, args.length);

    BatchPoints actual = (BatchPoints) args[0];

    Point expectedPoint = Point.measurement("fake")
            .field(InfluxDBReporter.VALUE_COLUMN, 100L)
            .tag(TagUtil.PREFIX_TAG, "influxdb.reporter.test")
            .build();

    BatchPoints expected = BatchPoints.database("database").build();
    expected.point(expectedPoint);

    assertEquals(expected.getDatabase(), actual.getDatabase());
    assertEquals(expected.getPoints().size(), actual.getPoints().size());

    Point actualPoint = actual.getPoints().get(0);

    // All the fields on Point are private
    assertTrue(actualPoint.lineProtocol().startsWith("fake"));
    assertTrue(actualPoint.lineProtocol().contains("value=100"));
    assertTrue(actualPoint.lineProtocol().contains("prefix=influxdb.reporter.test"));
}
 
Example #5
Source File: InfluxDB.java    From iotdb-benchmark with Apache License 2.0 6 votes vote down vote up
@Override
public Status insertOneBatch(Batch batch) {
  BatchPoints batchPoints = BatchPoints.database(influxDbName)
      .retentionPolicy(defaultRp)
      .consistency(org.influxdb.InfluxDB.ConsistencyLevel.ALL).build();
  try {
    InfluxDataModel model;
    for (Record record : batch.getRecords()) {
      model = createDataModel(batch.getDeviceSchema(), record.getTimestamp(),
          record.getRecordDataValue());
      batchPoints.point(model.toInfluxPoint());
    }

    influxDbInstance.write(batchPoints);

    return new Status(true);
  } catch (Exception e) {
    LOGGER.warn(e.getMessage());
    return new Status(false, 0, e, e.toString());
  }
}
 
Example #6
Source File: BatchOptionsTest.java    From influxdb-java with MIT License 6 votes vote down vote up
@Test
public void testWriteWithRetryOnBatchingNotEnabled() {
  String dbName = "write_unittest_" + System.currentTimeMillis();
  try {

    influxDB.query(new Query("CREATE DATABASE " + dbName));
    influxDB.setDatabase(dbName);

    BatchPoints batchPoints = createBatchPoints(dbName, "m0", 200);
    influxDB.writeWithRetry(batchPoints);

    QueryResult result = influxDB.query(new Query("select * from m0", dbName));
    Assertions.assertNotNull(result.getResults().get(0).getSeries());
    Assertions.assertEquals(200, result.getResults().get(0).getSeries().get(0).getValues().size());
  } finally {
    influxDB.query(new Query("DROP DATABASE " + dbName));
  }

}
 
Example #7
Source File: TicketTest.java    From influxdb-java with MIT License 6 votes vote down vote up
/**
 * Test for ticket #39
 *
 */
@Test
public void testTicket39() {
	String dbName = "ticket39_" + System.currentTimeMillis();
	this.influxDB.query(new Query("CREATE DATABASE " + dbName));
	BatchPoints batchPoints = BatchPoints
			.database(dbName)
			.tag("async", "true")
			.retentionPolicy(TestUtils.defaultRetentionPolicy(this.influxDB.version()))
			.consistency(InfluxDB.ConsistencyLevel.ALL)
			.build();
	Point.Builder builder = Point.measurement("my_type");
	builder.addField("my_field", "string_value");
	Point point = builder.build();
	batchPoints.point(point);
	this.influxDB.write(batchPoints);
	this.influxDB.query(new Query("DROP DATABASE " + dbName));
}
 
Example #8
Source File: InfluxDBConverter.java    From RuuviCollector with MIT License 6 votes vote down vote up
public static BatchPoints toLegacyInflux(EnhancedRuuviMeasurement measurement) {
    List<Point> points = new ArrayList<>();
    createAndAddLegacyFormatPointIfNotNull(points, "temperature", measurement.getTemperature(), null, null);
    createAndAddLegacyFormatPointIfNotNull(points, "humidity", measurement.getHumidity(), null, null);
    createAndAddLegacyFormatPointIfNotNull(points, "pressure", measurement.getPressure(), null, null);
    createAndAddLegacyFormatPointIfNotNull(points, "acceleration", measurement.getAccelerationX(), "axis", "x");
    createAndAddLegacyFormatPointIfNotNull(points, "acceleration", measurement.getAccelerationY(), "axis", "y");
    createAndAddLegacyFormatPointIfNotNull(points, "acceleration", measurement.getAccelerationZ(), "axis", "z");
    createAndAddLegacyFormatPointIfNotNull(points, "acceleration", measurement.getAccelerationTotal(), "axis", "total");
    createAndAddLegacyFormatPointIfNotNull(points, "batteryVoltage", measurement.getBatteryVoltage(), null, null);
    createAndAddLegacyFormatPointIfNotNull(points, "rssi", measurement.getRssi(), null, null);
    return BatchPoints
        .database(Config.getInfluxDatabase())
        .tag("protocolVersion", String.valueOf(measurement.getDataFormat()))
        .tag("mac", measurement.getMac())
        .points(points.toArray(new Point[points.size()]))
        .build();
}
 
Example #9
Source File: BatchProcessorTest.java    From influxdb-java with MIT License 6 votes vote down vote up
@Test
public void testSchedulerExceptionHandlingCallback() throws InterruptedException, IOException {
  InfluxDB mockInfluxDB = mock(InfluxDBImpl.class);
  BiConsumer<Iterable<Point>, Throwable> mockHandler = mock(BiConsumer.class);
  BatchProcessor batchProcessor = BatchProcessor.builder(mockInfluxDB).actions(Integer.MAX_VALUE)
      .interval(1, TimeUnit.NANOSECONDS).exceptionHandler(mockHandler).build();

  doThrow(new RuntimeException()).when(mockInfluxDB).write(any(BatchPoints.class));

  Point point = Point.measurement("cpu").field("6", "").build();
  BatchProcessor.HttpBatchEntry batchEntry1 = new BatchProcessor.HttpBatchEntry(point, "db1", "");
  BatchProcessor.HttpBatchEntry batchEntry2 = new BatchProcessor.HttpBatchEntry(point, "db2", "");

  batchProcessor.put(batchEntry1);
  Thread.sleep(200); // wait for scheduler

  verify(mockHandler, times(1)).accept(argThat(Matchers.hasItems(point, point)), any(RuntimeException.class));
}
 
Example #10
Source File: HttpStatServiceImpl.java    From EserKnife with Apache License 2.0 6 votes vote down vote up
@Override
public void batchInsert(List<NodeHttpStatInfo> nodeHttpStatInfos) {
    if(InflusDbUtil.FLAG){
        if (CollectionUtils.isNotEmpty(nodeHttpStatInfos)) {
            InfluxDB influxDB = InflusDbUtil.getConnection();
            BatchPoints batchPoints = BatchPoints.database(InflusDbUtil.DATABASE)
                    .retentionPolicy("autogen").consistency(InfluxDB.ConsistencyLevel.ALL).build();
            long times = System.currentTimeMillis();
            for (NodeHttpStatInfo nodeHttpStatInfo : nodeHttpStatInfos) {
                Point point = Point.measurement("http").time(times, TimeUnit.MILLISECONDS)
                        .tag("host", nodeHttpStatInfo.getHost())
                        .tag("clusterName", nodeHttpStatInfo.getClustName())
                        .addField("createTime", DateUtil.formatYYYYMMddHHMMSS(nodeHttpStatInfo.getCreateTime()))
                        .addField("totalOpened", nodeHttpStatInfo.getTotal_opened())
                        .addField("currentOpen", nodeHttpStatInfo.getCurrent_open())
                        .build();
                batchPoints.point(point);
            }
            influxDB.write(batchPoints);
        }
    }else{
        httpStatDao.batchInsert(nodeHttpStatInfos);
    }
}
 
Example #11
Source File: BatchProcessorTest.java    From influxdb-java with MIT License 6 votes vote down vote up
@Test
public void testBatchWriteWithDifferenctRp() throws InterruptedException, IOException {
    InfluxDB mockInfluxDB = mock(InfluxDBImpl.class);
    BatchProcessor batchProcessor = BatchProcessor.builder(mockInfluxDB).actions(Integer.MAX_VALUE)
        .interval(1, TimeUnit.NANOSECONDS).build();

    Point point = Point.measurement("cpu").field("6", "").build();
    BatchProcessor.HttpBatchEntry batchEntry1 = new BatchProcessor.HttpBatchEntry(point, "db1", "rp_1");
    BatchProcessor.HttpBatchEntry batchEntry2 = new BatchProcessor.HttpBatchEntry(point, "db1", "rp_2");

    batchProcessor.put(batchEntry1);
    batchProcessor.put(batchEntry2);

    Thread.sleep(200); // wait for scheduler
    // same dbname with different rp should write two batchs instead of only one.
    verify(mockInfluxDB, times(2)).write(any(BatchPoints.class));
}
 
Example #12
Source File: BatchProcessorTest.java    From influxdb-java with MIT License 6 votes vote down vote up
@Test
public void testFlushWritesBufferedPointsAndDoesNotShutdownScheduler() throws InterruptedException {
    InfluxDB mockInfluxDB = mock(InfluxDBImpl.class);
    BatchProcessor batchProcessor = BatchProcessor.builder(mockInfluxDB)
            .actions(Integer.MAX_VALUE)
            .interval(1, TimeUnit.NANOSECONDS).build();

    Point point = Point.measurement("test").addField("region", "a").build();
    BatchProcessor.HttpBatchEntry httpBatchEntry = new BatchProcessor.HttpBatchEntry(point, "http", "http-rp");

    batchProcessor.put(httpBatchEntry);
    Thread.sleep(100); // wait for scheduler
    // Our put should have been written
    verify(mockInfluxDB).write(any(BatchPoints.class));

    // Force a flush which should not stop the scheduler
    batchProcessor.flush();

    batchProcessor.put(httpBatchEntry);
    Thread.sleep(100); // wait for scheduler
    // Our second put should have been written if the scheduler is still running
    verify(mockInfluxDB, times(2)).write(any(BatchPoints.class));

    verifyNoMoreInteractions(mockInfluxDB);
}
 
Example #13
Source File: InfluxTarget.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Batch batch) throws StageException {
  BatchPoints batchPoints = BatchPoints
      .database(conf.dbName)
      .retentionPolicy(conf.retentionPolicy)
      .consistency(conf.consistencyLevel)
      .build();

  Iterator<Record> recordIterator = batch.getRecords();

  while (recordIterator.hasNext()) {
    Record record = recordIterator.next();

    for (Point point : converter.getPoints(record)) {
      batchPoints.point(point);
    }
  }

  client.write(batchPoints);
}
 
Example #14
Source File: BatchDAO.java    From skywalking with Apache License 2.0 6 votes vote down vote up
@Override
public void synchronous(List<PrepareRequest> prepareRequests) {
    if (CollectionUtils.isEmpty(prepareRequests)) {
        return;
    }

    if (log.isDebugEnabled()) {
        log.debug("batch sql statements execute, data size: {}", prepareRequests.size());
    }

    final BatchPoints.Builder builder = BatchPoints.builder();
    prepareRequests.forEach(e -> {
        builder.point(((InfluxInsertRequest) e).getPoint());
    });

    client.write(builder.build());
}
 
Example #15
Source File: InfluxDBImpl.java    From influxdb-java with MIT License 5 votes vote down vote up
@Override
public void write(final String database, final String retentionPolicy, final Point point) {
  if (this.batchEnabled.get()) {
    HttpBatchEntry batchEntry = new HttpBatchEntry(point, database, retentionPolicy);
    this.batchProcessor.put(batchEntry);
  } else {
    BatchPoints batchPoints = BatchPoints.database(database)
                                         .retentionPolicy(retentionPolicy).build();
    batchPoints.point(point);
    this.write(batchPoints);
    this.unBatchedCount.increment();
  }
  this.writeCount.increment();
}
 
Example #16
Source File: StatsCollector.java    From cloudstack with Apache License 2.0 5 votes vote down vote up
/**
 * Writes batches of InfluxDB database points into a given database.
 */
protected void writeBatches(InfluxDB influxDbConnection, String dbName, List<Point> points) {
    BatchPoints batchPoints = BatchPoints.database(dbName).build();
    influxDbConnection.enableBatch(BatchOptions.DEFAULTS);

    for (Point point : points) {
        batchPoints.point(point);
    }

    influxDbConnection.write(batchPoints);
}
 
Example #17
Source File: RetryCapableBatchWriter.java    From influxdb-java with MIT License 5 votes vote down vote up
private void addToBatchQueue(final BatchPoints batchPoints) {
  boolean hasBeenMergedIn = false;
  if (batchQueue.size() > 0) {
    BatchPoints last = batchQueue.getLast();
    if (last.getPoints().size() + batchPoints.getPoints().size() <= requestActionsLimit) {
      hasBeenMergedIn = last.mergeIn(batchPoints);
    }
  }
  if (!hasBeenMergedIn) {
      batchQueue.add(batchPoints);
  }
  // recalculate local counter and evict old batches on merge as well
  usedRetryBufferCapacity += batchPoints.getPoints().size();
  evictTooOldFailedWrites();
}
 
Example #18
Source File: InfluxDBPusher.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
/**
 * Push multiple points at once.
 * @param points list of {@link Point}s to report
 */
public void push(List<Point> points) {
  BatchPoints.Builder batchPointsBuilder = BatchPoints.database(database);
  for (Point point : points) {
    batchPointsBuilder.point(point);
  }
  influxDB.write(batchPointsBuilder.build());
}
 
Example #19
Source File: CommonServiceImpl.java    From EserKnife with Apache License 2.0 5 votes vote down vote up
@Override
public void batchInsert(List<NodeCommonStatInfo> nodeCommonStatInfos) {
    if(InflusDbUtil.FLAG){
        if (CollectionUtils.isNotEmpty(nodeCommonStatInfos)) {
            InfluxDB influxDB = InflusDbUtil.getConnection();
            BatchPoints batchPoints = BatchPoints.database(InflusDbUtil.DATABASE)
                    .retentionPolicy("autogen").consistency(InfluxDB.ConsistencyLevel.ALL).build();
            long times = System.currentTimeMillis();
            for (NodeCommonStatInfo nodeCommonStatInfo : nodeCommonStatInfos) {
                Point point = Point.measurement("common").time(times, TimeUnit.MILLISECONDS)
                        .tag("clusterName", nodeCommonStatInfo.getClusterName())
                        .addField("createTime", DateUtil.formatYYYYMMddHHMMSS(nodeCommonStatInfo.getCreateTime()))
                        .addField("nodeCount", nodeCommonStatInfo.getNodeCount())
                        .addField("dataNodeCount",nodeCommonStatInfo.getDataNodeCount())
                        .addField("docCount", nodeCommonStatInfo.getDocCounts())
                        .addField("storeSize", nodeCommonStatInfo.getStoreSize())
                        .addField("indiceCount", nodeCommonStatInfo.getIndicesCount())
                        .addField("shardCount", nodeCommonStatInfo.getShardCounts())
                        .addField("clusterStatus", nodeCommonStatInfo.getClusterStatus())
                        .build();
                batchPoints.point(point);
            }
            influxDB.write(batchPoints);
        }
    }else{
        commonDAO.batchInsert(nodeCommonStatInfos);
    }
}
 
Example #20
Source File: ThreadPoolServiceImpl.java    From EserKnife with Apache License 2.0 5 votes vote down vote up
@Override
public void batchInsert(List<NodeThreadPoolStatInfo> nodeThreadPoolStatInfos) {
    if(InflusDbUtil.FLAG){
        if (CollectionUtils.isNotEmpty(nodeThreadPoolStatInfos)) {
            InfluxDB influxDB = InflusDbUtil.getConnection();
            BatchPoints batchPoints = BatchPoints.database(InflusDbUtil.DATABASE)
                    .retentionPolicy("autogen").consistency(InfluxDB.ConsistencyLevel.ALL).build();
            long times = System.currentTimeMillis();
            for (NodeThreadPoolStatInfo threadPoolStatInfo : nodeThreadPoolStatInfos) {
                Point point = Point.measurement("thread_pool").time(times, TimeUnit.MILLISECONDS)
                        .tag("threadType",threadPoolStatInfo.getThreadType())
                        .tag("host", threadPoolStatInfo.getHost())
                        .tag("clusterName", threadPoolStatInfo.getClusterName())
                        .addField("createTime", DateUtil.formatYYYYMMddHHMMSS(threadPoolStatInfo.getCreateTime()))
                        .addField("largest",threadPoolStatInfo.getLargest())
                        .addField("completed",threadPoolStatInfo.getCompleted())
                        .addField("active",threadPoolStatInfo.getActive())
                        .addField("rejected",threadPoolStatInfo.getRejected())
                        .addField("threads",threadPoolStatInfo.getThreads())
                        .addField("queue",threadPoolStatInfo.getQueue())
                        .addField("intervalCompleted",threadPoolStatInfo.getIntervalCompleted())
                        .addField("intervalRejected",threadPoolStatInfo.getIntervalRejected())
                        .build();
                batchPoints.point(point);
            }
            influxDB.write(batchPoints);
        }
    }else{
        threadPoolDao.batchInsert(nodeThreadPoolStatInfos);
    }
}
 
Example #21
Source File: OsStatServiceImpl.java    From EserKnife with Apache License 2.0 5 votes vote down vote up
@Override
public void batchInsert(List<NodeOSStatInfo> statInfos) {
    if(InflusDbUtil.FLAG){
        if(CollectionUtils.isNotEmpty(statInfos)){
            InfluxDB influxDB = InflusDbUtil.getConnection();
            BatchPoints batchPoints = BatchPoints
                    .database(InflusDbUtil.DATABASE)
                    .retentionPolicy("autogen")
                    .consistency(InfluxDB.ConsistencyLevel.ALL)
                    .build();
            long times = System.currentTimeMillis();
            for (NodeOSStatInfo nodeOSStatInfo : statInfos){
                Point point = Point.measurement("os")
                        .time(times, TimeUnit.MILLISECONDS)
                        .tag("host",nodeOSStatInfo.getHost())
                        .tag("clusterName",nodeOSStatInfo.getClusterName())
                        .addField("cpuPercent",nodeOSStatInfo.getCpuPercent())
                        .addField("loadAverage",nodeOSStatInfo.getLoadAverage())
                        .addField("memTotal",nodeOSStatInfo.getMemTotal())
                        .addField("memUsed",nodeOSStatInfo.getMemUsed())
                        .addField("memFree",nodeOSStatInfo.getMemFree())
                        .addField("memFreePercent",nodeOSStatInfo.getMemFreePercent())
                        .addField("memUsedPercent",nodeOSStatInfo.getMemUsedPercent())
                        .addField("swapTotal",nodeOSStatInfo.getSwapTotal())
                        .addField("swapUsed",nodeOSStatInfo.getSwapUsed())
                        .addField("swapFree",nodeOSStatInfo.getSwapFree())
                        .addField("createTime", DateUtil.formatYYYYMMddHHMMSS(nodeOSStatInfo.getCreateTime()))
                        .build();
                batchPoints.point(point);
            }
            influxDB.write(batchPoints);
        }
    }else{
        osStatDao.batchInsert(statInfos);
    }
}
 
Example #22
Source File: InfluxDBImpl.java    From influxdb-java with MIT License 5 votes vote down vote up
@Override
public void write(final BatchPoints batchPoints) {
  this.batchedCount.add(batchPoints.getPoints().size());
  RequestBody lineProtocol = RequestBody.create(MEDIA_TYPE_STRING, batchPoints.lineProtocol());
  String db = batchPoints.getDatabase();
  if (db == null) {
      db = this.database;
  }
  execute(this.influxDBService.writePoints(
      db,
      batchPoints.getRetentionPolicy(),
      TimeUtil.toTimePrecision(batchPoints.getPrecision()),
      batchPoints.getConsistency().value(),
      lineProtocol));
}
 
Example #23
Source File: InfluxDBImpl.java    From influxdb-java with MIT License 5 votes vote down vote up
@Override
public void writeWithRetry(final BatchPoints batchPoints) {
  if (isBatchEnabled()) {
    batchProcessor.getBatchWriter().write(Collections.singleton(batchPoints));
  } else {
    write(batchPoints);
  }
}
 
Example #24
Source File: RetryCapableBatchWriter.java    From influxdb-java with MIT License 5 votes vote down vote up
@Override
public synchronized void close() {
  // try to write everything queued / buffered
  for (BatchPoints points : batchQueue) {
    WriteResult result = tryToWrite(points);
    if (result.outcome != WriteResultOutcome.WRITTEN) {
      exceptionHandler.accept(points.getPoints(), result.throwable);
    }
  }
}
 
Example #25
Source File: InfluxdbResource.java    From camel-quarkus with Apache License 2.0 5 votes vote down vote up
@Path("/batch")
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.TEXT_PLAIN)
public int batch(Points points) {
    return producerTemplate.toF(
            "influxdb:%s?batch=true", INFLUXDB_CONNECTION_NAME)
            .withBody(points.toBatchPoints())
            .request(BatchPoints.class)
            .getPoints().size();
}
 
Example #26
Source File: BatchOptionsTest.java    From influxdb-java with MIT License 5 votes vote down vote up
@Test
public void testWriteWithRetryOnUnrecoverableError() throws InterruptedException {

  String dbName = "write_unittest_" + System.currentTimeMillis();
  InfluxDB spy = spy((InfluxDB) influxDB);
  doThrow(DatabaseNotFoundException.class).when(spy).write(any(BatchPoints.class));

  try {
    BiConsumer<Iterable<Point>, Throwable> mockHandler = mock(BiConsumer.class);
    BatchOptions options = BatchOptions.DEFAULTS.exceptionHandler(mockHandler).flushDuration(100);

    spy.query(new Query("CREATE DATABASE " + dbName));
    spy.setDatabase(dbName);
    spy.enableBatch(options);

    BatchPoints batchPoints = createBatchPoints(dbName, "m0", 200);
    spy.writeWithRetry(batchPoints);
    Thread.sleep(500);

    verify(mockHandler, times(1)).accept(any(), any());

    QueryResult result = influxDB.query(new Query("select * from m0", dbName));
    Assertions.assertNull(result.getResults().get(0).getSeries());
    Assertions.assertNull(result.getResults().get(0).getError());
  } finally {
    spy.disableBatch();
    spy.query(new Query("DROP DATABASE " + dbName));
  }

}
 
Example #27
Source File: BatchOptionsTest.java    From influxdb-java with MIT License 5 votes vote down vote up
private BatchPoints createBatchPoints(String dbName, String measurement, int n) {
  BatchPoints batchPoints = BatchPoints.database(dbName).build();
  for (int i = 1; i <= n; i++) {
    Point point = Point.measurement(measurement)
            .time(i,TimeUnit.MILLISECONDS)
            .addField("f1", (double) i)
            .addField("f2", (double) (i) * 1.1)
            .addField("f3", "f_v3").build();
    batchPoints.point(point);
  }

  return batchPoints;
}
 
Example #28
Source File: RetryCapableBatchWriterTest.java    From influxdb-java with MIT License 5 votes vote down vote up
BatchPoints getBP(int count) {
  BatchPoints.Builder b = BatchPoints.database("d1");
  for (int i = 0; i < count; i++) {
    b.point(Point.measurement("x1").addField("x", 1).build()).build();
  }
  return b.build();
}
 
Example #29
Source File: RetryCapableBatchWriterTest.java    From influxdb-java with MIT License 5 votes vote down vote up
@Test
public void testAllNonRecoverableExceptions() {
  
  InfluxDB mockInfluxDB = mock(InfluxDBImpl.class);
  BiConsumer errorHandler = mock(BiConsumer.class);
  RetryCapableBatchWriter rw = new RetryCapableBatchWriter(mockInfluxDB, errorHandler,
          150, 100);

  InfluxDBException nonRecoverable1 = InfluxDBException.buildExceptionForErrorState(createErrorBody("database not found: cvfdgf"));
  InfluxDBException nonRecoverable2 = InfluxDBException.buildExceptionForErrorState(createErrorBody("points beyond retention policy 'abc'"));
  InfluxDBException nonRecoverable3 = InfluxDBException.buildExceptionForErrorState(createErrorBody("unable to parse 'abc'"));
  InfluxDBException nonRecoverable4 = InfluxDBException.buildExceptionForErrorState(createErrorBody("hinted handoff queue not empty service='abc'"));
  InfluxDBException nonRecoverable5 = InfluxDBException.buildExceptionForErrorState(createErrorBody("field type conflict 'abc'"));
  InfluxDBException nonRecoverable6 = new InfluxDBException.RetryBufferOverrunException(createErrorBody("Retry BufferOverrun Exception"));
  InfluxDBException nonRecoverable7 = InfluxDBException.buildExceptionForErrorState(createErrorBody("user is not authorized to write to database"));
  InfluxDBException nonRecoverable8 = InfluxDBException.buildExceptionForErrorState(createErrorBody("authorization failed"));
  InfluxDBException nonRecoverable9 = InfluxDBException.buildExceptionForErrorState(createErrorBody("username required"));

  List<InfluxDBException> exceptions = Arrays.asList(nonRecoverable1, nonRecoverable2, nonRecoverable3,
      nonRecoverable4, nonRecoverable5, nonRecoverable6, nonRecoverable7, nonRecoverable8, nonRecoverable9);
  int size = exceptions.size();
  doAnswer(new TestAnswer() {
    int i = 0;
    @Override
    protected void check(InvocationOnMock invocation) {
      if (i < size) {
        throw exceptions.get(i++);
      }
    }
  }).when(mockInfluxDB).write(any(BatchPoints.class));
  
  BatchPoints bp = getBP(8);
  for (int i = 0; i < size; i++) {
    rw.write(Collections.singletonList(bp));
  }
  verify(errorHandler, times(size)).accept(any(), any());;
}
 
Example #30
Source File: PerformanceTests.java    From influxdb-java with MIT License 5 votes vote down vote up
@Disabled
@Test
public void testWritePerformance() {
	String dbName = "writepoints_" + System.currentTimeMillis();
	this.influxDB.query(new Query("CREATE DATABASE " + dbName));
	String rp = TestUtils.defaultRetentionPolicy(this.influxDB.version());

	long start = System.currentTimeMillis();
	for (int i = 0; i < COUNT; i++) {

		BatchPoints batchPoints = BatchPoints
				.database(dbName)
				.tag("blubber", "bla")
				.retentionPolicy(rp)
				.build();
		for (int j = 0; j < POINT_COUNT; j++) {
			Point point = Point
					.measurement("cpu")
					.addField("idle", (double) j)
					.addField("user", 2.0 * j)
					.addField("system", 3.0 * j)
					.build();
			batchPoints.point(point);
		}

		this.influxDB.write(batchPoints);
	}
	System.out.println("WritePoints for " + COUNT + " writes of " + POINT_COUNT + " Points took:" + (System.currentTimeMillis() - start));
	this.influxDB.query(new Query("DROP DATABASE " + dbName));
}