org.apache.kudu.client.SessionConfiguration.FlushMode Java Examples

@Override
public void deleteRows(Block rowIds) {
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        } finally {
            session.close();
        }
    } catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public void deleteRows(Block rowIds)
{
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        }
        finally {
            session.close();
        }
    }
    catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * If flink checkpoint is disable,synchronously write data to kudu.
 * <p>If flink checkpoint is enable, asynchronously write data to kudu by default.
 *
 * <p>(Note: async may result in out-of-order writes to Kudu.
 *  you also can change to sync by explicitly calling {@link KuduSink#withSyncFlushMode()} when initializing KuduSink. )
 *
 * @return flushMode
 */
private FlushMode getflushMode() {
    FlushMode flushMode = FlushMode.AUTO_FLUSH_SYNC;
    boolean enableCheckpoint = ((StreamingRuntimeContext) getRuntimeContext()).isCheckpointingEnabled();
    if(enableCheckpoint && this.flushMode == null) {
        flushMode = FlushMode.AUTO_FLUSH_BACKGROUND;
    }
    if(enableCheckpoint && this.flushMode != null) {
        flushMode = this.flushMode;
    }
    return flushMode;
}
 

public KuduConnector(String kuduMasters, KuduTableInfo tableInfo, Consistency consistency, WriteMode writeMode, FlushMode flushMode) throws IOException {
    this.client = client(kuduMasters);
    this.table = table(tableInfo);
    this.session = client.newSession();
    this.consistency = consistency;
    this.writeMode = writeMode;
    this.defaultCB = new ResponseCallback();
    this.session.setFlushMode(flushMode);
}
 

public KuduConnector(String kuduMasters, KuduTableInfo tableInfo, Consistency consistency, WriteMode writeMode, FlushMode flushMode) throws IOException {
    this.client = client(kuduMasters);
    this.table = table(tableInfo);
    this.session = client.newSession();
    this.consistency = consistency;
    this.writeMode = writeMode;
    this.defaultCB = new ResponseCallback();
    this.session.setFlushMode(flushMode);
}
 

/**
 * If flink checkpoint is disable,synchronously write data to kudu.
 * <p>If flink checkpoint is enable, asynchronously write data to kudu by default.
 *
 * <p>(Note: async may result in out-of-order writes to Kudu.
 *  you also can change to sync by explicitly calling {@link KuduSink#withSyncFlushMode()} when initializing KuduSink. )
 *
 * @return flushMode
 */
private FlushMode getflushMode() {
    FlushMode flushMode = FlushMode.AUTO_FLUSH_SYNC;
    boolean enableCheckpoint = ((StreamingRuntimeContext) getRuntimeContext()).isCheckpointingEnabled();
    if(enableCheckpoint && this.flushMode == null) {
        flushMode = FlushMode.AUTO_FLUSH_BACKGROUND;
    }
    if(enableCheckpoint && this.flushMode != null) {
        flushMode = this.flushMode;
    }
    return flushMode;
}
 

public KuduSink<OUT> withAsyncFlushMode() {
    this.flushMode = FlushMode.AUTO_FLUSH_BACKGROUND;
    return this;
}
 

@Test
public void testKuduPartialFailuresOnManualFlush() throws Exception {
    testKuduPartialFailure(FlushMode.MANUAL_FLUSH);
}
 

@Test
public void testKuduPartialFailuresOnAutoFlushBackground() throws Exception {
    testKuduPartialFailure(FlushMode.AUTO_FLUSH_BACKGROUND);
}
 

@Test
public void testKuduPartialFailuresOnAutoFlushSync() throws Exception {
    testKuduPartialFailure(FlushMode.AUTO_FLUSH_SYNC);
}
 

private void testKuduPartialFailure(FlushMode flushMode) throws Exception {
  // Test against different batch sizes (up until the point where every record can be buffered at once)
  for (int i = 1; i <= 11; i++) {
      testKuduPartialFailure(flushMode, i);
  }
}
 

public KuduConnector(String kuduMasters, KuduTableInfo tableInfo) throws IOException {
    this(kuduMasters, tableInfo, KuduConnector.Consistency.STRONG, KuduConnector.WriteMode.UPSERT, FlushMode.AUTO_FLUSH_SYNC);
}
 

public KuduSink<OUT> withSyncFlushMode() {
    this.flushMode = FlushMode.AUTO_FLUSH_SYNC;
    return this;
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
    if (connector != null) return;
    connector = new KuduConnector(kuduMasters, tableInfo, consistency, writeMode,FlushMode.AUTO_FLUSH_SYNC);
    serializer = serializer.withSchema(tableInfo.getSchema());
}
 

public KuduConnector(String kuduMasters, KuduTableInfo tableInfo) throws IOException {
    this(kuduMasters, tableInfo, KuduConnector.Consistency.STRONG, KuduConnector.WriteMode.UPSERT, FlushMode.AUTO_FLUSH_SYNC);
}
 

public KuduSink<OUT> withAsyncFlushMode() {
    this.flushMode = FlushMode.AUTO_FLUSH_BACKGROUND;
    return this;
}
 

public KuduSink<OUT> withSyncFlushMode() {
    this.flushMode = FlushMode.AUTO_FLUSH_SYNC;
    return this;
}
 

@Override
public void open(int taskNumber, int numTasks) throws IOException {
    if (connector != null) return;
    connector = new KuduConnector(kuduMasters, tableInfo, consistency, writeMode,FlushMode.AUTO_FLUSH_SYNC);
    serializer = serializer.withSchema(tableInfo.getSchema());
}