org.apache.kafka.common.utils.Utils Java Examples

The following examples show how to use org.apache.kafka.common.utils.Utils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClusterTestHarness.java    From kareldb with Apache License 2.0 7 votes vote down vote up
@Before
public void setUp() throws Exception {
    zookeeper = new EmbeddedZookeeper();
    zkConnect = String.format("localhost:%d", zookeeper.port());

    configs = new Vector<>();
    servers = new Vector<>();
    for (int i = 0; i < numBrokers; i++) {
        KafkaConfig config = getKafkaConfig(i);
        configs.add(config);

        KafkaServer server = TestUtils.createServer(config, Time.SYSTEM);
        servers.add(server);
    }

    String[] serverUrls = new String[servers.size()];
    ListenerName listenerType = ListenerName.forSecurityProtocol(getSecurityProtocol());
    for (int i = 0; i < servers.size(); i++) {
        serverUrls[i] =
            Utils.formatAddress(
                servers.get(i).config().advertisedListeners().head().host(),
                servers.get(i).boundPort(listenerType)
            );
    }
    bootstrapServers = Utils.join(serverUrls, ",");
}
 
Example #2
Source File: CustomPartitioner.java    From ad with Apache License 2.0 6 votes vote down vote up
/**
 * 决定消息被写入哪个分区
 * @param topic topic
 * @param key key
 * @param keyBytes key serialize byte
 * @param value value
 * @param valueBytes value serialize byte
 * @param cluster kakfa cluster
 * @return
 */
@Override
public int partition(String topic, Object key, byte[] keyBytes,
                     Object value, byte[] valueBytes, Cluster cluster) {
    // 所有分区信息
    List<PartitionInfo> partitionInfos = cluster.partitionsForTopic(topic);
    int partitionCount = partitionInfos.size();
    // 要求必须存在 key,如果key 是"name" 就分配到最后一个分区, 其他key hash取模
    if (keyBytes == null || !key.getClass().equals(String.class)) {
        throw new InvalidRecordException("kafka message must have a String key");
    }
    if (partitionCount == 1 || StringUtils.endsWithIgnoreCase("name", key.toString())) {
        return partitionCount - 1;
    }
    return Math.abs(Utils.murmur2(keyBytes)) % (partitionCount - 1);
}
 
Example #3
Source File: KafkaRequestHandler.java    From kop with Apache License 2.0 6 votes vote down vote up
protected void handleSyncGroupRequest(KafkaHeaderAndRequest syncGroup,
                                      CompletableFuture<AbstractResponse> resultFuture) {
    checkArgument(syncGroup.getRequest() instanceof SyncGroupRequest);
    SyncGroupRequest request = (SyncGroupRequest) syncGroup.getRequest();

    groupCoordinator.handleSyncGroup(
        request.groupId(),
        request.generationId(),
        request.memberId(),
        CoreUtils.mapValue(
            request.groupAssignment(), Utils::toArray
        )
    ).thenAccept(syncGroupResult -> {
        SyncGroupResponse response = new SyncGroupResponse(
            syncGroupResult.getKey(),
            ByteBuffer.wrap(syncGroupResult.getValue())
        );

        resultFuture.complete(response);
    });
}
 
Example #4
Source File: KarelDbEngine.java    From kareldb with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static <T> T getConfiguredInstance(String className, Map<String, ?> configs) {
    try {
        Class<T> cls = (Class<T>) Class.forName(className);
        if (cls == null) {
            return null;
        }
        Object o = Utils.newInstance(cls);
        if (o instanceof Configurable) {
            ((Configurable) o).configure(configs);
        }
        return cls.cast(o);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    }
}
 
Example #5
Source File: KeyValueSerdeResolver.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
private Serde<?> getKeySerde(String keySerdeString, ResolvableType resolvableType) {
	Serde<?> keySerde = null;
	try {
		if (StringUtils.hasText(keySerdeString)) {
			keySerde = Utils.newInstance(keySerdeString, Serde.class);
		}
		else {
			if (resolvableType != null &&
					(isResolvalbeKafkaStreamsType(resolvableType) || isResolvableKStreamArrayType(resolvableType))) {
				ResolvableType generic = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(0) : resolvableType.getGeneric(0);
				Serde<?> fallbackSerde = getFallbackSerde("default.key.serde");
				keySerde = getSerde(generic, fallbackSerde);
			}
			if (keySerde == null) {
				keySerde = Serdes.ByteArray();
			}
		}
		keySerde.configure(this.streamConfigGlobalProperties, true);
	}
	catch (ClassNotFoundException ex) {
		throw new IllegalStateException("Serde class not found: ", ex);
	}
	return keySerde;
}
 
Example #6
Source File: KeyValueSerdeResolver.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private Serde<?> getValueSerde(String valueSerdeString, ResolvableType resolvableType)
		throws ClassNotFoundException {
	Serde<?> valueSerde = null;
	if (StringUtils.hasText(valueSerdeString)) {
		valueSerde = Utils.newInstance(valueSerdeString, Serde.class);
	}
	else {

		if (resolvableType != null && ((isResolvalbeKafkaStreamsType(resolvableType)) ||
				(isResolvableKStreamArrayType(resolvableType)))) {
			Serde<?> fallbackSerde = getFallbackSerde("default.value.serde");
			ResolvableType generic = resolvableType.isArray() ? resolvableType.getComponentType().getGeneric(1) : resolvableType.getGeneric(1);
			valueSerde = getSerde(generic, fallbackSerde);
		}
		if (valueSerde == null) {

			valueSerde = Serdes.ByteArray();
		}
	}
	valueSerde.configure(streamConfigGlobalProperties, false);
	return valueSerde;
}
 
Example #7
Source File: KeyValueSerdeResolver.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
private Serde<?> getKeySerde(String keySerdeString) {
	Serde<?> keySerde;
	try {
		if (StringUtils.hasText(keySerdeString)) {
			keySerde = Utils.newInstance(keySerdeString, Serde.class);
		}
		else {
			keySerde = getFallbackSerde("default.key.serde");
		}
		keySerde.configure(this.streamConfigGlobalProperties, true);

	}
	catch (ClassNotFoundException ex) {
		throw new IllegalStateException("Serde class not found: ", ex);
	}
	return keySerde;
}
 
Example #8
Source File: DefaultMegabusSource.java    From emodb with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public List<Future> asyncSendFutures(List<Coordinate> coordinateList) {
    List<Future> futures = coordinateList
            .stream()
            // Using the minimum UUID here to make sure the time is always beyond the FCL so that the resolver is certain to put the document in to actual Megabus.
            // This way we wouldn't be in a situation where there is a ref in Ref topic but not in the Megabus topic.
            .map(coordinate -> new MegabusRef(coordinate.getTable(), coordinate.getId(), TimeUUIDs.minimumUuid(), _clock.instant(), MegabusRef.RefType.TOUCH))
            .collect(Collectors.groupingBy(ref -> {
                String key = Coordinate.of(ref.getTable(), ref.getKey()).toString();
                return Utils.toPositive(Utils.murmur2(key.getBytes())) % _topic.getPartitions();
            }, Collectors.toList()))
            .entrySet()
            .stream()
            .map(entry -> _producer.send(new ProducerRecord<>(_topic.getName(), entry.getKey(), TimeUUIDs.newUUID().toString(),
                    _objectMapper.valueToTree(entry.getValue()))))
            .collect(Collectors.toList());

    return futures;
}
 
Example #9
Source File: LiKafkaProducerConfig.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 6 votes vote down vote up
public <T> T getConfiguredInstance(String key, Class<T> t, Producer<byte[], byte[]> producer) {
  Class<?> c = getClass(key);
  if (c == null) {
    return null;
  }
  Object o = Utils.newInstance(c);

  if (!t.isInstance(o)) {
    throw new KafkaException(c.getName() + " is not an instance of " + t.getName());
  }

  if (o instanceof Configurable) {
    ((Configurable) o).configure(configsWithCurrentProducer(producer));
  }

  return t.cast(o);
}
 
Example #10
Source File: ConnectEmbedded.java    From hello-kafka-streams with Apache License 2.0 6 votes vote down vote up
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception {
    Time time = new SystemTime();
    DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig));

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);

    //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random
    String workerId = UUID.randomUUID().toString();
    worker = new Worker(workerId, time, config, offsetBackingStore);

    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter());
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter());
    configBackingStore.configure(config);

    //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything
    herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, "");
    this.connectorConfigs = connectorConfigs;

    shutdownHook = new ShutdownHook();
}
 
Example #11
Source File: EndToEndIntegrationTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldSelectAllFromUsers() throws Exception {
  final QueuedQueryMetadata queryMetadata = executeQuery(
      "SELECT * from %s;", userTable);

  BlockingQueue<KeyValue<String, GenericRow>> rowQueue = queryMetadata.getRowQueue();

  Set<String> actualUsers = new HashSet<>();
  Set<String> expectedUsers = Utils.mkSet("USER_0", "USER_1", "USER_2", "USER_3", "USER_4");
  while (actualUsers.size() < expectedUsers.size()) {
    KeyValue<String, GenericRow> nextRow = rowQueue.poll();
    if (nextRow != null) {
      List<Object> columns = nextRow.value.getColumns();
      assertEquals(6, columns.size());
      actualUsers.add((String) columns.get(1));
    }
  }
  assertEquals(expectedUsers, actualUsers);
}
 
Example #12
Source File: SerdeCryptoBase.java    From kafka-end-2-end-encryption with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
protected <T> T newInstance(Map<String, ?> map, String key, Class<T> klass) throws KafkaException {
    Object val = map.get(key);
    if (val == null) {
        throw new KafkaException("No value for '" + key + "' found");
    } else if (val instanceof String) {
        try {
            return (T) Utils.newInstance(Class.forName((String) val));
        } catch (Exception e) {
            throw new KafkaException(e);
        }
    } else if (val instanceof Class) {
        return (T) Utils.newInstance((Class<T>) val);
    } else {
        throw new KafkaException("Unexpected type '" + val.getClass() + "' for '" + key + "'");
    }
}
 
Example #13
Source File: KafkaIOTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public Producer<Integer, Long> apply(Map<String, Object> config) {

  // Make sure the config is correctly set up for serializers.
  Utils.newInstance(
          (Class<? extends Serializer<?>>)
              ((Class<?>) config.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
                  .asSubclass(Serializer.class))
      .configure(config, true);

  Utils.newInstance(
          (Class<? extends Serializer<?>>)
              ((Class<?>) config.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG))
                  .asSubclass(Serializer.class))
      .configure(config, false);

  // Returning same producer in each instance in a pipeline seems to work fine currently.
  // If DirectRunner creates multiple DoFn instances for sinks, we might need to handle
  // it appropriately. I.e. allow multiple producers for each producerKey and concatenate
  // all the messages written to each producer for verification after the pipeline finishes.

  return MOCK_PRODUCER_MAP.get(producerKey);
}
 
Example #14
Source File: ClusterStatusSASLTest.java    From common-docker with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 120000)
public void isKafkaReadyWithSASLAndSSLUsingZK() throws Exception {
  Properties clientSecurityProps = kafka.getClientSecurityConfig();

  boolean zkReady = ClusterStatus.isZookeeperReady(this.kafka.getZookeeperConnectString(), 30000);
  if (!zkReady) {
    throw new RuntimeException(
        "Could not reach zookeeper " + this.kafka.getZookeeperConnectString());
  }
  Map<String, String> endpoints = ClusterStatus.getKafkaEndpointFromZookeeper(
      this.kafka.getZookeeperConnectString(),
      30000
  );

  String bootstrap_broker = endpoints.get("SASL_SSL");
  Map<String, String> config = Utils.propsToStringMap(clientSecurityProps);
  config.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, bootstrap_broker);

  // Set password and enabled protocol as the Utils.propsToStringMap just returns toString()
  // representations and these properties don't have a valid representation.
  Password trustStorePassword = (Password) clientSecurityProps.get("ssl.truststore.password");
  config.put("ssl.truststore.password", trustStorePassword.value());
  config.put("ssl.enabled.protocols", "TLSv1.2");

  assertThat(ClusterStatus.isKafkaReady(config, 3, 10000)).isTrue();
}
 
Example #15
Source File: ClusterStatusSASLTest.java    From common-docker with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 120000)
public void isKafkaReadyWithSASLAndSSL() throws Exception {
  Properties clientSecurityProps = kafka.getClientSecurityConfig();

  Map<String, String> config = Utils.propsToStringMap(clientSecurityProps);
  config.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapBroker
      (SecurityProtocol.SASL_SSL));

  // Set password and enabled protocol as the Utils.propsToStringMap just returns toString()
  // representations and these properties don't have a valid representation.
  Password trustStorePassword = (Password) clientSecurityProps.get("ssl.truststore.password");
  config.put("ssl.truststore.password", trustStorePassword.value());
  config.put("ssl.enabled.protocols", "TLSv1.2");

  assertThat(ClusterStatus.isKafkaReady(config, 3, 10000)).isTrue();
}
 
Example #16
Source File: ClusterTestHarness.java    From kcache with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    zookeeper = new EmbeddedZookeeper();
    zkConnect = String.format("localhost:%d", zookeeper.port());

    configs = new Vector<>();
    servers = new Vector<>();
    for (int i = 0; i < numBrokers; i++) {
        KafkaConfig config = getKafkaConfig(i);
        configs.add(config);

        KafkaServer server = TestUtils.createServer(config, Time.SYSTEM);
        servers.add(server);
    }

    String[] serverUrls = new String[servers.size()];
    ListenerName listenerType = ListenerName.forSecurityProtocol(getSecurityProtocol());
    for (int i = 0; i < servers.size(); i++) {
        serverUrls[i] =
            Utils.formatAddress(
                servers.get(i).config().advertisedListeners().head().host(),
                servers.get(i).boundPort(listenerType)
            );
    }
    bootstrapServers = Utils.join(serverUrls, ",");
}
 
Example #17
Source File: SessionHandler.java    From DataLink with Apache License 2.0 6 votes vote down vote up
private void handleJoinGroupRequest(ChannelHandlerContext ctx, Request request) {
    JoinGroupRequest joinGroupRequest = (JoinGroupRequest) request.getBody();
    ResponseHeader responseHeader = new ResponseHeader(request.getHeader().correlationId());

    List<ProtocolEntry> protocols = joinGroupRequest.groupProtocols().stream().map(protocol -> new ProtocolEntry(protocol.name(), Utils.toArray(protocol.metadata()))).collect(Collectors.toList());
    coordinator.handleJoinGroup(
            joinGroupRequest.groupId(),
            joinGroupRequest.memberId(),
            request.getHeader().clientId(),
            request.getClientAddress().toString(),
            joinGroupRequest.rebalanceTimeout(),
            joinGroupRequest.sessionTimeout(),
            joinGroupRequest.protocolType(),
            protocols,
            (joinResult) -> {
                Map<String, ByteBuffer> members = joinResult.getMembers().entrySet().stream().collect(Collectors.toMap(k -> k.getKey(), k -> ByteBuffer.wrap(k.getValue())));
                JoinGroupResponse responseBody = new JoinGroupResponse(request.getHeader().apiVersion(), joinResult.getErrorCode(), joinResult.getGenerationId(),
                        joinResult.getSubProtocol(), joinResult.getMemberId(), joinResult.getLeaderId(), members);

                logger.trace(String.format("Sending join group response %s for correlation id %d to client %s.",
                        responseBody, request.getHeader().correlationId(), request.getHeader().clientId()));

                sendResponse(ctx, new Response(responseHeader, responseBody));
            }
    );
}
 
Example #18
Source File: ClientUtils.java    From kafka-graphs with Apache License 2.0 6 votes vote down vote up
/**
 * Create a temporary relative directory in the specified parent directory with the given prefix.
 *
 * @param parent The parent folder path name, if null using the default temporary-file directory
 * @param prefix The prefix of the temporary directory, if null using "kafka-" as default prefix
 * @return the temp dir
 */
public static File tempDirectory(final Path parent, String prefix) {
    final File file;
    prefix = prefix == null ? "kafka-" : prefix;
    try {
        file = parent == null
            ?
            Files.createTempDirectory(prefix).toFile()
            : Files.createTempDirectory(parent, prefix).toFile();
    } catch (final IOException ex) {
        throw new RuntimeException("Failed to create a temp dir", ex);
    }
    file.deleteOnExit();

    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
        try {
            Utils.delete(file);
        } catch (IOException e) {
            log.error("Error deleting {}", file.getAbsolutePath(), e);
        }
    }));

    return file;
}
 
Example #19
Source File: MirusOffsetTool.java    From mirus with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private static MirusOffsetTool newOffsetTool(Args args) throws IOException {
  // This needs to be the admin topic properties.
  // By default these are in the worker properties file, as this has the has admin producer and
  // consumer settings.  Separating these might be wise - also useful for storing state in
  // source cluster if it proves necessary.
  final Map<String, String> properties =
      !args.propertiesFile.isEmpty()
          ? Utils.propsToStringMap(Utils.loadProps(args.propertiesFile))
          : Collections.emptyMap();
  final DistributedConfig config = new DistributedConfig(properties);
  final KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
  offsetBackingStore.configure(config);

  // Avoid initializing the entire Kafka Connect plugin system by assuming the
  // internal.[key|value].converter is org.apache.kafka.connect.json.JsonConverter
  final Converter internalConverter = new JsonConverter();
  internalConverter.configure(config.originalsWithPrefix("internal.key.converter."), true);

  final OffsetSetter offsetSetter = new OffsetSetter(internalConverter, offsetBackingStore);
  final OffsetFetcher offsetFetcher = new OffsetFetcher(config, internalConverter);
  final OffsetSerDe offsetSerDe = OffsetSerDeFactory.create(args.format);

  return new MirusOffsetTool(args, offsetFetcher, offsetSetter, offsetSerDe);
}
 
Example #20
Source File: WorkerUtils.java    From uReplicator with Apache License 2.0 5 votes vote down vote up
public static Properties loadProperties(String configFile) {
  if (StringUtils.isEmpty(configFile)) {
    return null;
  }

  File file = new File(configFile);
  if (!file.exists()) {
    throw new IllegalArgumentException(String.format("File %s not found", configFile));
  }
  try {
    return Utils.loadProps(configFile);
  } catch (IOException e) {
    throw new IllegalArgumentException(String.format("Load config file %s failed", configFile), e);
  }
}
 
Example #21
Source File: KeyModPartitioner.java    From SkyEye with GNU General Public License v3.0 5 votes vote down vote up
@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    int partitionNum = 0;
    try {
        partitionNum = Utils.murmur2(keyBytes);
    } catch (Exception e) {
        partitionNum = key.hashCode();
    }

    return Math.abs(partitionNum % numPartitions);
}
 
Example #22
Source File: SessionHandler.java    From DataLink with Apache License 2.0 5 votes vote down vote up
private void handleSyncGroupRequest(ChannelHandlerContext ctx, Request request) {
    SyncGroupRequest syncGroupRequest = (SyncGroupRequest) request.getBody();
    coordinator.handleSyncGroup(
            syncGroupRequest.groupId(),
            syncGroupRequest.generationId(),
            syncGroupRequest.memberId(),
            syncGroupRequest.groupAssignment().entrySet().stream().collect(Collectors.toMap(k -> k.getKey(), k -> Utils.toArray(k.getValue()))),
            (memberState, errorCode) -> {
                SyncGroupResponse responseBody = new SyncGroupResponse(errorCode, ByteBuffer.wrap(memberState));
                ResponseHeader responseHeader = new ResponseHeader(request.getHeader().correlationId());
                sendResponse(ctx, new Response(responseHeader, responseBody));
            }
    );
}
 
Example #23
Source File: PartitionTest.java    From SkyEye with GNU General Public License v3.0 5 votes vote down vote up
public static void main(String[] args) {
    String app = "app-test1";
    String host = "jthink";
    int numPartitions = 9;

    String key = new StringBuilder(app).append(host).toString();
    byte[] keyBytes= ByteBuffer.allocate(4).putInt(key.hashCode()).array();
    int partitionNum = 0;
    try {
        partitionNum = Utils.murmur2(keyBytes);
    } catch (Exception e) {
        partitionNum = key.hashCode();
    }
    System.out.println(Math.abs(partitionNum % numPartitions));
}
 
Example #24
Source File: MegabusRefProducer.java    From emodb with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
boolean peekAndAckEvents() {
    // Poll for events on the megabus subscription
    long startTime = _clock.instant().getNano();
    List<EventData> result = _eventStore.peek(_subscriptionName, _eventsLimit);

    List<Future> futures = result.stream()
            .map(eventData -> UpdateRefSerializer.fromByteBuffer(eventData.getData()))
            .map(ref -> new MegabusRef(ref.getTable(), ref.getKey(), ref.getChangeId(), _clock.instant(), MegabusRef.RefType.NORMAL))
            .collect(Collectors.groupingBy(ref -> {
                String key = Coordinate.of(ref.getTable(), ref.getKey()).toString();
                return Utils.toPositive(Utils.murmur2(key.getBytes())) % _topic.getPartitions();
            }, Collectors.toList()))
            .entrySet()
            .stream()
            .map(entry -> _producer.send(new ProducerRecord<>(_topic.getName(), entry.getKey(), TimeUUIDs.newUUID().toString(),
                            _objectMapper.valueToTree(entry.getValue()))))
            .collect(Collectors.toList());

    // Last chance to check that we are the leader before doing anything that would be bad if we aren't.
    if (!isRunning()) {
        return false;
    }

    _producer.flush();

    futures.forEach(Futures::getUnchecked);

    long endTime = _clock.instant().getNano();
    trackAverageEventDuration(endTime - startTime, result.size());

    if (!result.isEmpty()) {
        _eventStore.delete(_subscriptionName, result.stream().map(EventData::getId).collect(Collectors.toList()), false);
    }

    return result.size() >= _skipWaitThreshold;
}
 
Example #25
Source File: OffsetCheckpoint.java    From kcache with Apache License 2.0 5 votes vote down vote up
/**
 * Write the given offsets to the checkpoint file. All offsets should be non-negative.
 *
 * @throws IOException if any file operation fails with an IO exception
 */
public void write(final Map<TopicPartition, Long> offsets) throws IOException {
    // if there is no offsets, skip writing the file to save disk IOs
    if (offsets.isEmpty()) {
        return;
    }

    synchronized (lock) {
        // write to temp file and then swap with the existing file
        final File temp = new File(file.getAbsolutePath() + ".tmp");
        LOG.trace("Writing tmp checkpoint file {}", temp.getAbsolutePath());

        final FileOutputStream fileOutputStream = new FileOutputStream(temp);
        try (final BufferedWriter writer = new BufferedWriter(
                new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8))) {
            writeIntLine(writer, VERSION);
            writeIntLine(writer, offsets.size());

            for (final Map.Entry<TopicPartition, Long> entry : offsets.entrySet()) {
                final TopicPartition tp = entry.getKey();
                final Long offset = entry.getValue();
                if (offset >= 0L) {
                    writeEntry(writer, tp, offset);
                } else {
                    LOG.warn("Received offset={} to write to checkpoint file for {}", offset, tp);
                }
            }

            writer.flush();
            fileOutputStream.getFD().sync();
        }

        LOG.trace("Swapping tmp checkpoint file {} {}", temp.toPath(), file.toPath());
        Utils.atomicMoveWithFallback(temp.toPath(), file.toPath());
    }
}
 
Example #26
Source File: CouchbaseSourceTask.java    From kafka-connect-couchbase with Apache License 2.0 5 votes vote down vote up
@Override
public void start(Map<String, String> properties) {
  this.connectorName = properties.get("name");

  CouchbaseSourceTaskConfig config;
  try {
    config = ConfigHelper.parse(CouchbaseSourceTaskConfig.class, properties);
    if (isNullOrEmpty(connectorName)) {
      throw new ConfigException("Connector must have a non-blank 'name' config property.");
    }
  } catch (ConfigException e) {
    throw new ConnectException("Couldn't start CouchbaseSourceTask due to configuration error", e);
  }

  LogRedaction.setRedactionLevel(config.logRedaction());
  RedactionLevel.set(toDcp(config.logRedaction()));

  filter = Utils.newInstance(config.eventFilter());
  sourceHandler = Utils.newInstance(config.sourceHandler());

  topic = config.topic();
  bucket = config.bucket();
  connectorNameInOffsets = config.connectorNameInOffsets();
  batchSizeMax = config.batchSizeMax();

  Short[] partitions = toBoxedShortArray(config.partitions());
  Map<Short, SeqnoAndVbucketUuid> partitionToSavedSeqno = readSourceOffsets(partitions);

  running = true;
  queue = new LinkedBlockingQueue<>();
  errorQueue = new LinkedBlockingQueue<>(1);
  couchbaseReader = new CouchbaseReader(config, connectorName, queue, errorQueue, partitions, partitionToSavedSeqno);
  couchbaseReader.start();
}
 
Example #27
Source File: KafkaMQClientHandler.java    From Thunder with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("resource")
private int getPartitionIndex(Consumer<String, byte[]> consumer, String topic, String key) {
    int partitionNumber = consumer.partitionsFor(topic).size();

    StringSerializer keySerializer = new StringSerializer();
    byte[] serializedKey = keySerializer.serialize(topic, key);

    int positive = Utils.murmur2(serializedKey) & 0x7fffffff;

    return positive % partitionNumber;
}
 
Example #28
Source File: ClientUtils.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
public static <T> T getConfiguredInstance(Class<T> cls, Map<String, ?> configs) {
    if (cls == null) {
        return null;
    }
    Object o = Utils.newInstance(cls);
    if (o instanceof Configurable) {
        ((Configurable) o).configure(configs);
    }
    return cls.cast(o);
}
 
Example #29
Source File: DefaultPartitioner.java    From jeesuite-libs with Apache License 2.0 5 votes vote down vote up
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
      List<PartitionInfo> partitions = cluster.availablePartitionsForTopic(topic);
      int numPartitions = partitions.size();

      try {			
      	long partitionHash = ((DefaultMessage)value).partitionHash();
      	//按hash分区
      	if(partitionHash > 0){
      		long index = partitionHash % numPartitions;
      		//System.out.println("numPartitions:"+numPartitions+",partitionHash:"+partitionHash + ",index:"+index);
      		return (int)index;
      	}
} catch (ClassCastException e) {}
      
      if (keyBytes == null) {
          int nextValue = counter.getAndIncrement();
          List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic);
          if (availablePartitions.size() > 0) {
              int part = DefaultPartitioner.toPositive(nextValue) % availablePartitions.size();
              return availablePartitions.get(part).partition();
          } else {
              // no partitions are available, give a non-available partition
              return DefaultPartitioner.toPositive(nextValue) % numPartitions;
          }
      } else {
          // hash the keyBytes to choose a partition
          return DefaultPartitioner.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
      }
  }
 
Example #30
Source File: KeyValueSerdeResolver.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private Serde<?> getFallbackSerde(String s) throws ClassNotFoundException {
	return this.binderConfigurationProperties.getConfiguration()
			.containsKey(s)
			? Utils.newInstance(this.binderConfigurationProperties
					.getConfiguration().get(s),
			Serde.class)
			: Serdes.ByteArray();
}