io.openmessaging.KeyValue Java Examples

public void init() {
    //init producer
    try {
        Class kvClass = Class.forName("io.openmessaging.demo.DefaultKeyValue");
        KeyValue keyValue = (KeyValue) kvClass.newInstance();
        keyValue.put("STORE_PATH", Constants.STORE_PATH);
        Class producerClass = Class.forName("io.openmessaging.v4.V4Producer");
        producer = (Producer) producerClass.getConstructor(new Class[]{KeyValue.class}).newInstance(new Object[]{keyValue});
        if (producer == null) {
            throw new InstantiationException("Init Producer Failed");
        }
    } catch (Exception e) {
        logger.error("please check the package name and class name:", e);
    }
    //init offsets,topic0-9,queue0-9
    for (int i = 0; i < 10; i++) {
        offsets.put("TOPIC_" + i, 0);
        offsets.put("QUEUE_" + i, 0);
    }

}
 

@Override
public void start(KeyValue props) {

    try {
        this.config = new Config();
        this.config.load(props);
        ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
                                        ByteBuffer.wrap(
                                            MysqlConstants.getPartition(config.mysqlAddr, config.mysqlPort).getBytes("UTF-8")));

        if (null != positionInfo && positionInfo.array().length > 0) {

            String positionJson = new String(positionInfo.array(), "UTF-8");
            JSONObject jsonObject = JSONObject.parseObject(positionJson);
            this.config.startType = "SPECIFIED";
            this.config.binlogFilename = jsonObject.getString(MysqlConstants.BINLOG_FILENAME);
            this.config.nextPosition = jsonObject.getLong(MysqlConstants.NEXT_POSITION);

        }
        this.replicator = new Replicator(config);
    } catch (Exception e) {
        log.error("Mysql task start failed.", e);
    }
    this.replicator.start();
}
 

public static <T> T populate(final KeyValue properties, final T obj) {
    Class<?> clazz = obj.getClass();
    try {

        final Set<String> keySet = properties.keySet();
        for (String key : keySet) {
            String[] keyGroup = key.split("[\\._]");
            for (int i = 0; i < keyGroup.length; i++) {
                keyGroup[i] = keyGroup[i].toLowerCase();
                keyGroup[i] = StringUtils.capitalize(keyGroup[i]);
            }
            String beanFieldNameWithCapitalization = StringUtils.join(keyGroup);
            try {
                setProperties(clazz, obj, "set" + beanFieldNameWithCapitalization, properties.getString(key));
            } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
                //ignored...
            }
        }
    } catch (RuntimeException e) {
        log.warn("Error occurs !", e);
    }
    return obj;
}
 

public static ProducerConfig convertProducerConfig(KeyValue attributes) {
    ProducerConfig producerConfig = new ProducerConfig();
    producerConfig.setTimeout(attributes.getLong(JoyQueueProducerBuiltinKeys.TIMEOUT, producerConfig.getTimeout()));
    producerConfig.setProduceTimeout(attributes.getLong(JoyQueueProducerBuiltinKeys.PRODUCE_TIMEOUT, producerConfig.getProduceTimeout()));
    producerConfig.setTransactionTimeout(attributes.getLong(JoyQueueProducerBuiltinKeys.TRANSACTION_TIMEOUT, producerConfig.getTransactionTimeout()));
    producerConfig.setFailover(attributes.getBoolean(JoyQueueProducerBuiltinKeys.FAILOVER, producerConfig.isFailover()));
    producerConfig.getRetryPolicy().setMaxRetrys(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.RETRIES, producerConfig.getRetryPolicy().getMaxRetrys()));
    producerConfig.setQosLevel(QosLevel.valueOf(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.QOSLEVEL, producerConfig.getQosLevel().value())));
    producerConfig.setCompress(attributes.getBoolean(JoyQueueProducerBuiltinKeys.COMPRESS, producerConfig.isCompress()));
    producerConfig.setCompressType(KeyValueHelper.getString(attributes, JoyQueueProducerBuiltinKeys.COMPRESS_TYPE, producerConfig.getCompressType()));
    producerConfig.setCompressThreshold(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.COMPRESS_THRESHOLD, producerConfig.getCompressThreshold()));
    producerConfig.setBatch(attributes.getBoolean(JoyQueueProducerBuiltinKeys.BATCH, producerConfig.isBatch()));
    producerConfig.setSelectorType(KeyValueHelper.getString(attributes, JoyQueueProducerBuiltinKeys.SELECTOR_TYPE, producerConfig.getSelectorType()));
    producerConfig.setBusinessIdLengthLimit(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.BUSINESSID_LENGTH_LIMIT, producerConfig.getBusinessIdLengthLimit()));
    producerConfig.setBodyLengthLimit(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.BODY_LENGTH_LIMIT, producerConfig.getBodyLengthLimit()));
    producerConfig.setBatchBodyLengthLimit(KeyValueHelper.getInt(attributes, JoyQueueProducerBuiltinKeys.BATCH_BODY_LENGTH_LIMIT, producerConfig.getBatchBodyLengthLimit()));
    return producerConfig;
}
 

AbstractOMSProducer(final KeyValue properties) {
    this.properties = properties;
    this.rocketmqProducer = new DefaultMQProducer();
    this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);

    if ("true".equalsIgnoreCase(System.getenv("OMS_RMQ_DIRECT_NAME_SRV"))) {
        String accessPoints = clientConfig.getAccessPoints();
        if (accessPoints == null || accessPoints.isEmpty()) {
            throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
        }

        this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';'));
    }

    this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup());

    String producerId = buildInstanceName();
    this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOperationTimeout());
    this.rocketmqProducer.setInstanceName(producerId);
    this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4);
    this.rocketmqProducer.setLanguage(LanguageCode.OMS);
    properties.put(OMSBuiltinKeys.PRODUCER_ID, producerId);
}
 

AbstractOMSProducer(final KeyValue properties) {
    this.properties = properties;
    this.rocketmqProducer = new DefaultMQProducer();
    this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);

    String accessPoints = clientConfig.getOmsAccessPoints();
    if (accessPoints == null || accessPoints.isEmpty()) {
        throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
    }
    this.rocketmqProducer.setNamesrvAddr(accessPoints.replace(',', ';'));
    this.rocketmqProducer.setProducerGroup(clientConfig.getRmqProducerGroup());

    String producerId = buildInstanceName();
    this.rocketmqProducer.setSendMsgTimeout(clientConfig.getOmsOperationTimeout());
    this.rocketmqProducer.setInstanceName(producerId);
    this.rocketmqProducer.setMaxMessageSize(1024 * 1024 * 4);
    properties.put(PropertyKeys.PRODUCER_ID, producerId);
}
 

public static <T> T populate(final KeyValue properties, final Class<T> clazz) {
    T obj = null;
    try {
        obj = clazz.newInstance();
        return populate(properties, obj);
    } catch (Throwable e) {
        log.warn("Error occurs !", e);
    }
    return obj;
}
 

public static void main(String[] args) throws Exception {
        KeyValue keyValue = OMS.newKeyValue();
        keyValue.put(OMSBuiltinKeys.ACCOUNT_KEY, "test_token");

        MessagingAccessPoint messagingAccessPoint = OMS.getMessagingAccessPoint("oms:joyqueue://[email protected]:50088/UNKNOWN", keyValue);

        ExtensionProducer extensionProducer = (ExtensionProducer) messagingAccessPoint.createProducer();
        extensionProducer.start();

        ExtensionTransactionalResult transactionalResult = extensionProducer.prepare();

        for (int i = 0; i < 10; i++) {
            // 可以发多条事务消息
            Message message = extensionProducer.createMessage("test_topic_0", "body".getBytes());

            // 添加事务id，设置过事务id的才会被补偿，补偿时会带上这个事务id，非必填
            // 建议根据业务使用有意义的事务id
            message.extensionHeader().get().setTransactionId("test_transactionId");

            SendResult sendResult = transactionalResult.send(message);
            System.out.println(sendResult.messageId());
        }

        // 提交事务
        transactionalResult.commit();

        // 回滚事务
//        transactionalResult.rollback();
    }
 

@Override
public String verifyAndSetConfig(KeyValue config) {

    for(String requestKey : Config.REQUEST_CONFIG){
        if(!config.containsKey(requestKey)){
            return "Request config key: " + requestKey;
        }
    }
    this.config = config;
    return "";
}
 

public PullConsumerImpl(final String queueName, final KeyValue properties) {
    this.properties = properties;
    this.targetQueueName = queueName;

    this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);

    String consumerGroup = clientConfig.getRmqConsumerGroup();
    if (null == consumerGroup || consumerGroup.isEmpty()) {
        throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it.");
    }
    pullConsumerScheduleService = new MQPullConsumerScheduleService(consumerGroup);

    this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer();

    String accessPoints = clientConfig.getOmsAccessPoints();
    if (accessPoints == null || accessPoints.isEmpty()) {
        throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
    }
    this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));

    this.rocketmqPullConsumer.setConsumerGroup(consumerGroup);

    int maxReDeliveryTimes = clientConfig.getRmqMaxRedeliveryTimes();
    this.rocketmqPullConsumer.setMaxReconsumeTimes(maxReDeliveryTimes);

    String consumerId = OMSUtil.buildInstanceName();
    this.rocketmqPullConsumer.setInstanceName(consumerId);
    properties.put(PropertyKeys.CONSUMER_ID, consumerId);

    this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig);
}
 

protected static Producer buildProducer(ConsoleProducerConfig config) {
    KeyValue keyValue = OMS.newKeyValue();
    keyValue.put(OMSBuiltinKeys.ACCOUNT_KEY, config.getToken());
    keyValue.put(JoyQueueNameServerBuiltinKeys.NAMESPACE, StringUtils.defaultIfBlank(config.getNamespace(), ToolConsts.DEFAULT_NAMESPACE));

    for (Map.Entry<String, String> entry : config.getParams().entrySet()) {
        keyValue.put(entry.getKey(), entry.getValue());
    }

    MessagingAccessPoint messagingAccessPoint = OMS.getMessagingAccessPoint(
            String.format("oms:%s://%s@%s/%s",
                    ToolConsts.DRIVER, config.getApp(), config.getBootstrap(), StringUtils.defaultIfBlank(config.getRegion(), ToolConsts.DEFAULT_REGION)), keyValue);
    return messagingAccessPoint.createProducer();
}
 

public PullConsumerImpl(final String queueName, final KeyValue properties) {
    this.properties = properties;
    this.targetQueueName = queueName;

    this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);

    String consumerGroup = clientConfig.getRmqConsumerGroup();
    if (null == consumerGroup || consumerGroup.isEmpty()) {
        throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it.");
    }
    pullConsumerScheduleService = new MQPullConsumerScheduleService(consumerGroup);

    this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer();

    String accessPoints = clientConfig.getOmsAccessPoints();
    if (accessPoints == null || accessPoints.isEmpty()) {
        throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
    }
    this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));

    this.rocketmqPullConsumer.setConsumerGroup(consumerGroup);

    int maxReDeliveryTimes = clientConfig.getRmqMaxRedeliveryTimes();
    this.rocketmqPullConsumer.setMaxReconsumeTimes(maxReDeliveryTimes);

    String consumerId = OMSUtil.buildInstanceName();
    this.rocketmqPullConsumer.setInstanceName(consumerId);
    properties.put(PropertyKeys.CONSUMER_ID, consumerId);

    this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig);
}
 

public PullConsumerImpl(final KeyValue properties) {
    this.properties = properties;
    this.clientConfig = BeanUtils.populate(properties, ClientConfig.class);

    String consumerGroup = clientConfig.getConsumerId();
    if (null == consumerGroup || consumerGroup.isEmpty()) {
        throw new OMSRuntimeException("-1", "Consumer Group is necessary for RocketMQ, please set it.");
    }
    pullConsumerScheduleService = new MQPullConsumerScheduleService(consumerGroup);

    this.rocketmqPullConsumer = pullConsumerScheduleService.getDefaultMQPullConsumer();

    if ("true".equalsIgnoreCase(System.getenv("OMS_RMQ_DIRECT_NAME_SRV"))) {
        String accessPoints = clientConfig.getAccessPoints();
        if (accessPoints == null || accessPoints.isEmpty()) {
            throw new OMSRuntimeException("-1", "OMS AccessPoints is null or empty.");
        }
        this.rocketmqPullConsumer.setNamesrvAddr(accessPoints.replace(',', ';'));
    }

    this.rocketmqPullConsumer.setConsumerGroup(consumerGroup);

    int maxReDeliveryTimes = clientConfig.getRmqMaxRedeliveryTimes();
    this.rocketmqPullConsumer.setMaxReconsumeTimes(maxReDeliveryTimes);

    String consumerId = OMSUtil.buildInstanceName();
    this.rocketmqPullConsumer.setInstanceName(consumerId);
    properties.put(OMSBuiltinKeys.CONSUMER_ID, consumerId);

    this.rocketmqPullConsumer.setLanguage(LanguageCode.OMS);

    this.localMessageCache = new LocalMessageCache(this.rocketmqPullConsumer, clientConfig);
}
 

public AccessPointContainer(String id, String url, List<KeyValueAttribute> attributes) {
    KeyValue keyValue = OMS.newKeyValue();
    for (KeyValueAttribute attribute : attributes) {
        keyValue.put(attribute.getKey(), attribute.getValue());
    }
    this.id = id;
    this.url = url;
    this.attributes = keyValue;
}
 

public static void main(String[] args) throws Exception {
    KeyValue keyValue = OMS.newKeyValue();
    keyValue.put(OMSBuiltinKeys.ACCOUNT_KEY, "test_token");

    MessagingAccessPoint messagingAccessPoint = OMS.getMessagingAccessPoint("oms:joyqueue://[email protected]:50088/UNKNOWN", keyValue);

    Consumer consumer = messagingAccessPoint.createConsumer();
    consumer.start();

    // 绑定主题，将要消费的主题
    consumer.bindQueue("test_topic_0");

    // 这里只是简单例子，根据具体情况进行调度处理
    // 参数是超时时间，只是网络请求的超时
    while (true) {
        System.out.println("doReceive");

        // 批量拉取消息
        List<Message> messages = consumer.batchReceive(1000 * 10);

        if (CollectionUtils.isEmpty(messages)) {
            continue;
        }

        for (Message message : messages) {
            // 拉取到消息，打印日志并ack
            System.out.println(String.format("receive, message: %s", messages));
            consumer.ack(message.getMessageReceipt());
        }

        Thread.currentThread().sleep(1000 * 1);
    }
}
 

public static void main(String[] args) {

        final String app = "test_app";
        final String token = "some token";
        final String dataCenter = "default";
        final String brokerAddr = "127.0.0.1:50088";
        final String topic = "test_topic_0";
        // oms:joyqueue://[email protected]:50088/default
        final String url = "oms:joyqueue://" + app +  "@" + brokerAddr + "/" + dataCenter;

        KeyValue keyValue = OMS.newKeyValue();
        keyValue.put(OMSBuiltinKeys.ACCOUNT_KEY, token);

        MessagingAccessPoint messagingAccessPoint = OMS.getMessagingAccessPoint(url, keyValue);

        // 使用MessagingAccessPoint创建producer
        Producer producer = messagingAccessPoint.createProducer();
        producer.start();

        // 使用producer.createMessage方法创建message
        Message message = producer.createMessage(topic, "Message body".getBytes());

        // 设置messageKey，非必填
        // 如果需要相对顺序消息，也可以使用messageKey作为key指定分区
//        message.extensionHeader().get().setMessageKey("test_key");

        // 自定义发送的partition
        // 最好根据元数据自定义分配，不要写死partitions
//        List<QueueMetaData.Partition> partitions = producer.getQueueMetaData("test_topic_0").partitions();
//        QueueMetaData.Partition partition = partitions.get((int) SystemClock.now() % partitions.size());
//        message.extensionHeader().get().setPartition(partition.partitionId());

        // 生产消息，不抛异常就算成功，sendResult里的messageId暂时没有意义
        SendResult sendResult = producer.send(message);

        // 打印生产结果
        System.out.println(String.format("messageId: %s", sendResult.messageId()));
    }
 

public static BytesMessage msgConvert(org.apache.rocketmq.common.message.MessageExt rmqMsg) {
    BytesMessage omsMsg = new BytesMessageImpl();
    omsMsg.setBody(rmqMsg.getBody());

    KeyValue headers = omsMsg.sysHeaders();
    KeyValue properties = omsMsg.userHeaders();

    final Set<Map.Entry<String, String>> entries = rmqMsg.getProperties().entrySet();

    for (final Map.Entry<String, String> entry : entries) {
        if (isOMSHeader(entry.getKey())) {
            headers.put(entry.getKey(), entry.getValue());
        } else {
            properties.put(entry.getKey(), entry.getValue());
        }
    }

    omsMsg.putSysHeaders(BuiltinKeys.MESSAGE_ID, rmqMsg.getMsgId());

    omsMsg.putSysHeaders(BuiltinKeys.DESTINATION, rmqMsg.getTopic());

    omsMsg.putSysHeaders(BuiltinKeys.SEARCH_KEYS, rmqMsg.getKeys());
    omsMsg.putSysHeaders(BuiltinKeys.BORN_HOST, String.valueOf(rmqMsg.getBornHost()));
    omsMsg.putSysHeaders(BuiltinKeys.BORN_TIMESTAMP, rmqMsg.getBornTimestamp());
    omsMsg.putSysHeaders(BuiltinKeys.STORE_HOST, String.valueOf(rmqMsg.getStoreHost()));
    omsMsg.putSysHeaders(BuiltinKeys.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp());
    return omsMsg;
}
 

@Test
public void getMessagingAccessPoint() {
    String testURI = "oms:test-vendor://[email protected]/us-east:default_space";

    KeyValue keyValue = OMS.newKeyValue();
    keyValue.put(OMSBuiltinKeys.DRIVER_IMPL, "io.openmessaging.internal.TestVendor");
    MessagingAccessPoint messagingAccessPoint = OMS.getMessagingAccessPoint(testURI, keyValue);
    assertThat(messagingAccessPoint).isExactlyInstanceOf(TestVendor.class);
}
 

@Override public void sendOneway(Message message, KeyValue properties) {
    throw new UnsupportedOperationException("Unsupported");
}
 

@Override
public PushConsumer createPushConsumer(KeyValue properties) {
    return new PushConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));
}
 

@Override
public KeyValue attributes() {
    return accessPointProperties;
}
 

@Override
public PushConsumer createPushConsumer(KeyValue properties) {
    return new PushConsumerImpl(OMSUtil.buildKeyValue(this.accessPointProperties, properties));
}
 

@Override
public KeyValue sysHeaders() {
    return sysHeaders;
}
 

@Override
public void ack(String messageId, KeyValue properties) {
	throw new UnsupportedOperationException("Unsupported");
}
 

@Override
public KeyValue put(String key, long value) {
    kvs.put(key, Long.toString(value));
    return this;
}
 

@Override
public KeyValue headers() {
    return headers;
}
 

@Override public void ack(String messageId, KeyValue properties) {
    throw new UnsupportedOperationException("Unsupported");
}
 

@Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> rmqMsgList,
            ConsumeConcurrentlyContext contextRMQ) {
            MessageExt rmqMsg = rmqMsgList.get(0);
            BytesMessage omsMsg = OMSUtil.msgConvert(rmqMsg);

            MessageListener listener = PushConsumerImpl.this.subscribeTable.get(rmqMsg.getTopic());

            if (listener == null) {
                throw new OMSRuntimeException("-1",
                    String.format("The topic/queue %s isn't attached to this consumer", rmqMsg.getTopic()));
            }

            final KeyValue contextProperties = OMS.newKeyValue();
//            让一段代码执行完毕后再往下执行
            final CountDownLatch sync = new CountDownLatch(1);

            contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS, ConsumeConcurrentlyStatus.RECONSUME_LATER.name());

            MessageListener.Context context = new MessageListener.Context() {
                @Override
                public KeyValue attributes() {
                    return contextProperties;
                }

                @Override
                public void ack() {
                    sync.countDown();
                    contextProperties.put(NonStandardKeys.MESSAGE_CONSUME_STATUS,
                        ConsumeConcurrentlyStatus.CONSUME_SUCCESS.name());
                }
            };
            long begin = System.currentTimeMillis();
            listener.onReceived(omsMsg, context);
            long costs = System.currentTimeMillis() - begin;
            long timeoutMills = clientConfig.getRmqMessageConsumeTimeout() * 60 * 1000;
            try {
                sync.await(Math.max(0, timeoutMills - costs), TimeUnit.MILLISECONDS);
            } catch (InterruptedException ignore) {
            }

            return ConsumeConcurrentlyStatus.valueOf(contextProperties.getString(NonStandardKeys.MESSAGE_CONSUME_STATUS));
        }
 

@Override
public KeyValue properties() {
    return properties;
}
 

@Override
public KeyValue put(String key, String value) {
    return null;
}