com.alibaba.otter.canal.instance.core.CanalInstance Java Examples

The following examples show how to use com.alibaba.otter.canal.instance.core.CanalInstance. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: EntryCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    EntryMetricsHolder holder = new EntryMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
    holder.latestExecTime = handler.getLatestExecuteTime();
    holder.transactionCounter = handler.getTransactionCounter();
    Preconditions.checkNotNull(holder.latestExecTime);
    Preconditions.checkNotNull(holder.transactionCounter);
    EntryMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale EntryCollector for instance {}.", destination);
    }
}

Example #2

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void start() {
    if (!isStart()) {
        super.start();
        // 如果存在provider,则启动metrics service
        loadCanalMetrics();
        metrics.setServerPort(metricsPort);
        metrics.initialize();
        canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {

            public CanalInstance apply(String destination) {
                return canalInstanceGenerator.generate(destination);
            }
        });

        // lastRollbackPostions = new MapMaker().makeMap();
    }
}

Example #3

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void stop() {
    super.stop();
    for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
        try {
            CanalInstance instance = entry.getValue();
            if (instance.isStart()) {
                try {
                    String destination = entry.getKey();
                    MDC.put("destination", destination);
                    entry.getValue().stop();
                    logger.info("stop CanalInstances[{}] successfully", destination);
                } finally {
                    MDC.remove("destination");
                }
            }
        } catch (Exception e) {
            logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
        }
    }
    metrics.terminate();
}

Example #4

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

public void stop(String destination) {
    CanalInstance canalInstance = canalInstances.remove(destination);
    if (canalInstance != null) {
        if (canalInstance.isStart()) {
            try {
                MDC.put("destination", destination);
                canalInstance.stop();
                if (metrics.isRunning()) {
                    metrics.unregister(canalInstance);
                }
                logger.info("stop CanalInstances[{}] successfully", destination);
            } finally {
                MDC.remove("destination");
            }
        }
    }
}

Example #5

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

/**
 * 客户端订阅，重复订阅时会更新对应的filter信息
 */
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    if (!canalInstance.getMetaManager().isStart()) {
        canalInstance.getMetaManager().start();
    }

    canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

    Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
    if (position == null) {
        position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
        if (position != null) {
            canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
        }
        logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
    } else {
        logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
    }

    // 通知下订阅关系变化
    canalInstance.subscribeChange(clientIdentity);
}

Example #6

Source File: SinkCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    SinkMetricsHolder holder = new SinkMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime();
    Preconditions.checkNotNull(holder.eventsSinkBlockingTime);
    SinkMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remote stale SinkCollector for instance {}.", destination);
    }
}

Example #7

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }

    synchronized (canalInstance) {
        // 清除batch信息
        canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
        // rollback eventStore中的状态信息
        canalInstance.getEventStore().rollback();
        logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
    }
}

Example #8

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

6 votes

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }

    synchronized (canalInstance) {
        // 清除batch信息
        canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
        // rollback eventStore中的状态信息
        canalInstance.getEventStore().rollback();
        logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
    }
}

Example #9

Source File: SpringCanalInstanceGenerator.java From canal with Apache License 2.0

6 votes

public CanalInstance generate(String destination) {
    synchronized (CanalInstanceGenerator.class) {
        try {
            // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
            System.setProperty("canal.instance.destination", destination);
            this.beanFactory = getBeanFactory(springXml);
            String beanName = destination;
            if (!beanFactory.containsBean(beanName)) {
                beanName = defaultName;
            }

            return (CanalInstance) beanFactory.getBean(beanName);
        } catch (Throwable e) {
            logger.error("generator instance failed.", e);
            throw new CanalException(e);
        } finally {
            System.setProperty("canal.instance.destination", "");
        }
    }
}

Example #10

Source File: GatewayInstance.java From DataLink with Apache License 2.0

6 votes

/**
 * 并发量不大，直接用互斥锁就行了
 */
public synchronized void registerEndpointInstance(String destination, CanalInstance canalInstance, KickoutListener kickoutListener, String filter) {
    String oldMDC = MDC.get(Constants.MDC_TASKID);
    try {
        MDC.put(Constants.MDC_TASKID, gwCanal.getName());

        logger.info("register an endpoint instance with name {} begin.", destination);
        this.endpointInstances.put(destination, new RegisterItem(canalInstance, kickoutListener, filter));
        logger.info("register an endpoint instance with name {} end.", destination);
    } finally {
        if (StringUtils.isNotBlank(oldMDC)) {
            MDC.put(Constants.MDC_TASKID, oldMDC);
        } else {
            MDC.remove(Constants.MDC_TASKID);
        }
    }
}

Example #11

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 客户端订阅，重复订阅时会更新对应的filter信息
 */
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    if (!canalInstance.getMetaManager().isStart()) {
        canalInstance.getMetaManager().start();
    }

    canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

    Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
    if (position == null) {
        position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
        if (position != null) {
            canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
        }
        logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
    } else {
        logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
    }

    // 通知下订阅关系变化
    canalInstance.subscribeChange(clientIdentity);
}

Example #12

Source File: EntryCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    EntryMetricsHolder holder = new EntryMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    PrometheusCanalEventDownStreamHandler handler = assembleHandler(entrySink);
    holder.latestExecTime = handler.getLatestExecuteTime();
    holder.transactionCounter = handler.getTransactionCounter();
    Preconditions.checkNotNull(holder.latestExecTime);
    Preconditions.checkNotNull(holder.transactionCounter);
    EntryMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale EntryCollector for instance {}.", destination);
    }
}

Example #13

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

public void stop() {
    super.stop();
    for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
        try {
            CanalInstance instance = entry.getValue();
            if (instance.isStart()) {
                try {
                    String destination = entry.getKey();
                    MDC.put("destination", destination);
                    entry.getValue().stop();
                    logger.info("stop CanalInstances[{}] successfully", destination);
                } finally {
                    MDC.remove("destination");
                }
            }
        } catch (Exception e) {
            logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
        }
    }
    metrics.terminate();
}

Example #14

Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0

6 votes

public void start() {
    if (!isStart()) {
        super.start();
        // 如果存在provider,则启动metrics service
        loadCanalMetrics();
        metrics.setServerPort(metricsPort);
        metrics.initialize();
        canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>() {

            public CanalInstance apply(String destination) {
                return canalInstanceGenerator.generate(destination);
            }
        });

        // lastRollbackPostions = new MapMaker().makeMap();
    }
}

Example #15

Source File: ParserCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    ParserMetricsHolder holder;
    CanalEventParser parser = instance.getEventParser();
    if (parser instanceof AbstractMysqlEventParser) {
        holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
    } else if (parser instanceof GroupEventParser) {
        holder = groupHolder(destination, (GroupEventParser)parser);
    } else {
        throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
    }
    Preconditions.checkNotNull(holder);
    ParserMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale ParserCollector for instance {}.", destination);
    }
}

Example #16

Source File: CanalServerTest.java From canal with Apache License 2.0

6 votes

@Before
public void setUp() {
    CanalServerWithEmbedded embeddedServer = new CanalServerWithEmbedded();
    embeddedServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = buildCanal();
            return new CanalInstanceWithManager(canal, FILTER);
        }
    });

    nettyServer = CanalServerWithNetty.instance();
    nettyServer.setEmbeddedServer(embeddedServer);
    nettyServer.setPort(1088);
    nettyServer.start();
}

Example #17

Source File: CanalAdminController.java From canal with Apache License 2.0

6 votes

@Override
public String getRunningInstances() {
    try {
        Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
        List<String> runningInstances = new ArrayList<String>();
        instances.forEach((destination, instance) -> {
            if (instance.isStart()) {
                runningInstances.add(destination);
            }
        });

        return Joiner.on(",").join(runningInstances);
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
    }
    return "";
}

Example #18

Source File: ParserCollector.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    ParserMetricsHolder holder;
    CanalEventParser parser = instance.getEventParser();
    if (parser instanceof AbstractMysqlEventParser) {
        holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
    } else if (parser instanceof GroupEventParser) {
        holder = groupHolder(destination, (GroupEventParser)parser);
    } else {
        throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
    }
    Preconditions.checkNotNull(holder);
    ParserMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale ParserCollector for instance {}.", destination);
    }
}

Example #19

Source File: SinkCollector.java From canal with Apache License 2.0

6 votes

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    SinkMetricsHolder holder = new SinkMetricsHolder();
    holder.destLabelValues = Collections.singletonList(destination);
    CanalEventSink sink = instance.getEventSink();
    if (!(sink instanceof EntryEventSink)) {
        throw new IllegalArgumentException("CanalEventSink must be EntryEventSink");
    }
    EntryEventSink entrySink = (EntryEventSink) sink;
    holder.eventsSinkBlockingTime = entrySink.getEventsSinkBlockingTime();
    Preconditions.checkNotNull(holder.eventsSinkBlockingTime);
    SinkMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remote stale SinkCollector for instance {}.", destination);
    }
}

Example #20

Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0

5 votes

private void checkSubscribe(ClientIdentity clientIdentity) {
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        throw new CanalServerException(String.format("ClientIdentity:%s should subscribe first",
            clientIdentity.toString()));
    }
}

Example #21

Source File: BaseCanalServerWithEmbededTest.java From canal with Apache License 2.0

5 votes

@Before
public void setUp() {
    server = CanalServerWithEmbedded.instance();
    server.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = buildCanal();
            return new CanalInstanceWithManager(canal, FILTER);
        }
    });
    server.start();
    server.start(DESTINATION);
}

Example #22

Source File: CanalMQStarter.java From canal with Apache License 2.0

5 votes

public synchronized void startDestination(String destination) {
    CanalInstance canalInstance = canalServer.getCanalInstances().get(destination);
    if (canalInstance != null) {
        stopDestination(destination);
        CanalMQRunnable canalMQRunnable = new CanalMQRunnable(destination);
        canalMQWorks.put(canalInstance.getDestination(), canalMQRunnable);
        executorService.execute(canalMQRunnable);
        logger.info("## Start the MQ work of destination:" + destination);
    }
}

Example #23

Source File: CanalInstanceExports.java From canal with Apache License 2.0

5 votes

void register(CanalInstance instance) {
    requiredInstanceRegistry(storeCollector).register(instance);
    requiredInstanceRegistry(entryCollector).register(instance);
    requiredInstanceRegistry(metaCollector).register(instance);
    requiredInstanceRegistry(sinkCollector).register(instance);
    requiredInstanceRegistry(parserCollector).register(instance);
    logger.info("Successfully register metrics for instance {}.", instance.getDestination());
}

Example #24

Source File: PrometheusService.java From canal with Apache License 2.0

5 votes

@Override
public void register(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
        return;
    }
    try {
        instanceExports.register(instance);
    } catch (Throwable t) {
        logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Register metrics for destination {}.", instance.getDestination());
}

Example #25

Source File: PrometheusService.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void unregister(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Try unregister metrics after destination {} is stopped.", instance.getDestination());
    }
    try {
        instanceExports.unregister(instance);
    } catch (Throwable t) {
        logger.warn("Unable to unregister instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Unregister metrics for destination {}.", instance.getDestination());
}

Example #26

Source File: DefaultSpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Example #27

Source File: PrometheusService.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void register(CanalInstance instance) {
    if (instance.isStart()) {
        logger.warn("Cannot register metrics for destination {} that is running.", instance.getDestination());
        return;
    }
    try {
        instanceExports.register(instance);
    } catch (Throwable t) {
        logger.warn("Unable to register instance exports for {}.", instance.getDestination(), t);
    }
    logger.info("Register metrics for destination {}.", instance.getDestination());
}

Example #28

Source File: GroupSpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Example #29

Source File: MemorySpringInstanceTest.java From canal with Apache License 2.0

5 votes

@Test
public void testInstance() {
    CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
    CanalInstance canalInstance = generator.generate("instance");
    Assert.notNull(canalInstance);

    canalInstance.start();
    try {
        Thread.sleep(10 * 1000);
    } catch (InterruptedException e) {
    }
    canalInstance.stop();
}

Example #30

Source File: SpringCanalInstanceGenerator.java From canal-1.1.3 with Apache License 2.0

5 votes

public CanalInstance generate(String destination) {
    String beanName = destination;
    if (!beanFactory.containsBean(beanName)) {
        beanName = defaultName;
    }

    return (CanalInstance) beanFactory.getBean(beanName);
}