org.springframework.batch.item.ItemProcessor Java Examples

The following examples show how to use org.springframework.batch.item.ItemProcessor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobConfiguration.java    From CogStack-Pipeline with Apache License 2.0 7 votes vote down vote up
@Bean
@Qualifier("compositeItemProcessorr")
public ItemProcessor<Document,Document> compositeItemProcessor() {
    CompositeItemProcessor processor = new CompositeItemProcessor<>();
    ArrayList<ItemProcessor<Document,Document>> delegates = new ArrayList<>();

    if(tikaItemProcessor !=null) delegates.add(tikaItemProcessor);
    if(pdfBoxItemProcessor !=null) delegates.add(pdfBoxItemProcessor);
    if(metadataItemProcessor !=null) delegates.add(metadataItemProcessor);
    if(dBLineFixerItemProcessor !=null) delegates.add(dBLineFixerItemProcessor);
    if(gateItemProcessor !=null) delegates.add(gateItemProcessor);
    if(deIdDocumentItemProcessor !=null) delegates.add(deIdDocumentItemProcessor);
    if(webserviceDocumentItemProcessor !=null) delegates.add(webserviceDocumentItemProcessor);
    if(pdfGenerationItemProcessor !=null) delegates.add(pdfGenerationItemProcessor);
    if(thumbnailGenerationItemProcessor !=null) delegates.add(thumbnailGenerationItemProcessor);


    delegates.add(jsonMakerItemProcessor);
    processor.setDelegates(delegates);
    return processor;
}
 
Example #2
Source File: StepService0002.java    From seed with Apache License 2.0 6 votes vote down vote up
private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            String validateResult = ValidatorUtil.validate(item);
            if(StringUtils.isNotEmpty(validateResult)){
                throw new SeedException("数据校验未通过-->" + validateResult);
            }
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            item.setRealName(item.getRealName() + "--00");
            item.setAge(item.getAge() + 10);
            item.setId(null);
            item.setUpdateTime(null);
            item.setCreateTime(new Date());
            LogUtil.getLogger().info("转换后==>{}", ReflectionToStringBuilder.toString(item));
            return item;
        }
    };
}
 
Example #3
Source File: BatchConfiguration.java    From Software-Architecture-with-Spring-5.0 with MIT License 6 votes vote down vote up
@Bean(name = STEP_PROCESS_CSV_FILE)
public Step readCsvFileAndPopulateDbTable(
        StepBuilderFactory stepBuilderFactory,
        PlatformTransactionManager platformTransactionManager,
        @Qualifier(DATA_READER) ItemReader<JavaChampion> itemReader,
        @Qualifier(DATA_PROCESSOR) ItemProcessor<JavaChampion, JavaChampion> itemProcessor,
        @Qualifier(DATA_WRITER) ItemWriter<JavaChampion> itemWriter) {

    StepBuilder builder = stepBuilderFactory.get(STEP_PROCESS_CSV_FILE);

    return builder.<JavaChampion, JavaChampion>chunk(10)
            .reader(itemReader)
            .processor(itemProcessor)
            .writer(itemWriter)
            .transactionManager(platformTransactionManager)
            .build();
}
 
Example #4
Source File: BudgetVtollConfig.java    From SpringBootBucket with MIT License 6 votes vote down vote up
/**
     * step步骤,包含ItemReader,ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "vtollStep1")
    public Step vtollStep1(StepBuilderFactory stepBuilderFactory,
                           @Qualifier("vtollReader") ItemReader<BudgetVtoll> reader,
                           @Qualifier("vtollWriter") ItemWriter<BudgetVtoll> writer,
                           @Qualifier("vtollProcessor") ItemProcessor<BudgetVtoll, BudgetVtoll> processor) {
        return stepBuilderFactory
                .get("vtollStep1")
                .<BudgetVtoll, BudgetVtoll>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行,一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 
Example #5
Source File: CantonConfig.java    From SpringBootBucket with MIT License 6 votes vote down vote up
/**
     * step步骤,包含ItemReader,ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "cantonStep1")
    public Step cantonStep1(StepBuilderFactory stepBuilderFactory,
                           @Qualifier("cantonReader") ItemReader<Canton> reader,
                           @Qualifier("cantonWriter") ItemWriter<Canton> writer,
                           @Qualifier("cantonProcessor") ItemProcessor<Canton, Canton> processor) {
        return stepBuilderFactory
                .get("cantonStep1")
                .<Canton, Canton>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行,一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 
Example #6
Source File: LogConfig.java    From SpringBootBucket with MIT License 6 votes vote down vote up
/**
     * step步骤,包含ItemReader,ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "logStep1")
    public Step logStep1(StepBuilderFactory stepBuilderFactory,
                         @Qualifier("logReader") ItemReader<Log> reader,
                         @Qualifier("logWriter") ItemWriter<Log> writer,
                         @Qualifier("logProcessor") ItemProcessor<Log, Log> processor) {
        return stepBuilderFactory
                .get("logStep1")
                .<Log, Log>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行,一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 
Example #7
Source File: AppConfig.java    From SpringBootBucket with MIT License 6 votes vote down vote up
/**
     * step步骤,包含ItemReader,ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "zappStep1")
    public Step zappStep1(StepBuilderFactory stepBuilderFactory,
                          @Qualifier("appReader") ItemReader<App> reader,
                          @Qualifier("appWriter") ItemWriter<App> writer,
                          @Qualifier("appProcessor") ItemProcessor<App, App> processor) {
        return stepBuilderFactory
                .get("zappStep1")
                .<App, App>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行,一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 
Example #8
Source File: StepService0003.java    From seed with Apache License 2.0 6 votes vote down vote up
private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            String validateResult = ValidatorUtil.validate(item);
            if(StringUtils.isNotEmpty(validateResult)){
                throw new SeedException("数据校验未通过-->" + validateResult);
            }
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            item.setRealName(item.getRealName() + "--11");
            item.setAge(item.getAge() - 5);
            LogUtil.getLogger().info("转换后==>{}", ReflectionToStringBuilder.toString(item));
            return item;
        }
    };
}
 
Example #9
Source File: StepService0004.java    From seed with Apache License 2.0 6 votes vote down vote up
private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            if(item.getRealName().contains("玄玉")){
                throw new SeedException("这是测试断点续跑的异常");
            }
            try {
                TimeUnit.SECONDS.sleep(4);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            return item;
        }
    };
}
 
Example #10
Source File: AddressPrinterJobConfiguration.java    From spring-batch-lightmin with Apache License 2.0 6 votes vote down vote up
@Bean
public ItemProcessor<Long, Address> addressPrinterProcessor() {
    return new ItemProcessor<Long, Address>() {

        @Autowired
        private BatchTaskAddressDao batchTaskAddressDAO;

        @Autowired
        private AddressDao addressDAO;

        @Override
        public Address process(final Long id) throws Exception {
            this.batchTaskAddressDAO.updateProcessingState(id, ProcessingState.PRINTED);
            return this.addressDAO.getAddressById(id);
        }
    };
}
 
Example #11
Source File: JobSkipConfiguration.java    From spring-cloud-task with Apache License 2.0 5 votes vote down vote up
@Bean
public Step step2() {
	return this.stepBuilderFactory.get("step2").chunk(3).faultTolerant()
			.skip(IllegalStateException.class).skipLimit(100)
			.reader(new SkipItemReader())
			.processor(new ItemProcessor<Object, Object>() {
				@Override
				public String process(Object item) throws Exception {
					return String.valueOf(Integer.parseInt((String) item) * -1);
				}
			}).writer(new SkipItemWriter()).build();
}
 
Example #12
Source File: StepService0006.java    From seed with Apache License 2.0 5 votes vote down vote up
private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            try {
                TimeUnit.SECONDS.sleep(6);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            return item;
        }
    };
}
 
Example #13
Source File: JobConfiguration.java    From CogStack-Pipeline with Apache License 2.0 5 votes vote down vote up
@Bean
@Qualifier("compositeSlaveStep")
public Step compositeSlaveStep(
                    ItemReader<Document> reader,
        @Qualifier("compositeItemProcessor") ItemProcessor<Document, Document> processor,
        @Qualifier("compositeESandJdbcItemWriter") ItemWriter<Document> writer,
        @Qualifier("slaveTaskExecutor")TaskExecutor taskExecutor,
        @Qualifier("nonFatalExceptionItemProcessorListener")
                            ItemProcessListener nonFatalExceptionItemProcessorListener,
        //@Qualifier("targetDatasourceTransactionManager")PlatformTransactionManager manager,
        StepBuilderFactory stepBuilderFactory
) {
    FaultTolerantStepBuilder stepBuilder = stepBuilderFactory.get("compositeSlaveStep")
            .<Document, Document> chunk(chunkSize)
            .reader(reader)
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipLimit(skipLimit)
            .skip(WebserviceProcessingFailedException.class);
    if (env.acceptsProfiles("jdbc_out_map")) {
      stepBuilder = stepBuilder.skip(InvalidDataAccessApiUsageException.class);
    }
    return stepBuilder.noSkip(Exception.class)
     //       .listener(nonFatalExceptionItemProcessorListener)
            .listener(new SkipListener())
            .taskExecutor(taskExecutor)
            .build();
}
 
Example #14
Source File: BatchConfiguration.java    From messaging with Apache License 2.0 5 votes vote down vote up
@Bean
Job job(JobBuilderFactory jobBuilderFactory,
 StepBuilderFactory stepBuilderFactory, JdbcTemplate template,
 ItemReader<Contact> fileReader,
 ItemProcessor<Contact, Contact> emailProcessor,
 ItemWriter<Contact> jdbcWriter) {

 Step setup = stepBuilderFactory.get("clean-contact-table")
  .tasklet((contribution, chunkContext) -> {
   template.update("delete from CONTACT");
   return RepeatStatus.FINISHED;
  }).build();

 Step fileToJdbc = stepBuilderFactory.get("file-to-jdbc-fileToJdbc")
  .<Contact, Contact>chunk(5)
  // <1>
  .reader(fileReader).processor(emailProcessor).writer(jdbcWriter)
  .faultTolerant().skip(InvalidEmailException.class)
  // <2>
  .skipPolicy((Throwable t, int skipCount) -> {
   LogFactory.getLog(getClass()).info("skipping ");
   return t.getClass().isAssignableFrom(InvalidEmailException.class);
  }).retry(HttpStatusCodeException.class) // <3>
  .retryLimit(2).build();

 return jobBuilderFactory.get("etl") // <4>
  .start(setup).next(fileToJdbc).build();
}
 
Example #15
Source File: BatchConfiguration.java    From messaging with Apache License 2.0 5 votes vote down vote up
@Bean
ItemProcessor<Contact, Contact> validatingProcessor(
 EmailValidationService emailValidationService) {
 return item -> {
  boolean valid = emailValidationService.isEmailValid(item.getEmail());
  item.setValidEmail(valid);
  if (!valid)
   throw new InvalidEmailException(item.getEmail());
  return item;
 };
}
 
Example #16
Source File: AddressMigrationJobConfiguration.java    From spring-batch-lightmin with Apache License 2.0 5 votes vote down vote up
@Bean
public Step addressMigrationStep(final StepBuilderFactory stepBuilderFactory,
                                 final ItemStreamReader<BatchTaskAddress> addressReader,
                                 final ItemProcessor<BatchTaskAddress, BatchTaskAddress> addressMigrationProcessor,
                                 final ItemWriter<BatchTaskAddress> addressWriter) throws Exception {
    return stepBuilderFactory.get("addressMigrationStep")
            .<BatchTaskAddress, BatchTaskAddress>chunk(1)
            .reader(addressReader)
            .processor(addressMigrationProcessor)
            .writer(addressWriter)
            .build();
}
 
Example #17
Source File: AddressPrinterJobConfiguration.java    From spring-batch-lightmin with Apache License 2.0 5 votes vote down vote up
@Bean
public Step addressPrinterStep(final StepBuilderFactory stepBuilderFactory,
                               final ItemStreamReader<Long> addressPrinterReader,
                               final ItemProcessor<Long, Address> addressPrinterProcessor,
                               final ItemWriter<Address> addressPrinterWriter) throws Exception {
    return stepBuilderFactory
            .get("addressPrinterStep")
            .<Long, Address>chunk(1)
            .reader(addressPrinterReader)
            .processor(addressPrinterProcessor)
            .writer(addressPrinterWriter)
            .allowStartIfComplete(Boolean.TRUE)
            .build();
}
 
Example #18
Source File: SpringBatchConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
public Step skipPolicyStep(@Qualifier("skippingItemProcessor") ItemProcessor<Transaction, Transaction> processor,
                           ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("skipPolicyStep")
            .<Transaction, Transaction>chunk(10)
            .reader(itemReader(invalidInputCsv))
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipPolicy(new CustomSkipPolicy())
            .build();
}
 
Example #19
Source File: ChunksConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
protected Step processLines(ItemReader<Line> reader, ItemProcessor<Line, Line> processor, ItemWriter<Line> writer) {
    return steps.get("processLines").<Line, Line> chunk(2)
      .reader(reader)
      .processor(processor)
      .writer(writer)
      .build();
}
 
Example #20
Source File: SpringBatchRetryConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
public Step retryStep(@Qualifier("retryItemProcessor") ItemProcessor<Transaction, Transaction> processor,
  ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory.get("retryStep")
      .<Transaction, Transaction>chunk(10)
      .reader(itemReader(inputCsv))
      .processor(processor)
      .writer(writer)
      .faultTolerant()
      .retryLimit(3)
      .retry(ConnectTimeoutException.class)
      .retry(DeadlockLoserDataAccessException.class)
      .build();
}
 
Example #21
Source File: SpringBatchConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
protected Step step1(@Qualifier("itemProcessor") ItemProcessor<Transaction, Transaction> processor, ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("step1")
            .<Transaction, Transaction> chunk(10)
            .reader(itemReader(inputCsv))
            .processor(processor)
            .writer(writer)
            .build();
}
 
Example #22
Source File: SpringBatchConfig.java    From tutorials with MIT License 5 votes vote down vote up
@Bean
public Step skippingStep(@Qualifier("skippingItemProcessor") ItemProcessor<Transaction, Transaction> processor,
                         ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("skippingStep")
            .<Transaction, Transaction>chunk(10)
            .reader(itemReader(invalidInputCsv))
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipLimit(2)
            .skip(MissingUsernameException.class)
            .skip(NegativeAmountException.class)
            .build();
}
 
Example #23
Source File: CsvBatchConfig.java    From Demo with Apache License 2.0 5 votes vote down vote up
@Bean
public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<Student> reader, ItemWriter<Student> writer,
        ItemProcessor<Student,Student> processor) {
    return stepBuilderFactory
            .get("step1")
            .<Student, Student>chunk(65000) //批处理每次提交 65000 条数据
            .reader(reader) // 给 step 绑定 reader
            .processor(processor) // 给 step 绑定 processor
            .writer(writer) // 给 step 绑定 writer
            .build();
}
 
Example #24
Source File: PersonJobConfig.java    From spring-batch-rest with Apache License 2.0 5 votes vote down vote up
@Bean @StepScope
ItemProcessor personProcessor(
        @Qualifier("personNameCaseChange") ItemProcessor personNameCaseChange,
        @Value("#{jobParameters['" + LAST_NAME_PREFIX + "']}") String lastNamePrefix) {
    CompositeItemProcessor p = new CompositeItemProcessor();
    p.setDelegates(newArrayList(
            personNameFilter(Optional.ofNullable(lastNamePrefix).orElseGet(() -> environment.getProperty(LAST_NAME_PREFIX))),
            personNameCaseChange));
    return p;
}
 
Example #25
Source File: BatchConfiguration.java    From spring-graalvm-native with Apache License 2.0 5 votes vote down vote up
@Bean
public Step step1(ItemReader<CustomerCredit> itemReader,
		ItemProcessor<CustomerCredit, CustomerCredit> itemProcessor,
		ItemWriter<CustomerCredit> itemWriter) {
	return this.stepBuilderFactory.get("step1")
			.<CustomerCredit, CustomerCredit>chunk(2)
			.reader(itemReader)
			.processor(itemProcessor)
			.writer(itemWriter)
			.build();
}
 
Example #26
Source File: BatchConfiguration.java    From Software-Architecture-with-Spring-5.0 with MIT License 5 votes vote down vote up
@Bean(name = DATA_PROCESSOR)
public ItemProcessor<JavaChampion, JavaChampion> processor() {
    return objectFromRow -> {
        log.info("Java Champion found: " + objectFromRow);
        return objectFromRow;
    };
}
 
Example #27
Source File: PersonJobConfig.java    From spring-batch-rest with Apache License 2.0 5 votes vote down vote up
@Bean
Step personStep(@Qualifier("personProcessor") ItemProcessor personProcessor) {
    return steps.get("personStep")
            .allowStartIfComplete(true)
            .<Person, Person>chunk(3)
            .reader(personReader())
            .processor(personProcessor)
            .writer(personWriter())
            .build();
}
 
Example #28
Source File: PersonJobConfig.java    From spring-batch-rest with Apache License 2.0 5 votes vote down vote up
@Bean @StepScope
ItemProcessor personProcessor(
        @Qualifier("personNameCaseChange") ItemProcessor personNameCaseChange,
        @Value("#{jobParameters['" + LAST_NAME_PREFIX + "']}") String lastNamePrefix) {
    CompositeItemProcessor p = new CompositeItemProcessor();
    p.setDelegates(newArrayList(
            personNameFilter(Optional.ofNullable(lastNamePrefix).orElseGet(() -> environment.getProperty(LAST_NAME_PREFIX))),
            personNameCaseChange));
    return p;
}
 
Example #29
Source File: PersonJobConfig.java    From spring-batch-rest with Apache License 2.0 5 votes vote down vote up
@Bean @StepScope
ItemProcessor personNameCaseChange(@Value("#{jobParameters['upperCase']}") Boolean upperCaseParam) {
    boolean upperCase = upperCaseParam == null ? false : upperCaseParam;
    log.info("personNameCaseChange(upperCase={})", upperCase);
    return new FunctionItemProcessor<Person, Person>(p -> new Person(
            upperCase ? p.firstName.toUpperCase() : p.firstName.toLowerCase(),
            upperCase ? p.lastName.toUpperCase() : p.lastName.toLowerCase()));
}
 
Example #30
Source File: PersonJobConfig.java    From spring-batch-rest with Apache License 2.0 5 votes vote down vote up
@Bean
Step personStep(@Qualifier("personProcessor") ItemProcessor personProcessor) {
    return steps.get("personStep")
            .allowStartIfComplete(true)
            .<Person, Person>chunk(3)
            .reader(personReader())
            .processor(personProcessor)
            .writer(personWriter())
            .build();
}