org.supercsv.cellprocessor.ift.CellProcessor Java Examples

/** This method returns array of cellprocessor for parsing csv.
 * @return CellProcessor[] array
 * */
private CellProcessor[] getProcessors(){
    CellProcessor[] processors = new CellProcessor[]{
            new NotNull(new ParseLong()),   //instrument_token
            new NotNull(new ParseLong()),   //exchange_token
            new NotNull(),                  //trading_symbol
            new org.supercsv.cellprocessor.Optional(),                 //company name
            new NotNull(new ParseDouble()), //last_price
            new org.supercsv.cellprocessor.Optional(new ParseDate("yyyy-MM-dd")),                 //expiry
            new org.supercsv.cellprocessor.Optional(),                 //strike
            new NotNull(new ParseDouble()), //tick_size
            new NotNull(new ParseInt()),    //lot_size
            new NotNull(),                  //instrument_type
            new NotNull(),                  //segment
            new NotNull()                   //exchange
    };
    return processors;
}
 

private static CellProcessor[] getProcessors() {
	return new CellProcessor[]{
		new ParseDouble(), // price
		new ParseDouble(), // volRemaining
		new ParseInt(), // typeID
		new ParseInt(), // range
		new ParseLong(), // orderID
		new ParseInt(), // volEntered
		new ParseInt(), // minVolume
		new ParseBool(), // bid
		new ParseDate(), // issueDate
		new ParseInt(), // duration
		new ParseLong(), // stationID
		new ParseLong(), // regionID
		new ParseLong(), // solarSystemID
		new ParseInt(), // jumps
		new Optional()
	};
}
 

private void loadProducts(String file, String[] nameMapping, CellProcessor[] processors) {
		System.out.println("Started loading the products");
       	initalizeBeanReader(file);
           Product prod;
           List<Product> productPuts = new ArrayList<Product>();
           int prodCount = 0;
           try {
			while ((prod = beanReader.read(Product.class, nameMapping,processors)) != null) {
				productPuts.add(prod);
				prodCount++;
				if (productPuts.size() % 10 == 0) {
					productRepository.save(productPuts);
					productPuts.clear();
				}
			}
		} catch (IOException e) {
			errorLog.add(e.toString());
		}
           activityLog.add("Products: Records Read: " + prodCount + RECORDS_ADDED_TO_GEM_FIRE + productRepository.count() );
}
 

private void loadCustomers(String file, String[] nameMapping, CellProcessor[] processors) {
		System.out.println("Started loading the customers");
       	initalizeBeanReader(file); 
           Customer cust;
           int custCount = 0;
           List<Customer> customerPuts = new ArrayList<Customer>();
           try {
			while ((cust = beanReader.read(Customer.class, nameMapping,processors)) != null) {
				customerPuts.add(cust);
				custCount++;
				if (customerPuts.size() % 10 == 0) {
					customerRepository.save(customerPuts);
					customerPuts.clear();
				}
			}
		} catch (IOException e) {
			errorLog.add(e.toString());
		}
           activityLog.add("Customers: Records Read: " + custCount + RECORDS_ADDED_TO_GEM_FIRE + customerRepository.count() );
}
 

/**
 * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. Empty
 * columns are read as null (hence the NotNull() for mandatory columns).
 * 
 * @return the cell processors
 */
private static CellProcessor[] getProcessors() {
	
	final String emailRegex = "[a-z0-9\\._]+@[a-z0-9\\.]+"; // just an example, not very robust!
	StrRegEx.registerMessage(emailRegex, "must be a valid email address");
	
	final CellProcessor[] processors = new CellProcessor[] { new UniqueHashCode(), // customerNo (must be unique)
		new NotNull(), // firstName
		new NotNull(), // lastName
		new ParseDate("dd/MM/yyyy"), // birthDate
		new ParseSqlTime("HH:mm:ss"),
		new NotNull(), // mailingAddress
		new Optional(new ParseBool()), // married
		new Optional(new ParseInt()), // numberOfKids
		new NotNull(), // favouriteQuote
		new StrRegEx(emailRegex), // email
		new LMinMax(0L, LMinMax.MAX_LONG) // loyaltyPoints
	};
	
	return processors;
}
 

/**
 * Sets up the processors used for the examples. There are 10 CSV columns, so 10 processors are defined. All values
 * are converted to Strings before writing (there's no need to convert them), and null values will be written as
 * empty columns (no need to convert them to "").
 * 
 * @return the cell processors
 */
private static CellProcessor[] getProcessors() {
	
	final CellProcessor[] processors = new CellProcessor[] { new UniqueHashCode(), // customerNo (must be unique)
		new NotNull(), // firstName
		new NotNull(), // lastName
		new FmtDate("dd/MM/yyyy"), // birthDate
		new NotNull(), // mailingAddress
		new Optional(new FmtBool("Y", "N")), // married
		new Optional(), // numberOfKids
		new NotNull(), // favouriteQuote
		new NotNull(), // email
		new LMinMax(0L, LMinMax.MAX_LONG) // loyaltyPoints
	};
	
	return processors;
}
 

@Test
public void testValidLocalTime() {
	for (CellProcessor p : processors) {
		assertEquals(LOCAL_TIME,
				p.execute(LOCAL_TIME_STRING, ANONYMOUS_CSVCONTEXT));
	}
}
 

@Test
public void testValidDateTimeString() {
	for (CellProcessor p : processors) {
		assertEquals(LOCAL_DATE_TIME_STRING,
				p.execute(LOCAL_DATE_TIME, ANONYMOUS_CSVCONTEXT));
	}
}
 

@Test
public void testDealWithLeadingTrailingWhitespace() throws IOException {
	List<String> data = Arrays.asList("     John   ", "   Connor     ");
	CellProcessor[] processors = { new Trim(), new Trim() };
	
	char customQuote = '"';
	CsvPreference customPreference = new Builder(customQuote, ',', "").surroundingSpacesNeedQuotes(false).build();
	String result = writeToCsv(data, processors, customPreference);
	
	Assert.assertEquals("John,Connor", result);
}
 

@Test
public void testValidIntervalString() {
	for (CellProcessor p : processors) {
		assertEquals(INTERVAL_STRING,
				p.execute(INTERVAL, ANONYMOUS_CSVCONTEXT));
	}
}
 

@Test
public void testInvalidDateFormat() {
	final CellProcessor p = new FmtDateTime("not valid");
	try {
		p.execute(DATE_TIME, ANONYMOUS_CSVCONTEXT);
		fail("expecting SuperCsvCellProcessorException");
	} catch (SuperCsvCellProcessorException e) {
		assertEquals("Failed to format value as a DateTime", e.getMessage());
	}
}
 

@Test
public void testNullInput() {
	for (CellProcessor p : processors) {
		try {
			p.execute(null, ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals(
					"this processor does not accept null input - "
							+ "if the column is optional then chain an Optional() processor before this one",
					e.getMessage());
		}
	}
}
 

public static Map<String, String> readWithCsvMapReader(Path dicoFile) throws Exception {

        Map<String, String> retMap = new HashMap<>();
        try (ICsvMapReader mapReader = new CsvMapReader(Files.newBufferedReader(dicoFile, StandardCharsets.UTF_8), CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE)) {
            final String[] header = mapReader.getHeader(true);
            LOGGER.debug(" cvsheader length: " + header.length);
            final CellProcessor[] rowProcessors = new CellProcessor[header.length];
            for (int i = 0; i < rowProcessors.length; i++) {
                if (i == 0) {
                    rowProcessors[i] = new NotNull();
                } else {
                    rowProcessors[i] = null;
                }
            }

            Map<String, Object> componentMap;
            while ((componentMap = mapReader.read(header, rowProcessors)) != null) {
                //System.out.println(String.format("lineNo=%s, rowNo=%s, mapping=%s", mapReader.getLineNumber(), mapReader.getRowNumber(), customerMap));
                String eurostagId = (String) componentMap.get(header[1]);
                String cimId = (String) componentMap.get(header[0]);
                if (eurostagId == null) {
                    LOGGER.warn("eurostagId=" + eurostagId + ", cimId=" + cimId);
                } else {
                    if (retMap.containsKey(eurostagId)) {
                        LOGGER.warn("eurostagId=" + eurostagId + " already in the map");
                    }
                    retMap.put(eurostagId, cimId);
                }
            }
        }

        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace("ids map: " + retMap);
        }
        return retMap;

    }
 

/**
 * Processes each element in the source List (using the corresponding processor chain in the processors array) and
 * adds it to the destination List. A <tt>null</tt> CellProcessor in the array indicates that no processing is
 * required and the element should be added as-is.
 * 
 * @param destination
 *            the List to add the processed elements to (which is cleared before it's populated)
 * @param source
 *            the List of source elements to be processed
 * @param processors
 *            the array of CellProcessors used to process each element. The number of elements in this array must
 *            match the size of the source List. A <tt>null</tt> CellProcessor in this array indicates that no
 *            processing is required and the element should be added as-is.
 * @param lineNo
 *            the current line number
 * @param rowNo
 *            the current row number
 * @throws NullPointerException
 *             if destination, source or processors are null
 * @throws SuperCsvConstraintViolationException
 *             if a CellProcessor constraint failed
 * @throws SuperCsvException
 *             if source.size() != processors.length, or CellProcessor execution failed
 */
public static void executeCellProcessors(final List<Object> destination, final List<?> source,
	final CellProcessor[] processors, final int lineNo, final int rowNo) {
	
	if( destination == null ) {
		throw new NullPointerException("destination should not be null");
	} else if( source == null ) {
		throw new NullPointerException("source should not be null");
	} else if( processors == null ) {
		throw new NullPointerException("processors should not be null");
	}
	
	// the context used when cell processors report exceptions
	final CsvContext context = new CsvContext(lineNo, rowNo, 1);
	context.setRowSource(new ArrayList<Object>(source));
	
	if( source.size() != processors.length ) {
		throw new SuperCsvException(String.format(
			"The number of columns to be processed (%d) must match the number of CellProcessors (%d): check that the number"
				+ " of CellProcessors you have defined matches the expected number of columns being read/written",
			source.size(), processors.length), context);
	}
	
	destination.clear();
	
	for( int i = 0; i < source.size(); i++ ) {
		
		context.setColumnNumber(i + 1); // update context (columns start at 1)
		
		if( processors[i] == null ) {
			destination.add(source.get(i)); // no processing required
		} else {
			destination.add(processors[i].execute(source.get(i), context)); // execute the processor chain
		}
	}
}
 

/**
 * Method used to parse the string supplied by the user. The string is converted
 * to a list of cell processors used to validate the CSV data.
 * @param schema Schema to parse
 */
private void parseSchema(String schema) {
    List<CellProcessor> processorsList = new ArrayList<CellProcessor>();

    String remaining = schema;
    while(remaining.length() > 0) {
        remaining = setProcessor(remaining, processorsList);
    }

    this.processors.set(processorsList.toArray(new CellProcessor[processorsList.size()]));
}
 

/**
 * Tests execution with a invalid number format String (should throw an Exception).
 */
@Test(expected = NullPointerException.class)
public void testWithNullNumberFormatString() {
	final double number = 12.34;
	CellProcessor invalidNumberFormatProcessor = new FmtNumber((String) null);
	invalidNumberFormatProcessor.execute(number, ANONYMOUS_CSVCONTEXT);
}
 

@Test
public void testValidDateTime() {
	for (CellProcessor p : processors) {
		assertEquals(DATE_TIME,
				p.execute(DATE_TIME_STRING, ANONYMOUS_CSVCONTEXT));
	}
}
 

@Test
public void testValidInterval() {
	for (CellProcessor p : processors) {
		Interval result = (Interval) p.execute(INTERVAL_STRING,
				ANONYMOUS_CSVCONTEXT);
		assertEquals(START, result.getStart().withZone(DateTimeZone.UTC));
		assertEquals(END, result.getEnd().withZone(DateTimeZone.UTC));
	}
}
 

/**
 * Tests execution with an invalid time (matches format, but data invalid) and non-lenient processors (should throw
 * an exception).
 */
@Test
public void testInvalidTimeWithNonLenient() {
	final String dodgyDate = "22:20:61";
	for( final CellProcessor cp : Arrays.asList(processor, processor3, processorChain, processorChain3) ) {
		try {
			cp.execute(dodgyDate, ANONYMOUS_CSVCONTEXT).toString();
			fail("should have thrown a SuperCsvCellProcessorException");
		}
		catch(SuperCsvCellProcessorException e) {}
	}
}
 

@Test
public void testNullInput() {
	for (CellProcessor p : processors) {
		try {
			p.execute(null, ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals(
					"this processor does not accept null input - "
							+ "if the column is optional then chain an Optional() processor before this one",
					e.getMessage());
		}
	}
}
 

@Test
public void testInvalidDateFormat() {
	final CellProcessor p = new FmtLocalDateTime("not valid");
	try {
		p.execute(LOCAL_DATE_TIME, ANONYMOUS_CSVCONTEXT);
		fail("expecting SuperCsvCellProcessorException");
	} catch (SuperCsvCellProcessorException e) {
		assertEquals("Failed to format value as a LocalDateTime",
				e.getMessage());
	}
}
 

@Test
public void testNonLocalDateTimeInput() {
	for (CellProcessor p : processors) {
		try {
			p.execute(123, ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals(
					"the input value should be of type org.joda.time.LocalDateTime but is java.lang.Integer",
					e.getMessage());
		}
	}
}
 

@Test
public void testNonStringInput() {
	for (CellProcessor p : processors) {
		try {
			p.execute(123, ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals(
					"the input value should be of type java.lang.String but is java.lang.Integer",
					e.getMessage());
		}
	}
}
 

@Test
public void testNonLocalDateTimeInput() {
	for (CellProcessor p : processors) {
		try {
			p.execute(123, ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals(
					"the input value should be of type org.joda.time.LocalTime but is java.lang.Integer",
					e.getMessage());
		}
	}
}
 

/**
 * Get cellprocessor to parse String as Double.
 *
 * @param cellProcessor
 *          next processor in the chain.
 * @return CellProcessor
 */
private static CellProcessor addParseDouble(CellProcessor cellProcessor)
{
  if (cellProcessor == null) {
    return new ParseDouble();
  }
  return new ParseDouble((DoubleCellProcessor)cellProcessor);
}
 

/**
 * Get cellprocessor to parse String as Character.
 *
 * @param cellProcessor
 *          next processor in the chain.
 * @return CellProcessor
 */
private static CellProcessor addParseChar(CellProcessor cellProcessor)
{
  if (cellProcessor == null) {
    return new ParseChar();
  }
  return new ParseChar((DoubleCellProcessor)cellProcessor);
}
 

@Test
public void testEscapedQuoteInQuotedField() throws IOException {
	List<String> data = Arrays.asList("Joh\"n", "Con\"nor");
	CellProcessor[] processors = { new NotNull(), new NotNull() };
	
	CsvPreference customPreference = new Builder('"', ',', "").build();
	String result = writeToCsv(data, processors, customPreference);
	
	Assert.assertEquals("\"Joh\"\"n\",\"Con\"\"nor\"", result);
}
 

@Test
public void testDealWithLeadingTrailingWhitespace() throws IOException {
	String csv = "     John    ,     Connor   ";
	CellProcessor[] processors = { new Trim(), new Trim() };
	
	char customQuote = '"';
	CsvPreference customPreference = new Builder(customQuote, ',', "").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("John", result.get(0));
	Assert.assertEquals("Connor", result.get(1));
}
 

/**
 * {@inheritDoc}
 */
public void write(ResultSet resultSet, CellProcessor[] writeProcessors) throws SQLException, IOException {
	if( resultSet == null ) {
		throw new NullPointerException("ResultSet cannot be null");
	}
	if( writeProcessors == null ) {
		throw new NullPointerException("CellProcessor[] cannot be null");
	}
	
	writeHeaders(resultSet); // increments row and line number
	writeContents(resultSet, writeProcessors); // increments row and line number before writing of each row
}
 

@Test
public void testUnparsableString() {
	for (CellProcessor p : processors) {
		try {
			p.execute("not valid", ANONYMOUS_CSVCONTEXT);
			fail("expecting SuperCsvCellProcessorException");
		} catch (SuperCsvCellProcessorException e) {
			assertEquals("Failed to parse value", e.getMessage());
		}
	}
}