Jave read csv

/**
 * Reads a {@link CategoryDataset} from a CSV file or input source.
 *
 * @param in  the input source.
 *
 * @return A category dataset.
 *
 * @throws IOException if there is an I/O problem.
 */
public CategoryDataset readCategoryDataset(Reader in) throws IOException {

    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    BufferedReader reader = new BufferedReader(in);
    List columnKeys = null;
    int lineIndex = 0;
    String line = reader.readLine();
    while (line != null) {
        if (lineIndex == 0) {  // first line contains column keys
            columnKeys = extractColumnKeys(line);
        }
        else {  // remaining lines contain a row key and data values
            extractRowKeyAndData(line, dataset, columnKeys);
        }
        line = reader.readLine();
        lineIndex++;
    }
    return dataset;

}
 

@Bean(name = STEP_PROCESS_CSV_FILE)
public Step readCsvFileAndPopulateDbTable(
        StepBuilderFactory stepBuilderFactory,
        PlatformTransactionManager platformTransactionManager,
        @Qualifier(DATA_READER) ItemReader<JavaChampion> itemReader,
        @Qualifier(DATA_PROCESSOR) ItemProcessor<JavaChampion, JavaChampion> itemProcessor,
        @Qualifier(DATA_WRITER) ItemWriter<JavaChampion> itemWriter) {

    StepBuilder builder = stepBuilderFactory.get(STEP_PROCESS_CSV_FILE);

    return builder.<JavaChampion, JavaChampion>chunk(10)
            .reader(itemReader)
            .processor(itemProcessor)
            .writer(itemWriter)
            .transactionManager(platformTransactionManager)
            .build();
}
 

/**
 * Tests {@link CsvConverters.ReadCsv} throws an exception if no header information is provided.
 */
@Test(expected = NullPointerException.class)
public void testReadCsvWithoutHeaderInformation() {
  pipeline.apply(
      "TestReadCsvWithoutHeaderInformation",
      CsvConverters.ReadCsv.newBuilder()
          .setHasHeaders(null)
          .setInputFileSpec(NO_HEADER_CSV_FILE_PATH)
          .setHeaderTag(CSV_HEADERS)
          .setLineTag(CSV_LINES)
          .setCsvFormat("Default")
          .setDelimiter(",")
          .build());

  pipeline.run();
}
 

private int readEscape(int c) throws IOException {
    // assume c is the escape char (normally a backslash)
    c = in.read();
    int out;
    switch (c) {
        case 'r':
            out = '\r';
            break;
        case 'n':
            out = '\n';
            break;
        case 't':
            out = '\t';
            break;
        case 'b':
            out = '\b';
            break;
        case 'f':
            out = '\f';
            break;
        default:
            out = c;
    }
    return out;
}
 

public static <T> List<T> read(Class<T> clazz, InputStream stream, boolean withHeaders, char separator) throws IOException {
    CsvMapper mapper = new CsvMapper();

    mapper.enable(CsvParser.Feature.TRIM_SPACES);
    mapper.enable(CsvParser.Feature.ALLOW_TRAILING_COMMA);
    mapper.enable(CsvParser.Feature.INSERT_NULLS_FOR_MISSING_COLUMNS);
    mapper.enable(CsvParser.Feature.SKIP_EMPTY_LINES);
    mapper.disable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);
    CsvSchema schema = mapper.schemaFor(clazz).withColumnReordering(true);
    ObjectReader reader;
    if (separator == '\t') {
        schema = schema.withColumnSeparator('\t');
    }
    else {
        schema = schema.withColumnSeparator(',');
    }
    if (withHeaders) {
        schema = schema.withHeader();
    }
    else {
        schema = schema.withoutHeader();
    }
    reader = mapper.readerFor(clazz).with(schema);
    return reader.<T>readValues(stream).readAll();
}
 

public static List<Map<String, String>> readCSVasListMap(InputStream is){

		String[][] strings = readCSVasStrings(is);

		String[] header = strings[0];
		String[][] data = Arrays.copyOfRange(strings, 1, strings.length);

		List<Map<String, String>> list = new ArrayList<Map<String, String>>();

		for(String[] datum: data){
			Map<String, String> map = new HashMap<String, String>();
			for(int i=0; i<header.length; i++){
				String key = header[i];
				String value = datum[i];
				map.put(key, value);
			}
			list.add(map);
		}
		return list;
	}
 

public static List<String> readCSVFailedTC(String csvFile) {
	String line = "";
	String cvsSplitBy = ",";
	List<String> csv = new ArrayList<String>();
	String[] tempLine;

	try (BufferedReader br = new BufferedReader(new FileReader(csvFile))) {
		while ((line = br.readLine()) != null) {
			tempLine = line.split(cvsSplitBy);
			if (tempLine[5].trim().equalsIgnoreCase("fail")) {
				csv.add(tempLine[0]);
			}
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
	return csv;
}
 

/**
 * Reads the next line from the buffer and converts to a string array.
 *
 * @return a string array with each comma-separated element as a separate entry.
 *
 * @throws IOException if bad things happen during the read
 */
public String[] readNext() throws IOException {
  String[] result = null;
  do {
    String nextLine = getNextLine();
    if (nextLine == null) {
      return null;
    }
    String[] r = parser.parseLineMulti(nextLine);
    if (r.length > 0) {
      if (result == null) {
        result = r;
      } else {
        String[] t = new String[result.length + r.length];
        System.arraycopy(result, 0, t, 0, result.length);
        System.arraycopy(r, 0, t, result.length, r.length);
        result = t;
      }
    }
  } while (parser.isPending());
  return result;
}
 

private int readChar() throws IOException {
	if (mark) {
		int c = is.read();
		if (c == EOF) 
			inputEOF = true;
		else
			inputBuffer.add(c);
		return c;
	} else {
		if (inputBuffer.isEmpty()) {
			if (inputEOF)
				return EOF;
			int c = is.read();
			if (c == EOF)
				inputEOF = true;
			return c;
		} else {
			return inputBuffer.pop();
		}
	}
}
 

private List<String[]> readCsv(byte[] fileData) {
  List<String[]> values = new ArrayList<>();
  try {
    csvReader = getCsvReader(fileData, ',', '"', 0);
    ProjectLogger.log(
        "UserBulkMigrationActor:readCsv:csvReader initialized ".concat(csvReader.toString()),
        LoggerEnum.INFO.name());
    values = csvReader.readAll();
  } catch (Exception ex) {
    ProjectLogger.log(
        "UserBulkMigrationActor:readCsv:error occurred while getting csvReader",
        LoggerEnum.ERROR.name());
    throw new ProjectCommonException(
        ResponseCode.SERVER_ERROR.getErrorCode(),
        ResponseCode.SERVER_ERROR.getErrorMessage(),
        ResponseCode.SERVER_ERROR.getResponseCode());
  } finally {
    IOUtils.closeQuietly(csvReader);
  }
  return values;
}
 

@Test
public void readFileWithFormat() throws Exception {
    String file = writeCSV(getCSV());

    WorkflowJob p = j.jenkins.createProject(WorkflowJob.class, "p");
    p.setDefinition(new CpsFlowDefinition(
            "node {\n" +
            "  def format = CSVFormat.EXCEL.withHeader('key', 'value', 'attr').withSkipHeaderRecord(true)\n" +
            "  List<CSVRecord> records = readCSV file: '" + file + "', format: format\n" +
            "  assert records.size() == 2\n" +
            "  assert records[0].get('key') == 'a'\n" +
            "  assert records[0].get('value') == 'b'\n" +
            "  assert records[0].get('attr') == 'c'\n" +
            "  assert records[1].get('key') == '1'\n" +
            "  assert records[1].get('value') == '2'\n" +
            "  assert records[1].get('attr') == '3'\n" +
            "}", true));
    j.assertBuildStatusSuccess(p.scheduleBuild2(0));
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

/**
 * Reads the given file and for any given line checks if the tag is equal to the one in
 * countryList.
 */
private static void readCSV(String filename, TreeMap<String, Country> countryTreeMap)
		throws IOException {
	/* The same reader as in SaveGameReader */
	InputStreamReader reader = new InputStreamReader(new FileInputStream(filename),
			"ISO8859_1"); // This encoding seems to work for ö
	// and ü
	BufferedReader scanner = new BufferedReader(reader);

	String line;
	while ((line = scanner.readLine()) != null) {
		String[] dataArray = line.split(";"); // Splitting the line

		if (countryTreeMap.containsKey(dataArray[0])) {
			countryTreeMap.get(dataArray[0]).setOfficialName(dataArray[1]);
		}


	}
	// Close the file once all data has been read.
	scanner.close();

}
 

private static CSVParser readCsvRecords(List<String> header, String content) {
    try {
        CSVFormat csvFormat = CSVFormat.RFC4180;
        if (header.isEmpty()) {
            csvFormat = csvFormat.withFirstRecordAsHeader();
        }

        return csvFormat.
                withIgnoreSurroundingSpaces().
                withIgnoreEmptyLines().
                withTrim().
                withDelimiter(',').
                parse(new StringReader(content));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

private static Collection<Collection<Double>> readRecords(BufferedReader reader, int columnCount) throws IOException {
    List<Collection<Double>> columns = Stream.generate(() -> new ArrayList<Double>()).limit(columnCount).collect(toList());

    while (true) {
        String line = reader.readLine();

        if (line == null) {
            break;
        }

        Collection<Optional<Double>> record = stream(line.split(FIELD_SEPARATOR)).map(s -> parseField(s)).collect(toList());

        int index = 0;
        for (Optional<Double> field : record) {
            field.ifPresent(columns.get(index)::add);
            index++;
        }
    }

    return columns;
}
 

private static Map readSettings() throws Exception {
    final Map result = new HashMap();

    try (FileReader fr = new FileReader("settings.properties"); BufferedReader br = new BufferedReader(fr)) {

        String line = null;
        while ((line = br.readLine()) != null) {
            if (line.trim().length() == 0 || line.startsWith("#") || line.indexOf("=") == -1) {
                continue;
            }

            result.put(line.substring(0, line.indexOf("=")), line.substring(line.indexOf("=") + 1));
        }
    }

    return result;
}
 

public String readLine() throws Exception {

		StringBuffer readLine = new StringBuffer();
		boolean bReadNext = true;

		while (bReadNext) {
			if (readLine.length() > 0) {
				readLine.append("\r\n");
			}
			String strReadLine = br.readLine();

			if (strReadLine == null) {
				return null;
			}
			readLine.append(strReadLine);

			if (countChar(readLine.toString(), '"', 0) % 2 == 1) {
				bReadNext = true;
			} else {
				bReadNext = false;
			}
		}
		return readLine.toString();
	}
 

/**
 * Reads a {@link CategoryDataset} from a CSV file or input source.
 *
 * @param in  the input source.
 *
 * @return A category dataset.
 *
 * @throws IOException if there is an I/O problem.
 */
public CategoryDataset readCategoryDataset(Reader in) throws IOException {

    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    BufferedReader reader = new BufferedReader(in);
    List columnKeys = null;
    int lineIndex = 0;
    String line = reader.readLine();
    while (line != null) {
        if (lineIndex == 0) {  // first line contains column keys
            columnKeys = extractColumnKeys(line);
        }
        else {  // remaining lines contain a row key and data values
            extractRowKeyAndData(line, dataset, columnKeys);
        }
        line = reader.readLine();
        lineIndex++;
    }
    return dataset;

}
 

@Test
public void givenCSVFile_whenBufferedReader_thenContentsAsExpected() throws IOException {
    List<List<String>> records = new ArrayList<List<String>>();
    try (BufferedReader br = new BufferedReader(new FileReader(CSV_FILE))) {
        String line = "";
        while ((line = br.readLine()) != null) {
            String[] values = line.split(COMMA_DELIMITER);
            records.add(Arrays.asList(values));
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    for (int i = 0; i < EXPECTED_ARRAY.size(); i++) {
        Assert.assertArrayEquals(EXPECTED_ARRAY.get(i)
            .toArray(),
            records.get(i)
                .toArray());
    }
}
 

/**
 * Reads a {@link CategoryDataset} from a CSV file or input source.
 * 
 * @param in  the input source.
 * 
 * @return A category dataset.
 * 
 * @throws IOException if there is an I/O problem.
 */
public CategoryDataset readCategoryDataset(Reader in) throws IOException {
    
    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    BufferedReader reader = new BufferedReader(in);
    List columnKeys = null;
    int lineIndex = 0;
    String line = reader.readLine();
    while (line != null) {
        if (lineIndex == 0) {  // first line contains column keys
            columnKeys = extractColumnKeys(line);
        }
        else {  // remaining lines contain a row key and data values
            extractRowKeyAndData(line, dataset, columnKeys);
        }
        line = reader.readLine();
        lineIndex++;
    }
    return dataset;     
     
}
 

private void readHeaders() {
    AtomicInteger fieldId = new AtomicInteger(0);
    reset();

    for (long i = 0; i < fileSize; i++) {
        byte current = get();
        if (current == TOKEN_LF) {
            Arrays.stream(new String(buffer.array()).split(","))
                    .map(header -> header.replaceAll("\"", ""))
                    .map(header -> (header.isEmpty()) ? "header_" + fieldId.incrementAndGet() : header)
                    .map(String::trim).forEach(header -> {
                headers.put(header, "<empty>");
            });
            break;
        } else {
            buffer.put(current);
        }
    }
    ((Buffer) buffer).clear();
}
 

@Test
public void readFromCsvWithCustomTimestampTest() throws FileNotFoundException {
    ClassLoader loader = Thread.currentThread().getContextClassLoader();
    URL uri = loader.getResource("row_id-column_id-custom_timestamp.csv");
    if (uri == null) {
        fail("row_id-column_id-custom_timestamp.csv not found");
    }
    SimpleDateFormat timestampFormat = CsvFileColumnIterator.getDefaultTimestampFormat();
    CsvFileColumnIterator iterator = CsvFileColumnIterator.fromPath(uri.getPath(), timestampFormat);
    List<Column> columns = new ArrayList<>(3);
    while (iterator.hasNext()) {
        columns.add(iterator.next());
    }
    List<Column> target = getTargetRows();
    assertEquals(3, columns.size());
    assertEquals(target, columns);
}
 

private void readEntity(int index, String[] currentLine) {
    String entityId;
    if (idIndex < 0) {
        entityId = "id" + index;
    } else {
        entityId = currentLine[idIndex];
    }

    final EntityProfile newProfile = new EntityProfile(entityId);
    for (int i = 0; i < attributeNames.length; i++) {
        if (attributesToExclude.contains(i)) {
            continue;
        }
        if (!currentLine[i].trim().isEmpty()) {
            newProfile.addAttribute(attributeNames[i], currentLine[i]);
        }
    }
    entityProfiles.add(newProfile);
}
 

public static List<Map<String, String>> readFile(InputStream inputStream)
{
    List<Map<String, String>> data = new LinkedList<>();
    CSVParser csvParser;
    try
    {
        csvParser = CSVParser.parse(inputStream, CHARSET_UTF8, CSV_FORMAT);
        for (CSVRecord record : csvParser.getRecords())
        {
            data.add(record.toMap());
        }

    }
    catch (IOException e)
    {
        throw new RuntimeException(e);
    }

    return data;
}
 

/**
 * Read quoted CSV file from HDFS using explicit plugins and CSV format.
 *
 * @throws Exception
 */
@Test(groups = {"features", "gpdb", "security"})
public void readCsvUsingCsvFormat() throws Exception {
    // set plugins and format
    exTable.setFragmenter("org.greenplum.pxf.plugins.hdfs.HdfsDataFragmenter");
    exTable.setAccessor("org.greenplum.pxf.plugins.hdfs.LineBreakAccessor");
    exTable.setResolver("org.greenplum.pxf.plugins.hdfs.StringPassResolver");
    exTable.setFormat("CSV");
    // create external table
    gpdb.createTableAndVerify(exTable);
    // create local CSV file
    String tempLocalDataPath = dataTempFolder + "/data.csv";
    CsvUtils.writeTableToCsvFile(dataTable, tempLocalDataPath);
    // copy local CSV to HDFS
    hdfs.copyFromLocal(tempLocalDataPath, hdfsFilePath);
    // verify results
    runTincTest("pxf.features.hdfs.readable.text.small_data.runTest");
}
 

/**
 * Read all csv data at once. This can only be done before readNextCsvLine() was called for the first time
 *
 * @return the list
 * @throws IOException
 *             Signals that an I/O exception has occurred.
 * @throws CsvDataException
 *             the csv data exception
 */
public List<List<String>> readAll() throws IOException, CsvDataException {
	if (singleReadStarted) {
		throw new IllegalStateException("Single readNextCsvLine was called before readAll");
	}

	try {
		List<List<String>> csvValues = new ArrayList<>();
		List<String> lineValues;
		while ((lineValues = readNextCsvLine()) != null) {
			csvValues.add(lineValues);
		}
		return csvValues;
	} finally {
		close();
	}
}
 

@Test
public void readFromCsvTest() throws FileNotFoundException {
    ClassLoader loader = Thread.currentThread().getContextClassLoader();
    URL uri = loader.getResource("row_id-column_id-timestamp.csv");
    if (uri == null) {
        fail("row_id-column_id-timestamp not found");
    }
    CsvFileColumnIterator iterator = CsvFileColumnIterator.fromPath(uri.getPath());
    List<Column> columns = new ArrayList<>(3);
    while (iterator.hasNext()) {
        columns.add(iterator.next());
    }
    List<Column> target = getTargetRows();
    assertEquals(3, columns.size());
    assertEquals(target, columns);

    // to get %100 test coverage
    assertFalse(iterator.hasNext());
    iterator.remove();
}
 

@Test
public void testReadInvalidContentsLenient() {
	try {
		final String fileContent = "abc|222|def|444\nkkz|777|888|hhg";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, IntValue.class, StringValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new IntValue(), new StringValue(), new IntValue() };
		
		assertNotNull(format.nextRecord(values));
		assertNull(format.nextRecord(values));
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

public static Set<Trace> readInputFromCSV(String fileName) {
    Charset charset = Charset.forName("US-ASCII");
    Path file = FileSystems.getDefault().getPath(fileName);
    Set<Trace> toReturn = new HashSet<>();
    try (BufferedReader reader = Files.newBufferedReader(file, charset)) {
        String line;
        while ((line = reader.readLine()) != null) {
            String[] events = line.split(",");
            if (events == null || events.length == 0) {
                throw new IOException(
                        "Input file not in correct format, empty line read");
            }

            toReturn.add(new Trace(events));
        }
    } catch (IOException x) {
        System.err.format("IOException: %s%n", x);
        return new HashSet<>();
    }

    return toReturn;
}
 

private static CellProcessor[] getProcessors4ClassifiedTweetIDSCCSV() {

		final CellProcessor[] processors = new CellProcessor[]{
				//new UniqueHashCode(), // tweetID (must be unique)
				new NotNull(),	// tweetID (must be unique): sometimes CAN be null!
				new NotNull(), 	// crisis name
				new Optional(),	// attribute name
				new Optional(), // attribute code
				new Optional(), // label name
				new Optional(), // label description
				new Optional(),	// label code
				new Optional(), // confidence
				new Optional(), // humanLabeled
		};
		return processors;
	}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

/**
 * Tests the read() method with no processors, populating an existing bean.
 */
@Test
public void testReadIntoExistingBean() throws IOException {
	
	final String[] header = beanReader.getHeader(true);
	assertArrayEquals(HEADER, header);
	
	assertEquals(JOHN_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(BOB_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(ALICE_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(BILL_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(MIRANDA_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(STEVE_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(ADA_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(SERGEI_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(LARRY_STRING, beanReader.read(new CustomerStringBean(), header));
	assertEquals(GRACE_STRING, beanReader.read(new CustomerStringBean(), header));
	assertNull(beanReader.read(new CustomerStringBean(), header));
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

/**This method parses csv and returns instrument list.
 * @param input is csv string.
 * @return  returns list of instruments.
 * @throws IOException is thrown when there is connection related error.
 * */
private List<Instrument> readCSV(String input) throws IOException {
    ICsvBeanReader beanReader = null;
    File temp = File.createTempFile("tempfile", ".tmp");
    BufferedWriter bw = new BufferedWriter(new FileWriter(temp));
    bw.write(input);
    bw.close();

    beanReader = new CsvBeanReader(new FileReader(temp), CsvPreference.STANDARD_PREFERENCE);
    String[] header = beanReader.getHeader(true);
    CellProcessor[] processors = getProcessors();
    Instrument instrument;
    List<Instrument> instruments = new ArrayList<>();
    while((instrument = beanReader.read(Instrument.class, header, processors)) != null ) {
        instruments.add(instrument);
    }
    return instruments;
}
 

/**
 * Tests the read() method using processors.
 */
@Test
public void testReadWithProcessors() throws IOException {
	
	final String[] header = beanReader.getHeader(true);
	assertArrayEquals(HEADER, header);
	
	assertEquals(JOHN, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(BOB, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(ALICE, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(BILL, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(MIRANDA, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(STEVE, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(ADA, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(SERGEI, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(LARRY, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertEquals(GRACE, beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
	assertNull(beanReader.read(CustomerBean.class, header, READ_PROCESSORS));
}
 

public void ReadTestCSVFile(Reader reader) throws IOException {
	ICsvBeanReader beanReader = new CsvBeanReader(reader, CsvPreference.STANDARD_PREFERENCE);
	final String[] header = beanReader.getHeader(true);
	assertEquals("customerNo", header[0]);
	CustomerBean customer = null;
	final String emailRegex = "[a-z0-9\\._]+@[a-z0-9\\.]+"; // just an example, not very robust!
	StrRegEx.registerMessage(emailRegex, "must be a valid email address");
	final CellProcessor[] processors = new CellProcessor[]{new UniqueHashCode(), // customerNo (must be unique)
			new NotNull(), // firstName
			new NotNull(), // lastName
			new ParseDate("dd/MM/yyyy"), // birthDate
			new ParseSqlTime("HH:mm:ss"), // birthTime
			new NotNull(), // mailingAddress
			new Optional(new ParseBool()), // married
			new Optional(new ParseInt()), // numberOfKids
			new NotNull(), // favouriteQuote
			new StrRegEx(emailRegex), // email
			new LMinMax(0L, LMinMax.MAX_LONG) // loyaltyPoints
	};
	customer = beanReader.read(CustomerBean.class, header, processors);
	assertEquals("1", customer.getCustomerNo());
	assertEquals("John", customer.getFirstName());
	assertEquals("[email protected]", customer.getEmail());
	assertEquals(0, customer.getLoyaltyPoints());
	beanReader.close();
}
 

/**
 * Extracts fields from a line, using CSVReader.
 *
 * @param line line as it's in the CSV raw file
 * @return a list of ordered fields
 */
private List<String> readLine(String line) {
    List<String> result = Collections.emptyList();
    try (CSVReader csvReader =
            new CSVReader(new InputStreamReader(IOUtils.toInputStream(line)), separator.getSeparator())) {
        String[] fields = csvReader.readNext();
        csvReader.close();
        if (fields != null && fields.length != 0) {
            result = Arrays.asList(fields).stream().collect(Collectors.toList());
        }
    } catch (IOException e) {
        LOGGER.info("Unable to read line {i} of sample", line, e);
    }
    // remove last fields if it is empty
    int size = result.size();
    if (size > 0 && StringUtils.isEmpty(result.get(size - 1))) {
        result.remove(size - 1);
    }
    return result;
}
 

/**
 * Reads a row of a CSV file and populates the a bean, using Dozer to map column values to the appropriate field. If
 * an existing bean is supplied, Dozer will populate that, otherwise Dozer will create an instance of type clazz
 * (only one of bean or clazz should be supplied). If processors are supplied then they are used, otherwise the raw
 * String values will be used.
 * 
 * @param bean
 *            the bean to populate (if null, then clazz will be used instead)
 * @param clazz
 *            the type to instantiate (only required if bean is null)
 * @param processors
 *            the (optional) cell processors
 * @return the populated bean
 * @throws IOException
 *             if an I/O error occurred
 */
private <T> T readIntoBean(final T bean, final Class<T> clazz, final CellProcessor[] processors) throws IOException {
	if( readRow() ) {
		if( processors == null ) {
			// populate bean data with the raw String values
			beanData.getColumns().clear();
			beanData.getColumns().addAll(getColumns());
		} else {
			// populate bean data with the processed values
			executeProcessors(beanData.getColumns(), processors);
		}
		
		if( bean != null ) {
			// populate existing bean
			dozerBeanMapper.map(beanData, bean);
			return bean;
		} else {
			// let Dozer create a new bean
			return dozerBeanMapper.map(beanData, clazz);
		}
		
	}
	
	return null; // EOF
}
 

/**
 * 从CSV文件流中读取一个CSV行。
 */
public String readLine() throws Exception {
	StringBuffer readLine = new StringBuffer();
	boolean bReadNext = true;
	while (bReadNext) {
		if (readLine.length() > 0) {
			readLine.append("\r\n");
		}
		String strReadLine = br.readLine();
		// readLine is Null
		if (strReadLine == null) {
			return null;
		}
		readLine.append(strReadLine);
		// 如果双引号是奇数的时候继续读取。考虑有换行的是情况。
		if (countChar(readLine.toString(), '"', 0) % 2 == 1) {
			bReadNext = true;
		} else {
			bReadNext = false;
		}
	}
	return readLine.toString();
}
 

public void readFromFile() throws IOException {
  Scanner in = new Scanner(new BufferedReader(new FileReader(file)));
  try {
    in.useDelimiter(",");
    headerLine = in.nextLine(); // Skip header row
    while (in.hasNextLine()) {
      String srgName = in.next();
      String mcpName = in.next();
      String side = in.next();
      String comment = in.nextLine().substring(1);
      srgMemberName2CsvData.put(
          srgName, new CsvData(srgName, mcpName, Integer.valueOf(side), comment));
    }
  } finally {
    in.close();
  }
}
 

@Override
public FrameBlock readFrameFromInputStream(InputStream is, ValueType[] schema, String[] names, 
		long rlen, long clen)
	throws IOException, DMLRuntimeException 
{
	//allocate output frame block
	ValueType[] lschema = createOutputSchema(schema, clen);
	String[] lnames = createOutputNames(names, clen);
	FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);

	//core read (sequential/parallel) 
	InputStreamInputFormat informat = new InputStreamInputFormat(is);
	InputSplit split = informat.getSplits(null, 1)[0];
	readCSVFrameFromInputSplit(split, informat, null, ret, schema, names, rlen, clen, 0, true);
	
	return ret;
}
 

public CSVReadTask(InputSplit split, SplitOffsetInfos offsets,
		TextInputFormat informat, JobConf job, MatrixBlock dest,
		long rlen, long clen, boolean hasHeader, String delim,
		boolean fill, double fillValue, int splitCount) 
{
	_split = split;
	_splitoffsets = offsets; // new SplitOffsetInfos(offsets);
	_sparse = dest.isInSparseFormat();
	_informat = informat;
	_job = job;
	_dest = dest;
	_rlen = rlen;
	_clen = clen;
	_isFirstSplit = (splitCount == 0);
	_hasHeader = hasHeader;
	_fill = fill;
	_fillValue = fillValue;
	_delim = delim;
	_rc = true;
	_splitCount = splitCount;
}
 

@Test
public void readCSVDataExport() throws Exception {
    String filename = ClassLoader.getSystemResource("executions/correlation/dummyDataExport.csv").getPath();

    RealMatrix matrix = CorrelationAnalysisEngine.readCSVDataExport(filename,fields);

    assertEquals(3, matrix.getColumnDimension());
    assertEquals(2, matrix.getRowDimension());

    assertEquals(0.4d, matrix.getEntry(0,0), 0.1d);
    assertEquals(1d, matrix.getEntry(0,1), 0.1d);
    assertEquals(4d, matrix.getEntry(0,2), 0.1d);

    assertEquals(0.6d, matrix.getEntry(1,0), 0.1d);
    assertEquals(2d, matrix.getEntry(1,1), 0.1d);
    assertEquals(8d, matrix.getEntry(1,2), 0.1d);
}
 

@Test
public void testReadMultipleFormatOverrideDefault() throws  Exception {

	String data2 = "date1\n06-19-2016";


	final List<String> strings = new ArrayList<String>();
	CsvParser.forEach(new StringReader(data2), new CheckedConsumer<String[]>() {
		@Override
		public void accept(String[] s) throws Exception {
			strings.add(s[0]);
		}
	});

	assertArrayEquals(new String[] { "date1",  "06-19-2016" }, strings.toArray(new String[0]));

}
 

@Test
public void testReadMultipleFormatOverrideDefault2() throws  Exception {

	String data2 = "date1\n06-19-2016";


	final List<String> strings = new ArrayList<String>();
	CsvParser.bufferSize(8).forEach(new StringReader(data2), new CheckedConsumer<String[]>() {
		@Override
		public void accept(String[] s) throws Exception {
			strings.add(s[0]);
		}
	});

	assertArrayEquals(new String[] { "date1",  "06-19-2016" }, strings.toArray(new String[0]));

}
 

@Override
public MatrixBlock readMatrixFromInputStream(InputStream is, long rlen, long clen, int blen, long estnnz) 
	throws IOException, DMLRuntimeException 
{
	//allocate output matrix block
	MatrixBlock ret = createOutputMatrixBlock(rlen, clen, (int)rlen, estnnz, true, false);
	
	//core read 
	long lnnz = readCSVMatrixFromInputStream(is, "external inputstream", ret, new MutableInt(0), rlen, clen, 
		blen, _props.hasHeader(), _props.getDelim(), _props.isFill(), _props.getFillValue(), true);
			
	//finally check if change of sparse/dense block representation required
	ret.setNonZeros( lnnz );
	ret.examSparsity();
	
	return ret;
}
 

/**
 * An example of reading using CsvBeanReader.
 */
private static void readWithCsvBeanReader() throws Exception {
	
	ICsvBeanReader beanReader = null;
	try {
		beanReader = new CsvBeanReader(new FileReader(CSV_FILENAME), CsvPreference.STANDARD_PREFERENCE);
		
		// the header elements are used to map the values to the bean (names must match)
		final String[] header = beanReader.getHeader(true);
		final CellProcessor[] processors = getProcessors();
		
		CustomerBean customer;
		while( (customer = beanReader.read(CustomerBean.class, header, processors)) != null ) {
			System.out.println(String.format("lineNo=%s, rowNo=%s, customer=%s", beanReader.getLineNumber(),
				beanReader.getRowNumber(), customer));
		}
		
	}
	finally {
		if( beanReader != null ) {
			beanReader.close();
		}
	}
}
 

@Override
public Response readAllCsv ( final String contextId )
{
    final DataContext context = this.provider.getContext ( contextId );

    if ( context == null )
    {
        logger.trace ( "Context '{}' not found", contextId ); //$NON-NLS-1$
        throw new WebApplicationException ( Status.NOT_FOUND );
    }

    final SortedMap<String, DataItemValue> values = context.getAllValues ();

    final StreamingOutput out = new StreamingOutput () {

        @Override
        public void write ( final OutputStream output ) throws IOException, WebApplicationException
        {
            streamAsCSV ( new PrintWriter ( new OutputStreamWriter ( output, StandardCharsets.UTF_8 ) ), values, ",", "\r\n" ); //$NON-NLS-1$  //$NON-NLS-2$
        }
    };
    return Response.ok ( out ).header ( "Content-Disposition", "attachment; filename=\"data.csv\"" ).build (); //$NON-NLS-1$
}
 

/**
 * Reads a {@link CategoryDataset} from a CSV file or input source.
 *
 * @param in  the input source.
 *
 * @return A category dataset.
 *
 * @throws IOException if there is an I/O problem.
 */
public CategoryDataset readCategoryDataset(Reader in) throws IOException {

    DefaultCategoryDataset dataset = new DefaultCategoryDataset();
    BufferedReader reader = new BufferedReader(in);
    List columnKeys = null;
    int lineIndex = 0;
    String line = reader.readLine();
    while (line != null) {
        if (lineIndex == 0) {  // first line contains column keys
            columnKeys = extractColumnKeys(line);
        }
        else {  // remaining lines contain a row key and data values
            extractRowKeyAndData(line, dataset, columnKeys);
        }
        line = reader.readLine();
        lineIndex++;
    }
    return dataset;

}
 

private void testReadForBrokenCSV(final CsvDozerBeanReader brokenCsvBeanReader, final List<SurveyResponse> responses,
		final String expectedMessage, final int expectedRowsRead) throws IOException {
	brokenCsvBeanReader.getHeader(true);
	brokenCsvBeanReader.configureBeanMapping(SurveyResponse.class, FIELD_MAPPING);
	Exception expected = null;
	for(int i = 0; i < 4; i++) {
		try{
			final SurveyResponse response = readSurveyResponse(brokenCsvBeanReader, USE_PROCESSORS, EXISTING_BEAN);
			if(response != null) {
				responses.add(response);
			}
		} catch (final SuperCsvException e) {
			expected = e;
		}
	}
	assertNotNull(expected);
	assertEquals(expectedMessage, expected.getLocalizedMessage());
	assertEquals(expectedRowsRead, responses.size());
}
 

public static double[] readDoubleArrayFromCSVFile(Path inFilePath,  int nrows, char delimiter) throws NumberFormatException, IOException {
    double[] mlDouble = new double[nrows];
    CsvReader cvsReader = new CsvReader(inFilePath.toString());
    cvsReader.setDelimiter(delimiter);
    int i = 0;
    while (cvsReader.readRecord()) {
        String[] rows = cvsReader.getValues();
        for (String col : rows) {
            mlDouble[i] = Double.parseDouble(col);
        }
        i++;
        if (i >= nrows) {
            break;
        }
    }
    return mlDouble;
}
 

/**
 * Tests the read() method with no processors.
 */
@Test
public void testRead() throws IOException {
	
	final String[] header = beanReader.getHeader(true);
	assertArrayEquals(HEADER, header);
	
	assertEquals(JOHN_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(BOB_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(ALICE_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(BILL_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(MIRANDA_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(STEVE_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(ADA_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(SERGEI_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(LARRY_STRING, beanReader.read(CustomerStringBean.class, header));
	assertEquals(GRACE_STRING, beanReader.read(CustomerStringBean.class, header));
	assertNull(beanReader.read(CustomerStringBean.class, header));
}
 

@Test
public void testReadCsvNoHeaderInputStream()
throws IOException {
	DataFrame<Object> df_noHeader = DataFrame.readCsv(ClassLoader.getSystemResourceAsStream("serialization_no_header.csv"), ",", "NA", false);
    final Object[][] expected = new Object[][] {
            new Object[] { "a", "a", "b", "b", "c", "c" },
            new Object[] { "alpha", "bravo", "charlie", "delta", "echo", "foxtrot" },
            new Object[] { 1L, 2L, 3L, 4L, 5L, 6L }
        };

    for (int i = 0; i < expected.length; i++) {
        assertArrayEquals(
                expected[i],
                df_noHeader.col(i).toArray()
            );
    }
}
 

@Test
public void testReadInvalidContentsLenient() {
	try {
		final String fileContent = "abc|222|def|444\nkkz|777|888|hhg";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter('|');
		format.setFieldTypesGeneric(StringValue.class, IntValue.class, StringValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new IntValue(), new StringValue(), new IntValue() };
		
		assertNotNull(format.nextRecord(values));
		assertNull(format.nextRecord(values));
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testReadCsvSemicolonInputStream()
throws IOException {
    DataFrame<Object> cdf = DataFrame.readCsv(ClassLoader.getSystemResourceAsStream("serialization_semicolon.csv"), ";");
    final Object[][] expected = new Object[][] {
            new Object[] { "a", "a", "b", "b", "c", "c" },
            new Object[] { "alpha", "bravo", "charlie", "delta", "echo", "foxtrot" },
            new Object[] { 1L, 2L, 3L, 4L, 5L, 6L }
        };

    for (int i = 0; i < expected.length; i++) {
        assertArrayEquals(
                expected[i],
                cdf.col(i).toArray()
            );
    }
}
 

private void readCSVData() throws IOException {
    File dataFile = new File(fileName);
    if (!dataFile.exists()) {
        File dataDir = new File(System.getProperty(Constants.PROP_PROJECT_DIR), "TestData");
        dataFile = new File(dataDir, fileName);
    }
    CSVReader reader = null;
    try {
        reader = new CSVReader(new FileReader(dataFile));
        data = reader.readAll();
        if (data == null || data.size() < 2) {
            throw new IllegalArgumentException("No data in CSV file?");
        }
        header = data.get(0);
        currentIndex = 1;
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}
 

public static List<String[]> readAll(Reader reader) {

        CSVParser parser = new CSVParserBuilder()
                .withSeparator(',')
                .withIgnoreQuotations(true)
                .build();

        CSVReader csvReader = new CSVReaderBuilder(reader)
                .withSkipLines(0)
                .withCSVParser(parser)
                .build();

        List<String[]> list = new ArrayList<>();
        try {
            list = csvReader.readAll();
            reader.close();
            csvReader.close();
        } catch (Exception ex) {
            Helpers.err(ex);
        }
        return list;
    }
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter('|');
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

/**
 * CSV読込処理を行います。
 * <p>大量データ処理を想定してメモリ内に全展開するのではなく、Iteratorを用いた
 * 行処理形式を利用しています。
 * @param logic
 */
public void read(final CsvReadLine logic) {
    InputStream ins = null;
    try {
        ins = fromBinary() ? new ByteArrayInputStream(data) : this.ins;
        readStream(ins, logic);
    } catch (RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new InvocationException("リソース処理中に例外が発生しました", e);
    } finally {
        if (fromBinary()) {
            closeQuietly(ins);
        }
    }
}
 

public void readStream(final InputStream in, final CsvReadLine logic) throws Exception {
    PushbackReader reader = new PushbackReader(new InputStreamReader(in, layout.getCharset()), 2);
    try {
        int lineNum = 0;
        boolean title = false;
        while (hasNext(in, reader)) {
            lineNum++;
            List<String> row = readStreamLine(reader); // ヘッダ定義でも行を読み込み、シークを先に進める
            if (lineNum == 1 && StringUtils.isNotBlank(layout.getHeader())) {
                title = true;
                continue; // ヘッダ定義存在時は最初の行をスキップ
            }
            logic.execute(title ? lineNum - 1 : lineNum, row);
        }
    } finally {
        closeQuietly(reader);
    }
}