Java Code Examples for org.apache.commons.csv.CSVParser#getRecords()

@Test
public void testNullValuesAndEmptyStrings() throws IOException {
	String dataSetFile = this.getClass().getResource("/trickyDataset.csv").toString();

	CsvFileAccess datasetCsvAccess = new CsvFileAccess(dataSetFile);
	Optional<CSVParser> csvParser = datasetCsvAccess.getCsvRecords();

	assertThat(csvParser.isPresent(), is(true));

	CSVParser csvRecords = csvParser.get();

	List<CSVRecord> records = csvRecords.getRecords();
	assertThat(records.size(), is(4));

	checkRow(records.get(0), is(""), is(nullValue()), is("hello"));
	checkRow(records.get(1), is("not empty"), is("not null"), is("trimmed"));
	checkRow(records.get(2), is(""), is(nullValue()), is("comma, here"));
	checkRow(records.get(3), is(""), is(nullValue()), is("  not trimmed  "));
}
 

private boolean parseCsvRecords(boolean status, CSVParser csvParser, Neo4jDBHandler dbHandler,
		Map<String, Integer> headerMap, String query)
		throws IOException, GraphDBException, InsightsCustomException {
	List<JsonObject> nodeProperties = new ArrayList<>();
	List<String> combo = new ArrayList<>();
	getCurrentRecords(combo, dbHandler);
	int record = 0;
	for (CSVRecord csvRecord : csvParser.getRecords()) {
		JsonObject json = getHierachyDetails(csvRecord, headerMap);
		record = record + 1;
		json.addProperty(DatataggingConstants.METADATA_ID, Instant.now().getNano() + record);
		json.addProperty(DatataggingConstants.CREATIONDATE, Instant.now().toEpochMilli());
		nodeProperties.add(json);
		updateComboList(combo, json);
	}
	JsonObject graphResponse = dbHandler.bulkCreateNodes(nodeProperties, null, query);
	if (graphResponse.get(DatataggingConstants.RESPONSE).getAsJsonObject().get(DatataggingConstants.ERRORS)
			.getAsJsonArray().size() > 0) {
		log.error(graphResponse);
		return status;
	}

	return true;
}
 

/**
 * Send records to getToolFileDetails() and store the output in neo4j database
 *
 * @param csvParser
 * @param label
 * @param insightsTimeField
 * @param insightsTimeFormat
 * @return boolean
 * @throws InsightsCustomException
 */
private boolean parseCsvRecords(CSVParser csvParser, String label, String insightsTimeField,
		String insightsTimeFormat) throws InsightsCustomException {
	List<JsonObject> nodeProperties = new ArrayList<>();
	String query = "UNWIND {props} AS properties " + "CREATE (n:" + label.toUpperCase() + ") "
			+ "SET n = properties";
	Map<String, Integer> headerMap = csvParser.getHeaderMap();
	try {
		if (headerMap.containsKey("")) {
			throw new InsightsCustomException("Error in file.");
		} else if (headerMap.containsKey(insightsTimeField)) {
			for (CSVRecord csvRecord : csvParser.getRecords()) {
				JsonObject json = getCSVRecordDetails(csvRecord, headerMap, insightsTimeField, insightsTimeFormat);
				nodeProperties.add(json);
			}
		} else {
			throw new InsightsCustomException("Insights Time Field not present in csv file");
		}
		insertDataInDatabase(nodeProperties, query);
		return true;
	} catch (Exception ex) {
		log.error("Error while parsing the .CSV records. {} ", ex.getMessage());
		throw new InsightsCustomException(ex.getMessage());
	}
}
 

public static List<Map<String, String>> readFile(InputStream inputStream)
{
    List<Map<String, String>> data = new LinkedList<>();
    CSVParser csvParser;
    try
    {
        csvParser = CSVParser.parse(inputStream, CHARSET_UTF8, CSV_FORMAT);
        for (CSVRecord record : csvParser.getRecords())
        {
            data.add(record.toMap());
        }

    }
    catch (IOException e)
    {
        throw new RuntimeException(e);
    }

    return data;
}
 

public static List<CSVRecord> getRecords(File file) {
    if (file.exists()) {
        try (Reader in = new FileReader(file)) {
            CSVParser parser = CSVFormat.EXCEL.withHeader().withSkipHeaderRecord().withIgnoreEmptyLines().parse(in);
            if (!parser.getHeaderMap().isEmpty()) {
                return parser.getRecords();
            }
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, null, ex);
        }
    } else {
        LOGGER.log(Level.SEVERE, "File [{0}] doesn''t exist", file.getAbsolutePath());
    }
    return new ArrayList<>();
}
 

@NotNull
private static List<CSVRecord> readCsvSkipHeader(@NotNull String pathToCsv, char delimiter) throws IOException {
    CSVFormat format = CSVFormat.DEFAULT.withDelimiter(delimiter);
    CSVParser parser = format.parse(new BufferedReader(new InputStreamReader(new FileInputStream(pathToCsv))));

    List<CSVRecord> records = parser.getRecords();
    return records.subList(1, records.size());
}
 

@Override
protected void importDatasource(Datasource datasource, List<String> geographyScope, List<String> temporalScope, List<String> datasourceLocation) throws Exception {

    CSVExtractor subjectLabelExtractor = new CSVExtractor(0);
    List<TimedValueExtractor> extractors = getExtractors(subjectLabelExtractor);

    String line = null;
    BufferedReader br = new BufferedReader(new FileReader(
            downloadUtils.fetchFile(new URL(DATAFILE), getProvider().getLabel(), DATAFILE_SUFFIX)));
    List<TimedValue> timedValueBuffer = new ArrayList<>();
    while ((line = br.readLine())!=null) {
        CSVParser parser = CSVParser.parse(line, CSVFormat.DEFAULT);
        List<CSVRecord> records = parser.getRecords();
        subjectLabelExtractor.setCsvRecord(records.get(0));
        for(TimedValueExtractor extractor: extractors){
            ((CSVExtractor)extractor.getValueExtractor()).setCsvRecord(records.get(0));
            try {
                timedValueBuffer.add(extractor.extract());
            }catch (UnknownSubjectLabelException e){
                // No reason to panic even if Subject does not exist and no reason to run the rest of the extractors
                // Keep Calm and Break
                break;
            }
        }
    }
    br.close();
    saveAndClearTimedValueBuffer(timedValueBuffer);
}
 

public static void extractAndSaveTimedValues(List<TimedValueExtractor> extractors, Importer importer, File localFile)
        throws IOException, ExtractorException {

    String line;
    BufferedReader br = new BufferedReader(new FileReader(localFile));
    List<TimedValue> timedValueBuffer = new ArrayList<>();
    while ((line = br.readLine())!=null) {
        CSVParser parser = CSVParser.parse(line, CSVFormat.DEFAULT);
        List<CSVRecord> records = parser.getRecords();
        for(TimedValueExtractor extractor: extractors){
            if (extractor.getSubjectLabelExtractor() instanceof CSVExtractor)
                ((CSVExtractor) extractor.getSubjectLabelExtractor()).setCsvRecord(records.get(0));
            if (extractor.getAttributeLabelExtractor() instanceof CSVExtractor)
                ((CSVExtractor) extractor.getAttributeLabelExtractor()).setCsvRecord(records.get(0));
            if (extractor.getTimestampExtractor() instanceof CSVExtractor)
                ((CSVExtractor) extractor.getTimestampExtractor()).setCsvRecord(records.get(0));
            if (extractor.getValueExtractor() instanceof CSVExtractor)
                ((CSVExtractor) extractor.getValueExtractor()).setCsvRecord(records.get(0));
            try {
                timedValueBuffer.add(extractor.extract());
            }catch (UnknownSubjectLabelException e){
                // No reason to panic even if Subject does not exist and no reason to run the rest of the extractors
                // Keep Calm and Break
                break;
            } catch (ExtractorException ee) {
                ee.getMessage();
            }
        }
    }
    br.close();
    importer.saveAndClearTimedValueBuffer(timedValueBuffer);
}
 

public static CSVHParser getCSVHParser(File file) {
    if (file.exists()) {
        try (Reader in = new FileReader(file)) {
            CSVParser parser = CSVFormat.EXCEL.withHeader().withSkipHeaderRecord().withIgnoreEmptyLines().parse(in);
            if (!parser.getHeaderMap().isEmpty()) {
                return new CSVHParser(parser.getHeaderMap(), parser.getRecords());
            }
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, null, ex);
        }
    } else {
        LOGGER.log(Level.SEVERE, "File [{0}] doesn''t exist", file.getAbsolutePath());
    }
    return null;
}
 

private List<String[]> getRecords(String content, CSVFormat format) throws IOException {
    List<String[]> records = new ArrayList<>();
    CSVParser parser = CSVParser.parse(content, format);
    for (CSVRecord record : parser.getRecords()) {
        String[] line = new String[record.size()];
        for (int i = 0; i < line.length; i++) {
            line[i] = record.get(i);
        }
        records.add(line);
    }
    return records;
}
 

public static void loadFileinTable(File file, JTable table) {
    if (file.exists()) {
        try (Reader in = new FileReader(file)) {
            CSVParser parser = CSVFormat.EXCEL.withHeader().withSkipHeaderRecord().withIgnoreEmptyLines().parse(in);
            if (!parser.getHeaderMap().isEmpty()) {
                DefaultTableModel model = (DefaultTableModel) table.getModel();
                for (String columnHeader : parser.getHeaderMap().keySet()) {
                    if (!columnHeader.trim().isEmpty()) {
                        model.addColumn(columnHeader);
                    }
                }
                List<CSVRecord> records = parser.getRecords();
                for (CSVRecord record : records) {
                    Object[] row = new Object[record.size()];
                    for (int i = 0; i < record.size(); i++) {
                        row[i] = record.get(i);
                    }
                    model.addRow(row);
                }
            }
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, null, ex);
        }
    } else {
        LOGGER.log(Level.SEVERE, "File [{0}] doesn''t exist", file.getAbsolutePath());
    }
}
 

/**
 * This method returns a list of CSV records for a data source.
 * @param access the access from which records need to be fetched.
 * @param logicalSource the used Logical Source.
 * @param rmlStore the QuadStore with the RML rules.
 * @return a list of records.
 * @throws IOException
 */
@Override
public List<Record> getRecords(Access access, Term logicalSource, QuadStore rmlStore) throws IOException, SQLException, ClassNotFoundException {
    List<Term> sources = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalSource, new NamedNode(NAMESPACES.RML + "source"), null));
    Term source = sources.get(0);
    CSVParser parser;

    if (source instanceof Literal) {
        // We are not dealing with something like CSVW.
        parser = getParserForNormalCSV(access);
    } else {
        List<Term> sourceType = Utils.getObjectsFromQuads(rmlStore.getQuads(source, new NamedNode(NAMESPACES.RDF + "type"), null));

        // Check if we are dealing with CSVW.
        if (sourceType.get(0).getValue().equals(NAMESPACES.CSVW + "Table")) {
            CSVW csvw = new CSVW(access.getInputStream(), rmlStore, logicalSource);
            parser = csvw.getCSVParser();
        } else {
            // RDBs fall under this.
            parser = getParserForNormalCSV(access);
        }
    }

    if (parser != null) {
        List<org.apache.commons.csv.CSVRecord> myEntries = parser.getRecords();

        return myEntries.stream()
                .map(record -> new CSVRecord(record, access.getDataTypes()))
                .collect(Collectors.toList());
    } else {
        // We still return an empty list of records when a parser is not found.
        // This is to support certain use cases with RDBs where queries might not be valid,
        // but you don't want the RMLMapper to crash.
        return new ArrayList<>();
    }
}
 

public static CSVRecord firstRecord(String s) {
    try {
        CSVParser parser = CSVParser.parse(s, CSVFormat.DEFAULT);
        List<CSVRecord> records = parser.getRecords();
        if (records.size() < 1) {
            throw new IllegalArgumentException("Unable to find first record: " + s);
        }
        return records.get(0);
    } catch (IOException e) {
        throw new IllegalArgumentException("Unable to find first record: " + s);
    }
}
 

/**
 * Retrieves all user identity mappings for the identity source. For the
 * full sync connector, the repository must provide a complete snapshot
 * of the mappings. This is reconciled against the current mappings
 * in Cloud Directory. All identity mappings returned here are
 * set in Cloud Directory. Any previously mapped users that are omitted
 * are unmapped.
 *
 * The connector does not create new users. All users are assumed to
 * exist in Cloud Directory.
 *
 * @param checkpoint Saved state if paging over large result sets. Not used
 *                   for this sample.
 * @return Iterator of user identity mappings
 * @throws IOException if unable to read user identity mappings
 */
@Override
public CheckpointCloseableIterable<IdentityUser> listUsers(byte[] checkpoint)
    throws IOException {
  List<IdentityUser> users = new ArrayList<>();
  try (Reader in = new FileReader(userMappingCsvPath)) {
    // Read user mappings from CSV file
    CSVParser parser = CSVFormat.RFC4180
        .withIgnoreSurroundingSpaces()
        .withIgnoreEmptyLines()
        .withCommentMarker('#')
        .parse(in);
    for (CSVRecord record : parser.getRecords()) {
      // Each record is in form: "primary_email", "external_id"
      String primaryEmailAddress = record.get(0);
      String externalId = record.get(1);
      if (primaryEmailAddress.isEmpty() || externalId.isEmpty()) {
        // Skip any malformed mappings
        continue;
      }
      log.info(() -> String.format("Adding user %s/%s",
          primaryEmailAddress, externalId));

      // Add the identity mapping
      IdentityUser user = context.buildIdentityUser(
          primaryEmailAddress, externalId);
      users.add(user);
    }
  }
  // [START_EXCLUDE]
  // [START cloud_search_identity_sdk_user_checkpoint_iterator]
  CheckpointCloseableIterable<IdentityUser> iterator =
    new CheckpointCloseableIterableImpl.Builder<IdentityUser>(users)
        .setHasMore(false)
        .setCheckpoint((byte[])null)
        .build();
  // [END cloud_search_identity_sdk_user_checkpoint_iterator]
  return iterator;
  // [END_EXCLUDE]
}
 

public void importData() throws Exception {
    TimestreamMetaTable metaTable = db.metaTable();
    TimestreamDataTable dataTable = db.dataTable(conf.getDataTableName());

    String [] fileHeader = {"分类", "名称", "监管号", "生产日期", "位置", "环节", "状态"};
    String csvFile = conf.getDataFile();
    CSVFormat format = CSVFormat.DEFAULT.withHeader(fileHeader).withIgnoreHeaderCase().withTrim();
    Reader reader = Files.newBufferedReader(Paths.get(csvFile));
    CSVParser csvParser = new CSVParser(reader, format);
    for (CSVRecord r : csvParser.getRecords()) {
        TimestreamIdentifier identifier = new TimestreamIdentifier.Builder(r.get("分类"))
                .addTag("名称", r.get("名称"))
                .addTag("监管号", r.get("监管号"))
                .build();

        TimestreamMeta meta = new TimestreamMeta(identifier);

        String loc = toLocationString(r.get("位置"));
        String links = r.get("环节");
        String status = r.get("状态");

        meta.addAttribute("loc", loc);
        meta.addAttribute("links", links);
        meta.addAttribute("status", status);
        metaTable.update(meta);

        Point point = new Point.Builder(this.getTimestamp(r, "生产日期"), TimeUnit.MILLISECONDS)
                .addField("loc", loc)
                .addField("links", links)
                .addField("status", status)
                .build();
        dataTable.asyncWrite(identifier, point);
        System.out.println(point.toString());
    }
    dataTable.flush();
}
 

@Override
public ProcessResult process(Doc doc) {
    if (!doc.hasField(field, String.class)) {
        return ProcessResult.failure(String.format("failed to parse csv, couldn't find field [%s] or not instance of String", field));
    }

    Map<String, Object> csv = new HashMap<>();
    List<CSVRecord> records;
    String csvString = doc.getField(field);

    try {
        CSVParser csvParser = CSVParser.parse(csvString,
                 CSVFormat.DEFAULT
                         .withDelimiter(getChar(separator))
                         .withQuote(getChar(quoteChar))
                         .withTrim());
        records = csvParser.getRecords();
    } catch (IOException e) {
        return ProcessResult.failure(String.format("failed to parse csv for csv [%s]", csvString),
                new ProcessorExecutionException("csv", e));
    }

    records.forEach(record -> {
        for (int i = 0; i < record.size(); i++) {
            if (skipEmptyColumns && StringUtils.isEmpty(record.get(i))) {
                continue;
            }

            boolean columnNameNotExists = columns.size() <= i || StringUtils.isEmpty(columns.get(i));
            if (!autoGenerateColumnNames && columnNameNotExists) {
                continue;
            }
            String fieldName = columnNameNotExists ? "column" + (i + 1) : columns.get(i);
            csv.put(fieldName, transform(fieldName, StringUtils.strip(record.get(i), quoteChar)));
        }
    });

    if (targetField != null) {
        doc.addField(targetField, csv);
    } else {
        csv.forEach(doc::addField);
    }

    return ProcessResult.success();
}
 

public static void main(String[] args) throws IOException, NumberFormatException, InterruptedException {
    if (args.length < 4) {
        log.info("Usage: \n param 1 - path to file \n param 2 - stream name to send the data \n param 3 - time in ms to wait to send each data \n param 4 - broker list");
    } else {
        Producer<String, String> producer = new Producer<String, String>(createProducerConfig(args[3]));
        Gson gson = new Gson();

        Reader in = new FileReader(args[0]);
        CSVParser parser = CSVFormat.DEFAULT.parse(in);

        List<String> columnNames = new ArrayList<>();
        for (CSVRecord csvRecord : parser.getRecords()) {

            if (columnNames.size() == 0) {
                Iterator<String> iterator = csvRecord.iterator();
                while (iterator.hasNext()) {
                    columnNames.add(iterator.next());
                }
            } else {
                StratioStreamingMessage message = new StratioStreamingMessage();

                message.setOperation(STREAM_OPERATIONS.MANIPULATION.INSERT.toLowerCase());
                message.setStreamName(args[1]);
                message.setTimestamp(System.currentTimeMillis());
                message.setSession_id(String.valueOf(System.currentTimeMillis()));
                message.setRequest_id(String.valueOf(System.currentTimeMillis()));
                message.setRequest("dummy request");

                List<ColumnNameTypeValue> sensorData = new ArrayList<>();
                for (int i = 0; i < columnNames.size(); i++) {

                    // Workaround
                    Object value = null;
                    try {
                        value = Double.valueOf(csvRecord.get(i));
                    } catch (NumberFormatException e) {
                        value = csvRecord.get(i);
                    }
                    sensorData.add(new ColumnNameTypeValue(columnNames.get(i), null, value));
                }

                message.setColumns(sensorData);

                String json = gson.toJson(message);
                log.info("Sending data: {}", json);
                producer.send(new KeyedMessage<String, String>(InternalTopic.TOPIC_DATA.getTopicName(),
                        STREAM_OPERATIONS.MANIPULATION.INSERT, json));

                log.info("Sleeping {} ms...", args[2]);
                Thread.sleep(Long.valueOf(args[2]));
            }
        }
        log.info("Program completed.");
    }
}
 

private HiscoreResultBuilder lookupUsername(String username, HttpUrl hiscoreUrl) throws IOException
{
	HttpUrl url = hiscoreUrl.newBuilder()
		.addQueryParameter("player", username)
		.build();

	log.debug("Built URL {}", url);

	Request okrequest = new Request.Builder()
		.url(url)
		.build();

	String responseStr;

	try (Response okresponse = client.newCall(okrequest).execute())
	{
		if (!okresponse.isSuccessful())
		{
			switch (okresponse.code())
			{
				case 404:
					return null;
				default:
					throw new IOException("Error retrieving data from Jagex Hiscores: " + okresponse);
			}
		}

		responseStr = okresponse.body().string();
	}

	CSVParser parser = CSVParser.parse(responseStr, CSVFormat.DEFAULT);

	HiscoreResultBuilder hiscoreBuilder = new HiscoreResultBuilder();
	hiscoreBuilder.setPlayer(username);

	int count = 0;

	for (CSVRecord record : parser.getRecords())
	{
		if (count++ >= HiscoreSkill.values().length)
		{
			log.warn("Jagex Hiscore API returned unexpected data");
			break; // rest is other things?
		}

		// rank, level, experience
		int rank = Integer.parseInt(record.get(0));
		int level = Integer.parseInt(record.get(1));

		// items that are not skills do not have an experience parameter
		long experience = -1;
		if (record.size() == 3)
		{
			experience = Long.parseLong(record.get(2));
		}

		Skill skill = new Skill(rank, level, experience);
		hiscoreBuilder.setNextSkill(skill);
	}

	return hiscoreBuilder;
}
 

private List<CSVRecord> getRecords(String csvString) throws IOException {
	CSVParser parser = CSVParser.parse(csvString, CSVFormat.DEFAULT.withHeader());
	return parser.getRecords();
}
 

public MemTable(String resource,String charset) throws  Exception {
    InputStreamReader r=new InputStreamReader( MemTable.class.getResourceAsStream(resource),charset);
    CSVParser parser= CSVFormat.DEFAULT.parse(r);
    records=parser.getRecords();



}