Java Code Examples for org.apache.commons.csv.CSVRecord

The following examples show how to use org.apache.commons.csv.CSVRecord. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: customized-symspell   Source File: SymSpellSearchBenchMark.java    License: MIT License 7 votes vote down vote up
private List<String> readQueries(String queryFile) {
  List<String> queries = new ArrayList<>();
  try {
    URL queryResourceUrl = this.getClass().getClassLoader().getResource(queryFile);
    CSVParser qparser = CSVParser
        .parse(queryResourceUrl, Charset.forName("UTF-8"),
            CSVFormat.DEFAULT.withDelimiter(' '));
    java.util.Iterator<CSVRecord> csvIterator = qparser.iterator();
    while (csvIterator.hasNext()) {
      CSVRecord csvRecord = csvIterator.next();
      queries.add(csvRecord.get(0));
    }
  } catch (IOException ex) {
    System.err.println("Error occured " + ex);
  }
  return queries;
}
 
Example 2
Source Project: repositoryminer   Source File: ExCommentCSVReader.java    License: Apache License 2.0 6 votes vote down vote up
private void readComments() throws IOException {
	List<CSVRecord> records = readCSV(COMMENTS_HEADER, config.getCommentsCSV());

	for (CSVRecord record : records) {
		Comment comment = new Comment(Integer.parseInt(record.get(0)),
				Double.parseDouble(record.get(1).replaceAll(",", ".")),
				Double.parseDouble(record.get(2).replaceAll(",", ".")),
				Double.parseDouble(record.get(3).replaceAll(",", ".")), record.get(4), record.get(6), record.get(7));

		String filename = FilenameUtils.normalize(record.get(5), true);

		
		if (!filesMap.containsKey(filename)) {
			filesMap.put(filename, new ArrayList<Integer>());
		}

		commentsMap.put(comment.getId(), comment);
		filesMap.get(filename).add(comment.getId());
	}
}
 
Example 3
Source Project: vividus   Source File: CsvTableTransformer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String transform(String tableAsString, TableParsers tableParsers, TableProperties properties)
{
    checkTableEmptiness(tableAsString);
    String csvPath = ExtendedTableTransformer.getMandatoryNonBlankProperty(properties, "csvPath");
    try
    {
        List<CSVRecord> result = csvReader.readCsvFile(findResource(getClass(), csvPath));
        return ExamplesTableProcessor.buildExamplesTable(result.get(0).toMap().keySet(), extractValues(result),
                properties, true);
    }
    catch (IOException e)
    {
        throw new UncheckedIOException("Problem during CSV file reading", e);
    }
}
 
Example 4
Source Project: RepoSense   Source File: GroupConfigCsvParser.java    License: MIT License 6 votes vote down vote up
/**
 * Processes the csv file line by line and adds created {@code Group} into {@code results}.
 */
@Override
protected void processLine(List<GroupConfiguration> results, CSVRecord record) throws InvalidLocationException {
    String location = get(record, LOCATION_POSITION);
    String groupName = get(record, GROUP_NAME_POSITION);
    List<String> globList = getAsList(record, FILES_GLOB_POSITION);

    GroupConfiguration groupConfig = null;
    groupConfig = findMatchingGroupConfiguration(results, location);

    FileType group = new FileType(groupName, globList);
    if (groupConfig.containsGroup(group)) {
        logger.warning(String.format(
                "Skipping group as %s has already been specified for the repository %s",
                group.toString(), groupConfig.getLocation()));
        return;
    }

    groupConfig.addGroup(group);
}
 
Example 5
public DatasetsRootElement transform(@NotNull final CSVParser csvRecords) {
	checkNotNull(csvRecords);

	Set<String> headings = csvRecords.getHeaderMap().keySet();

	DatasetsRootElement datasetsRootElement = new DatasetsRootElement();

	for (CSVRecord record : csvRecords) {
		DataSet dataSet = new DataSet();
		List<Setting> settings = dataSet.getSettings();

		for (String heading : headings) {
			Setting setting = new Setting();
			setting.setName(heading);
			setting.setValue(record.get(heading));

			settings.add(setting);
		}

		datasetsRootElement.getDataSets().getDataSets().add(dataSet);
	}

	return datasetsRootElement;
}
 
Example 6
Source Project: quaerite   Source File: ElevateQueryComparer.java    License: Apache License 2.0 6 votes vote down vote up
private static QuerySet loadQueries(Path file) throws Exception {
    QuerySet querySet = new QuerySet();
    Matcher uc = Pattern.compile("[A-Z]").matcher("");
    try (InputStream is = Files.newInputStream(file)) {
        try (Reader reader = new InputStreamReader(new BOMInputStream(is), "UTF-8")) {
            Iterable<CSVRecord> records = CSVFormat.EXCEL
                    .withFirstRecordAsHeader().parse(reader);
            for (CSVRecord record : records) {
                String q = record.get("query");
                Integer c = Integer.parseInt(record.get("count"));
                if (querySet.queries.containsKey(q)) {
                    LOG.warn("duplicate queries?! >" + q + "<");
                }

                querySet.set(q, c);
            }
        }
    }
    LOG.info("loaded " + querySet.queries.size() + " queries");
    return querySet;
}
 
Example 7
Source Project: development   Source File: BulkUserImportReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Iterator<Row> iterator() {
    return new Iterator<BulkUserImportReader.Row>() {
        Iterator<CSVRecord> i = csvParser.iterator();

        @Override
        public boolean hasNext() {
            return i.hasNext();
        }

        @Override
        public Row next() {
            CSVRecord record = i.next();
            return new Row(record);
        }

        @Override
        public void remove() {
            i.remove();
        }
    };
}
 
Example 8
Source Project: cia   Source File: WorldbankIndicatorApiImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read csv content.
 *
 * @param is
 *            the is
 * @return the list
 * @throws IOException
 *             Signals that an I/O exception has occurred.
 */
private static List<String> readCsvContent(final InputStream is) throws IOException {
	final BufferedReader reader = new BufferedReader(new InputStreamReader(is,StandardCharsets.UTF_8));
	for (int i = 0; i < IGNORE_TOP_HEADERS_LINE; i++) {
		final String ignoreFirstLinesWithHeaders = reader.readLine();
	}

	final CSVParser parser = CSVParser.parse(reader, CSVFormat.EXCEL.withHeader().withDelimiter(','));
	final List<CSVRecord> records = parser.getRecords();
	records.remove(0);

	final List<String> list = new ArrayList<>();

	for (final CSVRecord csvRecord : records) {
		list.add(csvRecord.get("Indicator Code"));
	}

	return list;
}
 
Example 9
Source Project: LicenseScout   Source File: LicenseStoreData.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads license name mappings from a CSV file.
 * 
 * @param inputStream an input stream to read the file contents from
 * @param log the logger
 * @throws IOException
 */
public void readNameMappings(final InputStream inputStream, final ILSLog log) throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(',').withCommentMarker('#');
    try (final BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"))) {
        final CSVParser csvParser = csvFormat.parse(br);
        for (final CSVRecord record : csvParser) {
            final String mappedName = record.get(0).trim();
            final int numLicenseIdentifiers = record.size() - 1;
            final List<License> licenses = new ArrayList<>();
            for (int i = 0; i < numLicenseIdentifiers; i++) {
                final String licenseIdentifier = record.get(i + 1).trim();
                final License license = getLicenseBySpdxIdentifier(licenseIdentifier);
                if (license != null) {
                    licenses.add(license);
                } else {
                    log.info("readNameMappings: license identifier not found: " + licenseIdentifier);
                }
            }
            nameMappings.put(mappedName, licenses);
        }
    }
}
 
Example 10
Source Project: core   Source File: Loc.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static List<Loc> readLocs(String fileName) {
	List<Loc> locs = new ArrayList<Loc>();
	
	try {
		Reader in = new BufferedReader(new InputStreamReader(
				new FileInputStream(fileName), "UTF-8"));
		CSVFormat formatter = 
				CSVFormat.DEFAULT.withHeader().withCommentMarker('-');
		
		// Parse the file
		Iterable<CSVRecord> records = formatter.parse(in);
		Iterator<CSVRecord> iterator = records.iterator();
		while (iterator.hasNext()) {
			// Determine the record to process
			CSVRecord record = iterator.next();
			Loc loc = getLoc(record);
			if (loc.accuracy < MAX_ALLOWED_ACCURACY)
			locs.add(loc);
		}
	} catch (Exception e) {
		e.printStackTrace();
	}
	
	return locs;
}
 
Example 11
Source Project: maestro-java   Source File: RouterLinkReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reader of csv file
 * @param reader reader
 * @return readed data
 * @throws IOException implementation specific
 */
@Override
protected RouterLinkDataSet readReader(Reader reader) throws IOException {
    Iterable<CSVRecord> records = CSVFormat.RFC4180
            .withCommentMarker('#')
            .withFirstRecordAsHeader()
            .withRecordSeparator(';')
            .withQuote('"')
            .withQuoteMode(QuoteMode.NON_NUMERIC)
            .parse(reader);



    for (CSVRecord record : records) {
        try {
            routerLinkProcessor.process(record.get(0), record.get(1), record.get(2), record.get(3), record.get(4),
                    record.get(5), record.get(6), record.get(7), record.get(8), record.get(9), record.get(10),
                    record.get(11), record.get(12), record.get(13));
        } catch (Throwable t) {
            logger.warn("Unable to parse record: {}", t.getMessage(), t);
        }
    }

    return routerLinkProcessor.getRouterLinkDataSet();
}
 
Example 12
Source Project: rs-api   Source File: Hiscores.java    License: ISC License 6 votes vote down vote up
/**
 * Gets a {@link Player} based on their display name.
 * @param displayName The player's display name.
 * @param table The table of {@link Hiscores}.
 * @return An {@link Optional} containing the {@link Player}, or {@link Optional#empty()} if no {@link Player} was found with that name.
 * @throws IOException If an I/O error occurs.
 * @see <a href="https://runescape.wiki/w/Application_programming_interface#Hiscores_Lite">Hiscores Lite</a>
 * @see <a href="https://runescape.wiki/w/Application_programming_interface#Ironman_Lite">Ironman Hiscores Lite</a>
 * @see <a href="https://runescape.wiki/w/Application_programming_interface#Hardcore_Ironman_Lite">Hardcore Ironman Hiscores Lite</a>
 */
public Optional<Player> playerInformation(String displayName, HiscoreTable table) throws IOException {
	Preconditions.checkNotNull(displayName);
	Preconditions.checkNotNull(table);

	String escapedName = NAME_SPACER.matcher(displayName).replaceAll("+");
	String url = String.format(PLAYER_INFORMATION_URL_FORMAT, table.getName(), escapedName);
	ImmutableList<CSVRecord> records = client.fromCSV(url);

	ImmutableList<String> skillNames = table.getSkillNames();
	ImmutableList<String> activityNames = table.getActivityNames();

	if (records.size() >= (skillNames.size() + activityNames.size())) {
		ImmutableMap<String, Skill> skills = readSkills(records, skillNames);
		ImmutableMap<String, HiscoreActivity> activities = readActivities(records, skillNames, activityNames);
		return Optional.of(new Player(skills, activities));
	} else {
		return Optional.empty();
	}
}
 
Example 13
Source Project: maestro-java   Source File: LegacyRateReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RateData readReader(Reader reader) throws IOException {
    Iterable<CSVRecord> records = CSVFormat.RFC4180
            .withCommentMarker('#')
            .withFirstRecordAsHeader()
            .withRecordSeparator(',')
            .withQuote('"')
            .withQuoteMode(QuoteMode.NON_NUMERIC)
            .parse(reader);

    for (CSVRecord record : records) {
        try {
            processor.process(record.get(0), record.get(1));
        } catch (Exception e) {
            logger.warn("Unable to parse record: {}", e.getMessage(), e);
        }
    }

    return processor.getRateData();
}
 
Example 14
Source Project: connector-sdk   Source File: CSVFileManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvFileManagerSkipHeaderFalseCsvFileWithOnlyHeaderNoRecord() throws IOException {
  File tmpfile = temporaryFolder.newFile("SkipHeaderFalseCsvFileWithOnlyHeaderNoRecord.csv");
  createFile(tmpfile, UTF_8, testCSVSingleWithOnlyHeaderNoRecords);
  Properties config = new Properties();
  config.put(CSVFileManager.FILEPATH, tmpfile.getAbsolutePath());
  config.put(UrlBuilder.CONFIG_COLUMNS, "term");
  config.put(CSVFileManager.UNIQUE_KEY_COLUMNS, "term");
  config.put(CONTENT_TITLE, "term");
  config.put(CONTENT_HIGH, "term");
  config.put(CSVFileManager.SKIP_HEADER, "false");
  config.put(CSVFileManager.CSVCOLUMNS, "term,definition");
  setupConfig.initConfig(config);

  CSVFileManager csvFileManager = CSVFileManager.fromConfiguration();
  CloseableIterable<CSVRecord> csvFile = csvFileManager.getCSVFile();
  CSVRecord csvRecord = getOnlyElement(csvFile);

  Item item = csvFileManager.createItem(csvRecord);
  assertEquals("term", item.getName());
  assertEquals(null, item.getAcl());
  assertEquals("term", item.getMetadata().getSourceRepositoryUrl());
}
 
Example 15
Source Project: connector-sdk   Source File: CSVFileManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvFileManagerCreateItemWithMultiKey() throws IOException {
  File tmpfile = temporaryFolder.newFile("testCreateItemWithMultiKey.csv");
  createFile(tmpfile, UTF_8, testCSVSingle);
  Properties config = new Properties();
  config.put(CSVFileManager.FILEPATH, tmpfile.getAbsolutePath());
  config.put(UrlBuilder.CONFIG_COLUMNS, "term");
  config.put(CSVFileManager.UNIQUE_KEY_COLUMNS, "term, definition");
  config.put(CONTENT_TITLE, "term");
  config.put(CONTENT_HIGH, "term,definition");
  setupConfig.initConfig(config);

  CSVFileManager csvFileManager = CSVFileManager.fromConfiguration();
  CloseableIterable<CSVRecord> csvFile = csvFileManager.getCSVFile();
  CSVRecord csvRecord = getOnlyElement(csvFile);

  Item item = csvFileManager.createItem(csvRecord);
  assertEquals("moma search||Google internal search", item.getName());
  assertEquals(null, item.getAcl());
  assertEquals("moma search", item.getMetadata().getSourceRepositoryUrl());
}
 
Example 16
Source Project: rival   Source File: UIPParser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public DataModelIF<Long, Long> parseData(final File f) throws IOException {
    DataModelIF<Long, Long> dataset = new DataModel<>();
    Reader in = new InputStreamReader(new FileInputStream(f), "UTF-8");

    Iterable<CSVRecord> records;
    if (isHasHeader()) {
        records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).withHeader().parse(in);
    } else {
        records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).parse(in);
    }
    for (CSVRecord record : records) {
        long userID = Long.parseLong(record.get(getUserTok()));
        long itemID = Long.parseLong(record.get(getItemTok()));
        double preference = Double.parseDouble(record.get(getPrefTok()));
        dataset.addPreference(userID, itemID, preference);
    }
    in.close();
    return dataset;
}
 
Example 17
Source Project: connector-sdk   Source File: CSVFileManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvFileManagerEmptyDateTimeFields() throws IOException {
  File tmpfile = temporaryFolder.newFile("testEmptyDateTimeFields.csv");
  createFile(tmpfile, UTF_8, testCSVSingleWithStructuredDataEmptyDateTime);
  Properties config = new Properties();
  config.put(CSVFileManager.FILEPATH, tmpfile.getAbsolutePath());
  config.put(UrlBuilder.CONFIG_COLUMNS, "term,author");
  config.put(CONTENT_TITLE, "term");
  config.put(CSVFileManager.UNIQUE_KEY_COLUMNS, "term");
  setupConfig.initConfig(config);

  CSVFileManager csvFileManager = CSVFileManager.fromConfiguration();
  CloseableIterable<CSVRecord> csvFile = csvFileManager.getCSVFile();
  CSVRecord csvRecord = getOnlyElement(csvFile);

  Multimap<String, Object> multimap = csvFileManager.generateMultiMap(csvRecord);
  assertEquals(Collections.emptyList(), multimap.get("updated"));
}
 
Example 18
Source Project: connector-sdk   Source File: CSVFileManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvFileManagerEncodingUtf8() throws IOException {
  String utf8euro = "\u20ac";
  File tmpfile = temporaryFolder.newFile("testEncoding.csv");
  createFile(tmpfile, UTF_8,
      "term, definition",
      "euro, symbol=" + utf8euro);

  Properties config = new Properties();
  config.put(CSVFileManager.FILEPATH, tmpfile.getAbsolutePath());
  config.put(CSVFileManager.FILE_ENCODING, UTF_8.name());
  config.put(UrlBuilder.CONFIG_COLUMNS, "term");
  config.put(CONTENT_TITLE, "term");
  config.put(CSVFileManager.UNIQUE_KEY_COLUMNS, "term");
  setupConfig.initConfig(config);
  CSVFileManager csvFileManager = CSVFileManager.fromConfiguration();
  CSVRecord csvRecord = getOnlyElement(csvFileManager.getCSVFile());

  assertEquals("symbol=" + utf8euro,  csvRecord.get("definition"));
}
 
Example 19
Source Project: connector-sdk   Source File: CSVFileManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCsvFileManagerEncodingMismatch() throws IOException {
  String utf8euro = "\u20ac";
  File tmpfile = temporaryFolder.newFile("testEncoding.csv");
  createFile(tmpfile, Charset.forName("Cp1252"),
      "term, definition",
      "euro, symbol=" + utf8euro);

  Properties config = new Properties();
  config.put(CSVFileManager.FILEPATH, tmpfile.getAbsolutePath());
  config.put(CSVFileManager.FILE_ENCODING, UTF_8.name()); // Read using a different encoding
  config.put(UrlBuilder.CONFIG_COLUMNS, "term");
  config.put(CONTENT_TITLE, "term");
  config.put(CSVFileManager.UNIQUE_KEY_COLUMNS, "term");
  setupConfig.initConfig(config);
  CSVFileManager csvFileManager = CSVFileManager.fromConfiguration();
  CSVRecord csvRecord = getOnlyElement(csvFileManager.getCSVFile());

  assertNotEquals("symbol=" + utf8euro, csvRecord.get("definition"));
  assertThat(csvRecord.get("definition"), endsWith(UTF_8.newDecoder().replacement()));
}
 
Example 20
public SerieDto build(CSVRecord csvRecord) throws ImportException {
    try {
        return new SerieDto(
                formatLong(getId(csvRecord)),
                formatLong(csvRecord.get("movie_id")),
                formatLong(csvRecord.get("review_id")),
                csvRecord.get("title"),
                formatDate(csvRecord.get("review_date")),
                csvRecord.get("review"),
                formatFloat(csvRecord.get("rating")),
                getMaxRating(csvRecord),
                csvRecord.get("poster_path"),
                csvRecord.get("overview"),
                formatInteger(csvRecord.get("year")),
                csvRecord.get("release_date")
        );
    } catch (ParseException e) {
        throw new ImportException(context.getString(R.string.import_parsing_line_error_toast, csvRecord.get("title")), e);
    }
}
 
Example 21
Source Project: tabula-java   Source File: TestSpreadsheetExtractor.java    License: MIT License 6 votes vote down vote up
@Test
public void testFindSpreadsheetsFromCells() throws IOException {

    CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"),
            Charset.forName("utf-8"),
            CSVFormat.DEFAULT);

    List<Cell> cells = new ArrayList<>();

    for (CSVRecord record : parse) {
        cells.add(new Cell(Float.parseFloat(record.get(0)),
                Float.parseFloat(record.get(1)),
                Float.parseFloat(record.get(2)),
                Float.parseFloat(record.get(3))));
    }


    List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES);
    Collections.sort(expected, Rectangle.ILL_DEFINED_ORDER);
    List<Rectangle> foundRectangles = SpreadsheetExtractionAlgorithm.findSpreadsheetsFromCells(cells);
    Collections.sort(foundRectangles, Rectangle.ILL_DEFINED_ORDER);
    assertTrue(foundRectangles.equals(expected));
}
 
Example 22
/**
 * Parses the input payload in CSV format to a list of Objects
 * @param csvInput, the input received from the request in CSV format
 * @param schema, the data schema retrieved from environment variable
 * @return List of Objects, where each Object correspond to one feature of the input data
 * @throws IOException, if there is an exception thrown in the try-with-resources block
 */
public List<Object> convertCsvToObjectList(final String csvInput, final DataSchema schema) throws IOException {
    try (final StringReader sr = new StringReader(csvInput)) {
        final List<Object> valueList = Lists.newArrayList();
        final CSVParser parser = CSVFormat.DEFAULT.parse(sr);
        // We don not supporting multiple CSV lines as input currently
        final CSVRecord record = parser.getRecords().get(0);
        final int inputLength = schema.getInput().size();
        for (int idx = 0; idx < inputLength; ++idx) {
            ColumnSchema sc = schema.getInput().get(idx);
            // For CSV input, each value is treated as an individual feature by default
            valueList.add(this.convertInputDataToJavaType(sc.getType(), DataStructureType.BASIC, record.get(idx)));
        }
        return valueList;
    }
}
 
Example 23
public static void load(File location, AbstractDataModel sAbstractData) {
    CSVHParser parser = FileUtils.getCSVHParser(location);
    if (parser != null) {
        for (CSVRecord crecord : parser.getRecords()) {
            List record = sAbstractData.getNewRecord();
            for (int i = 0; i < crecord.size(); i++) {
                record.add(crecord.get(i));
            }
            sAbstractData.addRecord(record);
        }
    }
}
 
Example 24
Source Project: RepoSense   Source File: CsvParser.java    License: MIT License 5 votes vote down vote up
/**
 * Returns the values in {@code record} as a list with the {@link CsvParser#OVERRIDE_KEYWORD} prefix removed.
 * Returns an empty list if {@code record} at {@code colNum} is empty.
 */
protected List<String> getAsListWithoutOverridePrefix(final CSVRecord record, int colNum) {
    List<String> data = getAsList(record, colNum);
    if (isElementOverridingStandaloneConfig(record, colNum)) {
        data.set(0, data.get(0).replaceFirst(OVERRIDE_KEYWORD, ""));
        data.removeIf(String::isEmpty);
    }
    return data;
}
 
Example 25
Source Project: marklogic-contentpump   Source File: DelimitedTextReader.java    License: Apache License 2.0 5 votes vote down vote up
protected String[] getLine(CSVRecord record)
        throws IOException {
    Iterator<String> recordIterator = record.iterator();
    int recordSize = record.size();
    String[] values = new String[recordSize];
    for (int i = 0; i < recordSize; i++) {
        if (recordIterator.hasNext()) {
            values[i] = (String)recordIterator.next();
        } else {
            throw new IOException("Record size doesn't match the real size");
        }
    }
    return values;
}
 
Example 26
@Override
protected void importDatasource(Datasource datasource, List<String> geographyScope, List<String> temporalScope, List<String> datasourceLocation) throws Exception {

    CSVExtractor subjectLabelExtractor = new CSVExtractor(0);
    List<TimedValueExtractor> extractors = getExtractors(subjectLabelExtractor);

    String line = null;
    BufferedReader br = new BufferedReader(new FileReader(
            downloadUtils.fetchFile(new URL(DATAFILE), getProvider().getLabel(), DATAFILE_SUFFIX)));
    List<TimedValue> timedValueBuffer = new ArrayList<>();
    while ((line = br.readLine())!=null) {
        CSVParser parser = CSVParser.parse(line, CSVFormat.DEFAULT);
        List<CSVRecord> records = parser.getRecords();
        subjectLabelExtractor.setCsvRecord(records.get(0));
        for(TimedValueExtractor extractor: extractors){
            ((CSVExtractor)extractor.getValueExtractor()).setCsvRecord(records.get(0));
            try {
                timedValueBuffer.add(extractor.extract());
            }catch (UnknownSubjectLabelException e){
                // No reason to panic even if Subject does not exist and no reason to run the rest of the extractors
                // Keep Calm and Break
                break;
            }
        }
    }
    br.close();
    saveAndClearTimedValueBuffer(timedValueBuffer);
}
 
Example 27
Source Project: easy-sync   Source File: MemTable.java    License: Apache License 2.0 5 votes vote down vote up
public CSVRecord find(String name ,String value){
    Map<String,CSVRecord> m=map.get(name);
    if(m!=null){
        return  m.get(value);
    }
    for(CSVRecord record :records){
        if(record.get(name).equals(value)){
            return record;
        }

    }
    return null;
}
 
Example 28
Source Project: geowave   Source File: SceneFeatureIterator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SimpleFeature apply(final CSVRecord input) {
  if (input == null) {
    return null;
  }
  final String entityId = input.get("entityId");
  final double cloudCover = Double.parseDouble(input.get("cloudCover"));
  final String processingLevel = input.get("processingLevel");
  final int path = Integer.parseInt(input.get("path"));
  final int row = Integer.parseInt(input.get("row"));
  final String downloadUrl = input.get("download_url");

  final MultiPolygon shape = wrs2Geometry.getGeometry(path, row);
  featureBuilder.add(shape);
  featureBuilder.add(entityId);
  Date aquisitionDate;
  final SimpleDateFormat sdf = new SimpleDateFormat(AQUISITION_DATE_FORMAT);
  try {
    aquisitionDate = sdf.parse(input.get("acquisitionDate"));
    featureBuilder.add(aquisitionDate);
  } catch (final ParseException e) {
    LOGGER.warn("Unable to parse aquisition date", e);

    featureBuilder.add(null);
  }

  featureBuilder.add(cloudCover);
  featureBuilder.add(processingLevel);
  featureBuilder.add(path);
  featureBuilder.add(row);
  featureBuilder.add(downloadUrl);
  return featureBuilder.buildFeature(entityId);
}
 
Example 29
Source Project: quaerite   Source File: QueryLoader.java    License: Apache License 2.0 5 votes vote down vote up
public static void loadJudgments(ExperimentDB experimentDB, Path file,
                                 boolean freshStart) throws IOException, SQLException {
    if (freshStart) {
        experimentDB.clearJudgments();
    }

    Map<String, Judgments> judgmentsMap = null;
    try (InputStream is = Files.newInputStream(file)) {
        try (Reader reader = new InputStreamReader(new BOMInputStream(is), "UTF-8")) {
            Iterable<CSVRecord> records = CSVFormat.EXCEL
                    .withFirstRecordAsHeader().parse(reader);
            boolean hasJudgments = (((CSVParser) records)).getHeaderMap().containsKey(
                    DOCUMENT_ID) ? true : false;
            boolean hasQuerySet = (((CSVParser) records).getHeaderMap().containsKey(
                    QUERY_SET)) ? true : false;
            boolean hasCount = (((CSVParser) records).getHeaderMap().containsKey(
                    COUNT)) ? true : false;
            boolean hasQueryId = (((CSVParser) records).getHeaderMap().containsKey(
                    QUERY_ID)) ? true : false;
            Set<String> queryStringNames = getQueryStringNames(((CSVParser) records)
                    .getHeaderMap().keySet());
            if (hasQueryId) {
                judgmentsMap = loadJudgmentsWithId(hasJudgments, hasQuerySet, hasCount,
                        queryStringNames, records);
            } else {
                judgmentsMap = loadJudmentsWithoutId(hasJudgments, hasQuerySet,
                        hasCount, queryStringNames, records);
            }
        }
    }
    for (Judgments judgments : judgmentsMap.values()) {
        experimentDB.addJudgment(judgments);
    }
}
 
Example 30
@VisibleForTesting
TumorLocationCurator(@NotNull InputStream mappingInputStream) throws IOException {
    CSVParser parser = CSVParser.parse(mappingInputStream, Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader());
    for (CSVRecord record : parser) {
        String searchTerm = record.get("searchTerm");
        String primaryTumorLocation = record.get("primaryTumorLocation");
        String subType = record.get("subType");
        tumorLocationMap.put(searchTerm.toLowerCase(),
                ImmutableCuratedTumorLocation.of(Utils.capitalize(primaryTumorLocation), Utils.capitalize(subType), searchTerm));
    }
    // Need to create a copy of the key set so that we can remove elements from it without affecting the curation.
    unusedSearchTerms = Sets.newHashSet(tumorLocationMap.keySet());
}