Java Code Examples for org.apache.commons.csv.CSVRecord

The following are top voted examples for showing how to use org.apache.commons.csv.CSVRecord. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: tableschema-java   File: CsvDataSource.java   View source code 7 votes vote down vote up
@Override
public Iterator<String[]> iterator() throws Exception{
    Iterator<CSVRecord> iterCSVRecords = this.getCSVParser().iterator();
    
    Iterator<String[]> iterStringArrays = Iterators.transform(iterCSVRecords, (CSVRecord input) -> {
        Iterator<String> iterCols = input.iterator();
        
        List<String> cols = new ArrayList();
        while(iterCols.hasNext()){
            cols.add(iterCols.next());
        }
        
        String[] output = cols.toArray(new String[0]);
        
        return output;
    });
    
    return iterStringArrays;
}
 
Example 2
Project: buenojo   File: HangManExerciseHintLoader.java   View source code 7 votes vote down vote up
private ArrayList<Map<String, String>> parse()  {
	final ArrayList<Map<String, String>> list;
	final CSVParser parser;
	final URL resource;
	final CSVFormat csvFormat;
	final Charset charset;
	list = new ArrayList<Map<String, String>> ();
	try {
		resource = ResourceHelper.getResource(isFromGameResourceInput(),fileName);
		charset= FileEncodingDetectorHelper.guessEncodingAndGetCharset(resource);
		csvFormat = CSVFormatHelper.getDefaultCSVFormat();
		parser = CSVParser.parse(resource, charset, csvFormat);
		for (CSVRecord record : parser )
			list.add((Map<String, String>)record.toMap());
	}
	catch (IOException e) {
		log.error("Fail", e);
	}
	return list;
	
}
 
Example 3
Project: tableschema-java   File: CsvDataSource.java   View source code 6 votes vote down vote up
@Override
public void write(String outputFilePath) throws Exception{            
   try(Writer out = new BufferedWriter(new FileWriter(outputFilePath));
           CSVPrinter csvPrinter = new CSVPrinter(out, CSVFormat.RFC4180)) {
        
        if(this.getHeaders() != null){
            csvPrinter.printRecord(this.getHeaders());
        }
        
        Iterator<CSVRecord> recordIter = this.getCSVParser().iterator();
        while(recordIter.hasNext()){
            CSVRecord record = recordIter.next();
            csvPrinter.printRecord(record);
        }
        
        csvPrinter.flush();
            
   }catch(Exception e){
        throw e;
   }
}
 
Example 4
Project: careconnect-reference-implementation   File: TerminologyLoaderDao.java   View source code 6 votes vote down vote up
@Override
public void accept(CSVRecord theRecord) {
    String code = theRecord.get("LOINC_NUM");
    if (isNotBlank(code)) {
        String longCommonName = theRecord.get("LONG_COMMON_NAME");
        String shortName = theRecord.get("SHORTNAME");
        String consumerName = theRecord.get("CONSUMER_NAME");
        String display = firstNonBlank(longCommonName, shortName, consumerName);

        ConceptEntity concept = new ConceptEntity(myCodeSystemVersion, code);
        concept.setDisplay(display);

        Validate.isTrue(!myCode2Concept.containsKey(code));
        myCode2Concept.put(code, concept);
    }
}
 
Example 5
Project: buenojo   File: HangManExerciseDelimitedAreaLoader.java   View source code 6 votes vote down vote up
private ArrayList<Map<String,String>> parse()  {

		ArrayList<Map<String,String>> list;
		CSVParser parser;
		URL resource;
		CSVFormat csvFormat;
		Charset charset;
		list = new ArrayList<Map<String,String>> ();
		try {
			resource = ResourceHelper.getResource(isFromGameResourceInput(),fileName);
			charset= FileEncodingDetectorHelper.guessEncodingAndGetCharset(resource);
			csvFormat = CSVFormatHelper.getDefaultCSVFormat();
			parser = CSVParser.parse(resource, charset, csvFormat);
			for (CSVRecord record : parser )
				list.add(record.toMap());
		}
		catch (IOException e) {
			log.error("Fail", e);
		}
		return list;
		
	}
 
Example 6
Project: buenojo   File: TagPoolCSVParser.java   View source code 6 votes vote down vote up
private List <Tag>  createTags(CSVParser parser, Course course) {
	tagMap = new HashMap<>();
	ArrayList<Tag> tagList = new ArrayList<Tag>();
	for (CSVRecord csvRecord : parser) {
		String name = csvRecord.get(TagPoolColumn.TAG.ordinal()).toString();
		if (!tagMap.containsKey(name)){
			Tag tag = new Tag();
    	    tag.setName(name);
    	    tag.setCourse(course);
    
    	    tagMap.put(name, tag);
    	    
    	    tag.setNumber(tagMap.size());
    	    tagList.add(tag);
    	    
		}

	 }
	
	return tagList;
}
 
Example 7
Project: buenojo   File: SatelliteImageMetadataParser.java   View source code 6 votes vote down vote up
public List<Map<String,String>> parse() throws BuenOjoCSVParserException {
	List<Map<String,String>> list = new ArrayList<>();
	CSVParser parser = null;
	try {
		parser = CSVFormat.RFC4180.withHeader()
								.withDelimiter(',')
								.withAllowMissingColumnNames(true)
								.parse(new InputStreamReader(this.inputStreamSource.getInputStream()));
	} catch (IOException e) {
		throw new BuenOjoCSVParserException(e.getMessage());
	}
	
	for (CSVRecord record :parser) {
		Map<String,String> map = record.toMap();
		list.add(map);
	}
	
	return list;
	
}
 
Example 8
Project: buenojo   File: PhotoLocationSightPairCSVParser.java   View source code 6 votes vote down vote up
public List<PhotoLocationSightPair> parse () throws IOException, BuenOjoCSVParserException {
	CSVParser parser =  CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream()));
	
	
	List<CSVRecord> records = parser.getRecords();
	if (records.size() == 0 ) {
		throw new BuenOjoCSVParserException("El archivos de miras no contiene registros");
	}
	ArrayList<PhotoLocationSightPair> sightPairs = new ArrayList<>(records.size());
	for (CSVRecord record : records) {
		
		PhotoLocationSightPair sight = new PhotoLocationSightPair();
		sight.setNumber(new Integer(record.get(PhotoLocationSightPairCSVColumn.id)));
		sight.setSatelliteX(new Integer(record.get(PhotoLocationSightPairCSVColumn.satCol)));
		sight.setSatelliteY(new Integer(record.get(PhotoLocationSightPairCSVColumn.satRow)));
		sight.setSatelliteTolerance(new Integer(record.get(PhotoLocationSightPairCSVColumn.satTolerancia)));
		sight.setTerrainX(new Integer(record.get(PhotoLocationSightPairCSVColumn.terCol)));
		sight.setTerrainY(new Integer(record.get(PhotoLocationSightPairCSVColumn.terRow)));
		sight.setTerrainTolerance(new Integer(record.get(PhotoLocationSightPairCSVColumn.terTolerancia)));
		
		sightPairs.add(sight);
		
	}
	return sightPairs;
}
 
Example 9
Project: buenojo   File: ImageCompletionSolutionCSVParser.java   View source code 6 votes vote down vote up
public List<TagPair> parse() throws IOException {
	
	CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream()));
	
	
	ArrayList<TagPair> tagPairs = new ArrayList<>(AVG_ITEMS);
	for (CSVRecord record : parser ){
		
		TagPair pair = new TagPair();
		Integer tagSlotId = new Integer(record.get("id"));
		Integer tagNumber = new Integer(record.get("etiqueta"));
		
		pair.setTagSlotId(tagSlotId);
		Optional<Tag> optionalTag = tagList.stream().filter(isEqualToTagNumber(tagNumber)).findFirst();
		if (optionalTag.isPresent()){
			Tag tag = optionalTag.get(); 
			pair.setTag(tag);
			tagPairs.add(pair);			
		}else {
			log.debug("Attempt to get invalid tag with number: "+tagNumber);
		}
	}
	
	return tagPairs;
}
 
Example 10
Project: careconnect-reference-implementation   File: RITerminologyLoader.java   View source code 6 votes vote down vote up
@Override
public void accept(CSVRecord theRecord) {
    String code = theRecord.get("LOINC_NUM");
    if (isNotBlank(code)) {
        String longCommonName = theRecord.get("LONG_COMMON_NAME");
        String shortName = theRecord.get("SHORTNAME");
        String consumerName = theRecord.get("CONSUMER_NAME");
        String display = firstNonBlank(longCommonName, shortName, consumerName);

        ConceptEntity concept = new ConceptEntity(myCodeSystemVersion, code);
        concept.setDisplay(display);

        Validate.isTrue(!myCode2Concept.containsKey(code));
        myCode2Concept.put(code, concept);
    }
}
 
Example 11
Project: buenojo   File: TagCircleCSVParser.java   View source code 6 votes vote down vote up
public List<TagCircle> parse() throws IOException, BuenOjoCSVParserException {

		ArrayList<TagCircle> list = new ArrayList<>(MAX_CIRCLES);

		CSVParser parser = CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(false).parse(new InputStreamReader(this.inputStream));
		
		for (CSVRecord record : parser ){
			TagCircle circle = new TagCircle();
			circle.setNumber(new Integer(record.get("id")));
			circle.setX(new Integer(record.get("col")));
			circle.setY(new Integer(record.get("row")));
			circle.setRadioPx(new Float(record.get("radioPx")));
			list.add(circle);

		}
		if (list.size()>MAX_CIRCLES){
			throw new BuenOjoCSVParserException("el archivo contiene mas de "+MAX_CIRCLES+ "áreas circulares");
		}
		return list;

	}
 
Example 12
Project: buenojo   File: PhotoLocationLandscapeLevelsCSVParser.java   View source code 6 votes vote down vote up
public Integer[] parseLevels() throws BuenOjoCSVParserException, IOException{
	CSVRecord record = getRecords().get(0);
	String levels = record.get(PhotoLocationLandscapeLevelsCSVColumns.levels.ordinal());
	
	//Matcher m = Pattern.compile("(\\d*):(\\d*)").matcher(levels);
	String[] m = levels.split(":");
	if (m==null) {
		throw new BuenOjoCSVParserException("el formato del nivel es incorrecto: "+levels);
	}
	
	Integer[] n = new Integer[PhotoLocationLevelComponents.count.ordinal()];
	int lower = PhotoLocationLevelComponents.lowerLevel.ordinal();
	int higher = PhotoLocationLevelComponents.higherLevel.ordinal();
	n[lower] = new Integer(m[lower]);
	n[higher] = new Integer(m[higher]);

	return n;
	
}
 
Example 13
Project: ciesvium   File: DataSet.java   View source code 6 votes vote down vote up
/**
 * Creates a new dataset with column labels and data read from the given Reader, using a specified input format.
 *
 * @param reader       the Reader to read column labels and data from
 * @param input_format the format
 */
@SuppressWarnings("WeakerAccess")
public DataSet(final Reader reader, final CSVFormat input_format) {

    this();

    try (CSVParser parser = new CSVParser(reader, input_format.withHeader())) {

        labels.addAll(getColumnLabels(parser));

        for (final CSVRecord record : parser) {

            final List<String> items = csvRecordToList(record);
            final int size = items.size();

            // Don't add row if the line was empty.
            if (size > 1 || (size == 1 && items.get(0).length() > 0)) {
                records.add(items);
            }
        }

        reader.close();
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 14
Project: Assignment_1   File: ReadFile.java   View source code 6 votes vote down vote up
/**
 * This method reads the file from @see {@link ReadFolder} and put into an array list the data we need.
 * We use here the API commons-csv.
 * Attention : tu run with the API you need to import him into the project @see README.
 * @param folderName.
 * @exception IOException | NumberFormatException : print error reading file.
 */
public void read(String folderName) {
	try {
		Reader in = new FileReader(folderName + "/" + file.getName());
		BufferedReader br = new BufferedReader(in);
		String firstLine = br.readLine();
		if (checkTheFile(firstLine)) {
			Iterable<CSVRecord> records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(br);	
			for (CSVRecord record : records) 
				if (record.get("Type").equals("WIFI") && !record.get("FirstSeen").contains("1970")) 
					inputWifi(record, firstLine);
			in.close();
			br.close();
		}
	}
	catch(IOException | NumberFormatException ex) { // If there is an error.
		System.out.println("Error reading file : " + ex);
		System.exit(0);
	}
}
 
Example 15
Project: oscm   File: BulkUserImportReader.java   View source code 6 votes vote down vote up
@Override
public Iterator<Row> iterator() {
    return new Iterator<BulkUserImportReader.Row>() {
        Iterator<CSVRecord> i = csvParser.iterator();

        @Override
        public boolean hasNext() {
            return i.hasNext();
        }

        @Override
        public Row next() {
            CSVRecord record = i.next();
            return new Row(record);
        }

        @Override
        public void remove() {
            i.remove();
        }
    };
}
 
Example 16
Project: file-format-streaming-converter   File: XlsxToCsvConverterTest.java   View source code 6 votes vote down vote up
private void compareCSVFiles(String actualPath, String expectedPath) {
    try (CSVParser parserTranslatedFile = createCsvParser(actualPath, ",")) {
        try (CSVParser parserExpectedFile = createCsvParser(expectedPath, ",")) {
            Iterator<CSVRecord> translatedIterator = parserTranslatedFile.iterator();
            Iterator<CSVRecord> expectedIterator = parserExpectedFile.iterator();
            while (translatedIterator.hasNext() && expectedIterator.hasNext()) {
                CSVRecord translatedRecord = translatedIterator.next();
                CSVRecord expectedRecord = expectedIterator.next();
                compareCSVRecords(translatedRecord, expectedRecord);
            }
            if (translatedIterator.hasNext()) {
                fail("Actual file is longer (has more lines) than expected file");
            }
            if (expectedIterator.hasNext()) {
                fail("Expected file is longer (has more lines) than actual file");
            }
        }
    } catch (Exception e) {
        fail("Exception while iterating over files");
    }
}
 
Example 17
Project: projectBAC   File: ImportProfil.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données
 * @param parser
 * @return
 */
public int updateDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String serie = item.get(0).trim();
		String mention = item.get(1).trim();
		String specialite = item.get(2).trim();
		String section = item.get(3).trim();
		//enregistrer
		if (add(serie, mention, specialite, section)){
			res++;
		}
	}
	DBManager.quit();
	return res;
}
 
Example 18
Project: projectBAC   File: ImportCandidat.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données
 * @param parser
 * @return
 */
public int updateDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String id = item.get(0).trim();
		String serie = item.get(1).trim();
		String mention = item.get(2).trim();
		String specialite = item.get(3).trim();
		String section = item.get(4).trim();
		//enregistrer
		if (add(id, serie, mention, specialite, section)){
			res++;
		}
	}
	DBManager.quit();
	return res;
}
 
Example 19
Project: projectBAC   File: ImportNote.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données
 * @param parser
 * @return
 */
public int updateDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String id = item.get(0).trim();
		String code = item.get(1).trim();
		String note = item.get(2).trim();
		if (note.isEmpty()) {
			note=null;
		}
		//enregistrer
		if (add(id, code, note)){
			res++;
		}
	}
	DBManager.quit();
	return res;
}
 
Example 20
Project: projectBAC   File: ImportNote.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données des foreign keys
 * @param parser
 * @return
 */
public int updateFkDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String id = item.get(0).trim();
		String code = item.get(1).trim();
		String matiere = item.get(3).trim();
		//enregistrer
		if (!matiere.isEmpty()){
			if (addMatiere(id, code, matiere)){
				res++;
			}
		}
	}
	DBManager.quit();
	return res;
}
 
Example 21
Project: projectBAC   File: ImportEpreuve.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données des foreign keys
 * @param parser
 * @return
 */
public int updateFkDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String code = item.get(0).trim();
		String composition = item.get(2).trim();
		String rattrapage = item.get(3).trim();
		//enregistrer
		if (!composition.isEmpty()){
			if (addComposition(code, composition)){
				res++;
			}
		}
		if (!rattrapage.isEmpty()){
			if (addRattrapage(code, rattrapage)){
				res++;
			}
		}
	}
	DBManager.quit();
	return res;
}
 
Example 22
Project: projectBAC   File: ImportMatiere.java   View source code 6 votes vote down vote up
/**
 * sauvegarde dans la base de données
 * @param parser
 * @return
 */
public int updateDB(CSVParser parser) {
	int res = 0;
	DBManager.connect();
	for (CSVRecord item : parser) {
		String code = item.get(0).trim();
		String libelle = item.get(1).trim();
		String epreuve = item.get(2).trim();
		//enregistrer
		if (add(code, libelle, epreuve)){
			res++;
		}
	}
	DBManager.quit();
	return res;
}
 
Example 23
Project: careconnect-reference-implementation   File: TerminologyLoaderDao.java   View source code 6 votes vote down vote up
@Override
public void accept(CSVRecord theRecord) {
    String id = theRecord.get("id");
    String date = theRecord.get("effectiveTime");

    if (!myConceptIdToMostRecentDate.containsKey(id) || myConceptIdToMostRecentDate.get(id).compareTo(date) < 0) {
        boolean active = "1".equals(theRecord.get("active"));
        if (active) {
            myValidConceptIds.add(id);
        } else {
            myValidConceptIds.remove(id);
        }
        myConceptIdToMostRecentDate.put(id, date);
    }

}
 
Example 24
Project: embulk-input-parquet_hadoop   File: CSVAsJSONIterator.java   View source code 6 votes vote down vote up
@Override
public String next()
{
    CSVRecord record = inner.next();
    StringWriter json = new StringWriter();
    try {
        JsonGenerator gen = jsonFactory.createJsonGenerator(json);
        gen.writeStartObject();
        for (CSVHeaderMap.Entry entry : headerMap.entries()) {
            String name = entry.getName();
            String value = record.get(entry.getIndex());

            gen.writeFieldName(name);
            entry.getWriter().write(gen, value);
        }
        gen.writeEndObject();
        gen.close();
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
    return json.toString();
}
 
Example 25
Project: 484_P7_1-Java   File: TestSpreadsheetExtractor.java   View source code 6 votes vote down vote up
@Test
public void testFindSpreadsheetsFromCells() throws IOException {

    CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"),
            Charset.forName("utf-8"),
            CSVFormat.DEFAULT);

    List<Cell> cells = new ArrayList<>();

    for (CSVRecord record : parse) {
        cells.add(new Cell(Float.parseFloat(record.get(0)),
                Float.parseFloat(record.get(1)),
                Float.parseFloat(record.get(2)),
                Float.parseFloat(record.get(3))));
    }


    List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES);
    Collections.sort(expected, Rectangle.ILL_DEFINED_ORDER);
    List<Rectangle> foundRectangles = SpreadsheetExtractionAlgorithm.findSpreadsheetsFromCells(cells);
    Collections.sort(foundRectangles, Rectangle.ILL_DEFINED_ORDER);
    assertTrue(foundRectangles.equals(expected));
}
 
Example 26
Project: 484_P7_1-Java   File: TestBasicExtractor.java   View source code 6 votes vote down vote up
@Test
public void testNaturalOrderOfRectanglesOneMoreTime() throws IOException {
    CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestBasicExtractor-RECTANGLE_TEST_NATURAL_ORDER.csv"),
            Charset.forName("utf-8"),
            CSVFormat.DEFAULT);

    List<Rectangle> rectangles = new ArrayList<>();

    for (CSVRecord record : parse) {
        rectangles.add(new Rectangle(Float.parseFloat(record.get(0)),
                Float.parseFloat(record.get(1)),
                Float.parseFloat(record.get(2)),
                Float.parseFloat(record.get(3))));
    }


    //List<Rectangle> rectangles = Arrays.asList(RECTANGLES_TEST_NATURAL_ORDER);
    Utils.sort(rectangles, Rectangle.ILL_DEFINED_ORDER);

    for (int i = 0; i < (rectangles.size() - 1); i++) {
        Rectangle rectangle = rectangles.get(i);
        Rectangle nextRectangle = rectangles.get(i + 1);
        
        assertTrue(rectangle.compareTo(nextRectangle) < 0);
    }
}
 
Example 27
Project: hdr-histogram-plotter   File: HdrReader.java   View source code 6 votes vote down vote up
public HdrData read(String filename) throws IOException {
    Reader in = new FileReader(filename);

    HdrData ret = new HdrData();
    List<Double> value = ret.getValue();
    List<Double> percentile = ret.getPercentile();

    Iterable<CSVRecord> records = CSVFormat.RFC4180
            .withCommentMarker('#')
            .withFirstRecordAsHeader()
            .parse(in);


    for (CSVRecord record : records) {
        String valueStr = record.get(0);
        String percentileStr = record.get(1);

        logger.debug("Value: {}", valueStr);
        logger.debug("Percentile: {}", percentileStr);

        value.add(Double.parseDouble(valueStr));
        percentile.add(Double.parseDouble(percentileStr) * 100);
    }

    return ret;
}
 
Example 28
Project: careconnect-reference-implementation   File: UploadExamples.java   View source code 6 votes vote down vote up
@Override
public void accept(CSVRecord theRecord) {

    for (IBaseResource resource : resources) {
        if (resource instanceof AllergyIntolerance) {
            AllergyIntolerance allergy = (AllergyIntolerance) resource;
            if (allergy.getIdentifier().size()>0 && theRecord.get("allergy.identifier").equals(allergy.getIdentifier().get(0).getValue())) {
                AllergyIntolerance.AllergyIntoleranceReactionComponent reaction = allergy.addReaction();
                reaction.addManifestation().addCoding()
                        .setSystem(CareConnectSystem.SNOMEDCT)
                        .setDisplay(theRecord.get("reaction.manifestation.display"))
                        .setCode(theRecord.get("reaction.manifestation"));
           //     System.out.println(ctx.newJsonParser().setPrettyPrint(true).encodeResourceToString(allergy));
            }
        }
    }

}
 
Example 29
Project: dataflow-opinion-analysis   File: FileIndexerPipeline.java   View source code 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {

	String rawInput = null;
	InputContent iContent = null;
	
	try {
		rawInput = c.element();
		if (rawInput == null)
			throw new Exception("ParseCSVFile: null raw content");
		
		
		FileIndexerPipelineOptions options = c.getPipelineOptions().as(FileIndexerPipelineOptions.class);
		Integer textColumnIdx = options.getTextColumnIdx();
		Integer collectionItemIdIdx = options.getCollectionItemIdIdx();
		
		InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(rawInput,StandardCharsets.UTF_8.name()));
		
		Iterable<CSVRecord> records = CSVFormat.DEFAULT
			.withFirstRecordAsHeader()
			.parse(isr);
		
		for (CSVRecord record : records) {
			
			String text = record.get(textColumnIdx);
			String documentCollectionId = IndexerPipelineUtils.DOC_COL_ID_CSV_FILE;
			String collectionItemId = record.get(collectionItemIdIdx);
			
			InputContent ic = new InputContent(
				null /*url*/, null /*pubTime*/, null /*title*/, null /*author*/, null /*language*/, 
				text, documentCollectionId, collectionItemId, 0 /*skipIndexing*/);					
			
			c.output(ic);
		}
		

	} catch (Exception e) {
		LOG.warn(e.getMessage());
	}
}
 
Example 30
Project: dataflow-opinion-analysis   File: FileIndexerPipeline.java   View source code 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) {

	String rawInput = null;
	InputContent iContent = null;
	
	try {
		rawInput = c.element();
		if (rawInput == null)
			throw new Exception("ParseCSVLine: null raw content");
		rawInput = rawInput.trim();
		if (rawInput.isEmpty())
			throw new Exception("ParseCSVLine: empty raw content or whitespace chars only");
		
		FileIndexerPipelineOptions options = c.getPipelineOptions().as(FileIndexerPipelineOptions.class);
		Integer textColumnIdx = options.getTextColumnIdx();
		Integer collectionItemIdIdx = options.getCollectionItemIdIdx();
		
		InputStreamReader isr = new InputStreamReader(IOUtils.toInputStream(rawInput,StandardCharsets.UTF_8.name()));
		
		Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(isr);
		
		for (CSVRecord record : records) { // should only be one record, but handle multi-record case as well
			
			String text = record.get(textColumnIdx);
			String documentCollectionId = IndexerPipelineUtils.DOC_COL_ID_CSV_FILE;
			String collectionItemId = record.get(collectionItemIdIdx);
			
			InputContent ic = new InputContent(
				null /*url*/, null /*pubTime*/, null /*title*/, null /*author*/, null /*language*/, 
				text, documentCollectionId, collectionItemId, 0 /*skipIndexing*/);					
			
			c.output(ic);
		}
		

	} catch (Exception e) {
		LOG.warn(e.getMessage());
	}
}
 
Example 31
Project: mintleaf   File: CsvRowListWrapper.java   View source code 5 votes vote down vote up
private CSVRecord getNextRecord() {
    CSVRecord record;
    try {
        record = this.iterator.next();
    } catch (NoSuchElementException var2) {
        record = null;
    }
    return record;
}
 
Example 32
Project: careconnect-reference-implementation   File: TerminologyLoaderDao.java   View source code 5 votes vote down vote up
@Override
public void accept(CSVRecord theRecord) {
    String parentCode = theRecord.get("IMMEDIATE_PARENT");
    String childCode = theRecord.get("CODE");
    String childCodeText = theRecord.get("CODE_TEXT");

    if (isNotBlank(parentCode) && isNotBlank(childCode)) {
        ConceptEntity parent = getOrCreate(parentCode, "(unknown)");
        ConceptEntity child = getOrCreate(childCode, childCodeText);

        parent.addChild(child, ConceptParentChildLink.RelationshipTypeEnum.ISA);
    }
}
 
Example 33
Project: buenojo   File: PhotoLocationLoader.java   View source code 5 votes vote down vote up
public void parseAndInject(PhotoLocationExercise exercise, InputStreamSource inputStreamSource) throws BuenOjoCSVParserException, IOException{
	CSVParser parser =  CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(inputStreamSource.getInputStream()));
	List<CSVRecord> records = parser.getRecords();
	if (records.size() > 1) {
		throw new BuenOjoCSVParserException("El archivo contiene más de un ejercicio");
	}
	
	if (records.size() == 0) {
		throw new BuenOjoCSVParserException("El archivo de ejericio es inválido");
	}
	
	CSVRecord record = records.get(0);
	String name = record.get(MetadataColumns.name);
	String description = record.get(MetadataColumns.description);
	String difficulty = record.get(MetadataColumns.difficulty);
	String seconds = record.get(MetadataColumns.seconds);
	String totalScore = record.get(MetadataColumns.totalScore.ordinal());
	String imageName  = record.get(MetadataColumns.imageName.ordinal());
	
	exercise.setDescription(description);
	exercise.setName(name);
	exercise.setDifficulty(difficultyFromString(difficulty));
	exercise.setTotalTimeInSeconds(new Integer(seconds));
	exercise.setTotalScore(new Float(totalScore));
	exercise.setExtraPhotosCount(3);
	List<PhotoLocationImage> imgs = photoLocationImageRepository.findAll();
	
	Optional<PhotoLocationImage> opt = imgs.stream().filter(p ->  p.getImage().getName().equals(imageName)).collect(Collectors.toList()).stream().findFirst();
	if (!opt.isPresent()){
		throw new BuenOjoCSVParserException("la imagen '"+imageName+"' no existe en la base de datos");
	}
	PhotoLocationImage img = opt.get();
	img.setKeywords(exercise.getLandscapeKeywords());
	photoLocationImageRepository.save(img);
	exercise.setTerrainPhoto(img);
			
}
 
Example 34
Project: buenojo   File: TagPoolCSVParser.java   View source code 5 votes vote down vote up
private List<TagPool> createTagPool(CSVParser parser) throws BuenOjoCSVParserException{
	ArrayList<TagPool> list = new ArrayList<>();
	for (CSVRecord record : parser) {
		String name = record.get(TagPoolColumn.TAG.ordinal()).toString();

     	Tag tag = tagMap.get(name);
     	

     	for (int i = TagPoolColumn.SIMILAR_1.ordinal(); (i <record.size()) && i <= TagPoolColumn.SIMILAR_3.ordinal(); i++) {
     		String similarTagName = record.get(i);
     		if (similarTagName != null && !StringUtils.isAnyEmpty(similarTagName)){
     			Tag similarTag = tagMap.get(similarTagName);
     			
     			if (similarTag != null){
	     			TagPool tagPool = new TagPool();
	     			tagPool.setTag(tag);
	     			tagPool.setSimilarTag(similarTag);
	     			tagPool.setSimilarity(i);
	     			list.add(tagPool);
     			} else {
     				throw new BuenOjoCSVParserException("no se pudo obtener la etiqueta con nombre: '"+similarTagName+"'");
     			}
     		}
     	}
	}
	return list;
	
}
 
Example 35
Project: buenojo   File: PhotoLocationLandscapeLevelsCSVParser.java   View source code 5 votes vote down vote up
private List<CSVRecord> getRecords () throws BuenOjoCSVParserException, IOException{
	CSVParser parser =  CSVFormat.RFC4180.withHeader().withDelimiter(',').withAllowMissingColumnNames(true).parse(new InputStreamReader(this.inputStreamSource.getInputStream()));
	
	List<CSVRecord> records = parser.getRecords();
	if (records.size()==0) {
		throw new BuenOjoCSVParserException("El archivo no contiene palabras clave");
	}
	if (records.size()>1) {
		throw new BuenOjoCSVParserException("El archivo contiene mas de un registro de palabras clave");
	}	
	return records;
}
 
Example 36
Project: buenojo   File: PhotoLocationKeywordsParser.java   View source code 5 votes vote down vote up
private PhotoLocationKeyword parseRecord(CSVRecord record){
	PhotoLocationKeyword keyword = new PhotoLocationKeyword();
	String description = record.get(PhotoLocationKeywordColumns.description.ordinal());
	keyword.setDescription(description);
	String name = record.get(PhotoLocationKeywordColumns.name.ordinal());
	keyword.setName(name);
	return keyword;
	
}
 
Example 37
Project: ciesvium   File: DataSet.java   View source code 5 votes vote down vote up
private static List<String> csvRecordToList(final CSVRecord record) {

        final List<String> list = new ArrayList<>();
        for (final String value : record) {
            list.add(value);
        }
        return list;
    }
 
Example 38
Project: janusgraph-utils   File: DataFileLoader.java   View source code 5 votes vote down vote up
public void loadFile(String fileName, Map<String, Object> propertiesMap, WorkerPool workers) throws Exception {
    log.info("Loading " + fileName);
    // long linesCount = BatchHelper.countLines(fileName);

    this.propertiesMap = propertiesMap;

    Reader in = new FileReader(fileName);
    Iterator<CSVRecord> iter = CSVFormat.EXCEL.withHeader().parse(in).iterator();
    // long freeMemory = Runtime.getRuntime().freeMemory()/1024/1024;
    // TODO Calculate targetThreadCount using the free memory and number of threads
    // to execute
    // Max record count per thread
    startWorkers(iter, Config.getConfig().getWorkersTargetRecordCount(), workers);
}
 
Example 39
Project: JanusGraphBench   File: DataFileLoader.java   View source code 5 votes vote down vote up
private void startWorkers(Iterator<CSVRecord> iter, long targetRecordCount, WorkerPool workers)
		throws Exception {
	while (iter.hasNext()) {
		long currentRecord = 0;
		List<Map<String,String>> sub = new ArrayList<Map<String,String>>(); 
		while (iter.hasNext() && currentRecord < targetRecordCount) {
			sub.add(iter.next().toMap());
			currentRecord++;
		}
		Constructor<Worker> constructor = workerClass.getConstructor(Iterator.class, Map.class, JanusGraph.class);
		Worker worker = constructor.newInstance(sub.iterator(), propertiesMap, graph);
		workers.submit(worker);
	}
}
 
Example 40
Project: JanusGraphBench   File: DataFileLoader.java   View source code 5 votes vote down vote up
public void loadFile(String fileName, Map<String, Object> propertiesMap, WorkerPool workers) throws Exception {
		log.info("Loading " + fileName);
//		long linesCount = BatchHelper.countLines(fileName);

		this.propertiesMap = propertiesMap;

		Reader in = new FileReader(fileName);
		Iterator<CSVRecord> iter = CSVFormat.EXCEL.withHeader().parse(in).iterator();
//		long freeMemory = Runtime.getRuntime().freeMemory()/1024/1024;
		// TODO Calculate targetThreadCount using the free memory and number of threads to execute
		//Max record count per thread
		startWorkers(iter, Config.getConfig().getWorkersTargetRecordCount(), workers);
	}