Java Code Examples for org.apache.commons.csv.CSVRecord#get()

public WishlistDataDto build(CSVRecord csvRecord) throws ImportException {
    try {
        return new WishlistDataDto(
                formatLong(getId(csvRecord)),
                formatInteger(csvRecord.get("movie_id")),
                csvRecord.get("title"),
                csvRecord.get("poster_path"),
                csvRecord.get("overview"),
                formatInteger(csvRecord.get("firstYear")),
                csvRecord.get("release_date"),
                getWishlistItemTypeFromString(csvRecord.get("wishlistItemType"))
        );
    } catch (ParseException e) {
        throw new ImportException(context.getString(R.string.import_parsing_line_error_toast, csvRecord.get("title")), e);
    }
}
 

@Override
public void accept(CSVRecord theRecord) {
    String id = theRecord.get("id");
    String date = theRecord.get("effectiveTime");

    if (!myConceptIdToMostRecentDate.containsKey(id) || myConceptIdToMostRecentDate.get(id).compareTo(date) < 0) {
        boolean active = "1".equals(theRecord.get("active"));
        if (active) {
            myValidConceptIds.add(id);
        } else {
            myValidConceptIds.remove(id);
        }
        myConceptIdToMostRecentDate.put(id, date);
    }

}
 

public void getNext(CAS cas) throws IOException, CollectionException {
	currentRecord++;
	JCas jcas;
	try {
		jcas = cas.getJCas();
	} catch (CASException e) {
		throw new CollectionException(e);
	}

	// Set document data
	CSVRecord record = recordsIterator.next();
	String docId = record.get(0); // external document id from CSV file

	jcas.setDocumentText(cleanBodyText(record.get(1)));
	jcas.setDocumentLanguage(record.size() > 3 ? record.get(3) : defaultLanguage);

	// Set metadata
	Metadata metaCas = new Metadata(jcas);
	metaCas.setDocId(docId);
	metaCas.setTimestamp(record.get(2));
	metaCas.addToIndexes();

	// metadata
	// is assumed to be provided from external prcessing in a separate file

}
 

public KinoDto build(CSVRecord csvRecord) throws ImportException {
    try {
        return new KinoDto(
                formatLong(getId(csvRecord)),
                formatLong(csvRecord.get("movie_id")),
                csvRecord.get("title"),
                formatDate(csvRecord.get("review_date")),
                csvRecord.get("review"),
                formatFloat(csvRecord.get("rating")),
                getMaxRating(csvRecord),
                csvRecord.get("poster_path"),
                csvRecord.get("overview"),
                formatInteger(csvRecord.get("year")),
                csvRecord.get("release_date")
        );
    } catch (ParseException e) {
        throw new ImportException(context.getString(R.string.import_parsing_line_error_toast, csvRecord.get("title")), e);
    }
}
 

private static void appendToList(Map<String, String> tpCandidates, CSVRecord csvRecord) {
	String targetWord = csvRecord.get(0);
	String[] variants = csvRecord.get(2).split(",");
	for (String variant : variants) {
		tpCandidates.put(variant, targetWord);
	}
}
 

private String[] toArray(CSVRecord record) {
  String[] array = (record == null) ? null : new String[record.size()];
  if (array != null) {
    for (int i = 0; i < record.size(); i++) {
      array[i] = record.get(i);
    }
  }
  return array;
}
 

public TestResults parse(File f) throws Exception
  {
    TestResults tr = new TestResults("CxIAST", true, TestResults.ToolType.IAST);

    java.io.Reader inReader = new java.io.FileReader(f);
    Iterable<CSVRecord> records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(inReader);
    for (CSVRecord record : records) {
      String checkerKey = record.get("Vulnerability Type");
      String url = record.get("URL");
//      System.out.println("URL = "+url); //For debugging YE

      TestCaseResult tcr = new TestCaseResult();
      tcr.setCategory(checkerKey);
      tcr.setCWE(cweLookup(checkerKey));
      Pattern testCasePattern = Pattern.compile("BenchmarkTest[0-9]{5}");
      Matcher testCaseMatcher = testCasePattern.matcher(url);
      if(testCaseMatcher.find()) {
        String testCase = testCaseMatcher.group(0);
//      System.out.println("testCase = "+testCase+" Test Num = "+testCase.substring(testCase.length()-5, testCase.length())); // For debugging YE
        tcr.setTestCaseName(testCase);
        //"BenchmarkTest00000" - "BenchmarkTest99999"
            tcr.setNumber(Integer.parseInt(testCase.substring(testCase.length()-5, testCase.length())));
            if (tcr.getCWE() != 0)
            {
                 tr.put(tcr);
            }
//      System.out.println(testCase+" "+tcr.getCWE()+" "+tcr.getCategory()); // For debugging YE
      }
    }
    tr.setTime("100");
    return tr;
  }
 

public void parseCSV(String csv) {
	try {
		Reader in = new StringReader(csv);
		CSVFormat format = CSVFormat.EXCEL.withHeader();
		CSVParser parser = format.parse(in);
		Set<String> colNames = parser.getHeaderMap().keySet();
		this.colNames.addAll(colNames);
		this.firstColIsIndex = true;
		for (CSVRecord record : parser) {
			List<String> row = new ArrayList<>();
			for (int i = 0; i<record.size(); i++) {
				String v = record.get(i);
				boolean isInt = false;
				try {
					Integer.parseInt(v);
					isInt = true;
				}
				catch (NumberFormatException nfe) {
					isInt = false;
				}
				if ( !isInt && !NumberUtils.isDigits(v) && NumberUtils.isCreatable(v) ) {
					v = String.format("%.4f",Precision.round(Double.valueOf(v), 4));
				}
				else {
					v = abbrevString(v, 25);
				}
				row.add(v);
			}
			rows.add(row);
		}
	}
	catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 

@VisibleForTesting
TumorLocationCurator(@NotNull InputStream mappingInputStream) throws IOException {
    CSVParser parser = CSVParser.parse(mappingInputStream, Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader());
    for (CSVRecord record : parser) {
        String searchTerm = record.get("searchTerm");
        String primaryTumorLocation = record.get("primaryTumorLocation");
        String subType = record.get("subType");
        tumorLocationMap.put(searchTerm.toLowerCase(),
                ImmutableCuratedTumorLocation.of(Utils.capitalize(primaryTumorLocation), Utils.capitalize(subType), searchTerm));
    }
    // Need to create a copy of the key set so that we can remove elements from it without affecting the curation.
    unusedSearchTerms = Sets.newHashSet(tumorLocationMap.keySet());
}
 

private List<String[]> getRecords(String content, CSVFormat format) throws IOException {
    List<String[]> records = new ArrayList<>();
    CSVParser parser = CSVParser.parse(content, format);
    for (CSVRecord record : parser.getRecords()) {
        String[] line = new String[record.size()];
        for (int i = 0; i < line.length; i++) {
            line[i] = record.get(i);
        }
        records.add(line);
    }
    return records;
}
 

@Override
public GenericRow extract(CSVRecord from, GenericRow to) {
  for (String fieldName : _fields) {
    String value = from.isSet(fieldName) ? from.get(fieldName) : null;
    if (value == null || StringUtils.isEmpty(value)) {
      to.putValue(fieldName, null);
      // NOTE about CSV behavior for empty string e.g. foo,bar,,zoo or foo,bar,"",zoo. These both are equivalent to a CSVParser
      // Empty string has to be treated as null, as this could be a column of any data type.
      // This could be incorrect for STRING dataType, as "" could be a legit entry, different than null.
    } else {
      String[] stringValues = StringUtils.split(value, _multiValueDelimiter);
      int numValues = stringValues.length;
      // NOTE about CSV behavior for multi value column - cannot distinguish between multi value column with just 1 entry vs single value
      // MV column with single value will be treated as single value until DataTypeTransformer.
      // Transform functions on such columns will have to handle the special case.
      if (numValues > 1) {
        Object[] array = new Object[numValues];
        int index = 0;
        for (String stringValue : stringValues) {
            array[index++] = stringValue;
        }
        to.putValue(fieldName, array);
      } else {
        to.putValue(fieldName, value);
      }
    }
  }
  return to;
}
 

private static Map<String, Judgments> loadJudgmentsWithId(
        boolean hasJudgments, boolean hasQuerySet, boolean hasCount,
        Set<String> queryStringNames, Iterable<CSVRecord> records) throws SQLException {

    //queryId, judgments
    Map<String, Judgments> judgmentsMap = new HashMap<>();
    for (CSVRecord record : records) {
        String querySet = (hasQuerySet) ? record.get(QUERY_SET) : QueryInfo.DEFAULT_QUERY_SET;
        QueryStrings queryStrings = getQueryStrings(queryStringNames, record);
        int queryCount = (hasCount) ? Integer.parseInt(record.get(COUNT)) : 1;

        String queryId = record.get(QUERY_ID);
        if (StringUtils.isBlank(queryId)) {
            throw new IllegalArgumentException("If the csv has a '" + QUERY_ID + "' column, " +
                    "there must be a non-empty value for every row");
        }
        QueryInfo newQueryInfo = new QueryInfo(queryId, querySet, queryStrings, queryCount);
        if (judgmentsMap.containsKey(queryId)) {
            QueryInfo cachedQueryInfo = judgmentsMap.get(queryId).getQueryInfo();
            if (!newQueryInfo.equals(cachedQueryInfo)) {
                throw new IllegalArgumentException("There's a mismatch between the previously loaded:" +
                        cachedQueryInfo + "\nand the QueryInfo loaded for this row: " + newQueryInfo);
            }
        } else {
            judgmentsMap.put(queryId, new Judgments(newQueryInfo));
        }

        if (hasJudgments) {
            String documentId = record.get(DOCUMENT_ID);
            double relevanceScore =
                    Double.parseDouble(record.get(RELEVANCE));
            Judgments judgments = judgmentsMap.get(newQueryInfo.getQueryId());
            judgments.addJudgment(documentId, relevanceScore);
        }
    }
    return judgmentsMap;
}
 

public static final List<Object[]> getAllRows( LogChannelInterface log, DataSetGroup group, DataSet dataSet ) throws KettleException {
  RowMetaInterface setRowMeta = dataSet.getSetRowMeta( true );
  setValueFormats( setRowMeta );
  String dataSetFilename = getDataSetFilename( group, dataSet.getTableName() );
  List<Object[]> rows = new ArrayList<>();
  final ValueMetaString constantValueMeta = new ValueMetaString( "constant" );

  try {
    FileObject file = KettleVFS.getFileObject( dataSetFilename );
    if ( !file.exists() ) {
      // This is fine.  We haven't put rows in yet.
      //
      return rows;
    }

    try (
      Reader reader = new InputStreamReader( new BufferedInputStream( KettleVFS.getInputStream( file ) ) );
      CSVParser csvParser = new CSVParser( reader, getCsvFormat( setRowMeta ) );
    ) {
      for ( CSVRecord csvRecord : csvParser ) {
        if ( csvRecord.getRecordNumber() > 1 ) {
          Object[] row = RowDataUtil.allocateRowData( setRowMeta.size() );
          for ( int i = 0; i < setRowMeta.size(); i++ ) {
            ValueMetaInterface valueMeta = setRowMeta.getValueMeta( i ).clone();
            constantValueMeta.setConversionMetadata( valueMeta );
            String value = csvRecord.get( i );
            row[ i ] = valueMeta.convertData( constantValueMeta, value );
          }
          rows.add( row );
        }
      }
    }
    return rows;
  } catch ( Exception e ) {
    throw new KettleException( "Unable to get all rows for CSV data set '" + dataSet.getName() + "'", e );
  }
}
 

@Override
public SimpleFeature apply(final CSVRecord input) {
  if (input == null) {
    return null;
  }
  final String entityId = input.get("entityId");
  final double cloudCover = Double.parseDouble(input.get("cloudCover"));
  final String processingLevel = input.get("processingLevel");
  final int path = Integer.parseInt(input.get("path"));
  final int row = Integer.parseInt(input.get("row"));
  final String downloadUrl = input.get("download_url");

  final MultiPolygon shape = wrs2Geometry.getGeometry(path, row);
  featureBuilder.add(shape);
  featureBuilder.add(entityId);
  Date aquisitionDate;
  final SimpleDateFormat sdf = new SimpleDateFormat(AQUISITION_DATE_FORMAT);
  try {
    aquisitionDate = sdf.parse(input.get("acquisitionDate"));
    featureBuilder.add(aquisitionDate);
  } catch (final ParseException e) {
    LOGGER.warn("Unable to parse aquisition date", e);

    featureBuilder.add(null);
  }

  featureBuilder.add(cloudCover);
  featureBuilder.add(processingLevel);
  featureBuilder.add(path);
  featureBuilder.add(row);
  featureBuilder.add(downloadUrl);
  return featureBuilder.buildFeature(entityId);
}
 

@Override
public void accept(CSVRecord theRecord)  {
    String id = theRecord.get("id");
    boolean active = "1".equals(theRecord.get("active"));
    if (!active) {
        return;
    }
    String conceptId = theRecord.get("conceptId");
    if (!myValidConceptIds.contains(conceptId)) {
        return;
    }

    String term = theRecord.get("term");

    String typeId = theRecord.get("typeId");

    try {
        ConceptEntity concept = getOrCreateConcept(myCodeSystemVersion, myId2concept, myCode2concept, id, conceptId);

        concept.setCode(conceptId);
        myCodeSystemVersion.getConcepts().add(concept);

        //codeSvc.save(concept);

        ConceptDesignation designation = null;
        for (ConceptDesignation designationSearch : concept.getDesignations()) {
            if (designationSearch.getDesignationId().equals(id)) {
                designation = designationSearch;
                break;
            }
        }
        if (designation == null) {
            designation = new ConceptDesignation();
            designation.setDesignationId(id);
            designation.setConceptEntity(concept);
            concept.getDesignations().add(designation);

        }
        designation.setTerm(term);
        if (concept.getDisplay() == null) {
            concept.setDisplay(term);
        }
        switch (typeId) {
            case "900000000000003001":
                designation.setUse(ConceptDesignation.DesignationUse.FullySpecifiedName);
                concept.setDisplay(term);
                break;
            case "900000000000013009":
                designation.setUse(ConceptDesignation.DesignationUse.Synonym);
                break;
            case "900000000000550004":
                designation.setUse(ConceptDesignation.DesignationUse.Definition);
                concept.setDisplay(term);
                break;
            default:
                ourLog.info("Unknown typeId=" + typeId);
        }


        codeSvc.save(designation);
    } catch (Exception ex) {
        ourLog.error(ex.getMessage());
    }

}
 

private void loadCache() throws IllegalStateException, IOException {
    if (lock.tryLock()) {
        try {
            final ComponentLog logger = getLogger();
            if (logger.isDebugEnabled()) {
                logger.debug("Loading lookup table from file: " + csvFile);
            }

            ConcurrentHashMap<String, Record> cache = new ConcurrentHashMap<>();
            try (final InputStream is = new FileInputStream(csvFile)) {
                try (final InputStreamReader reader = new InputStreamReader(is, charset)) {
                    final CSVParser records = csvFormat.withFirstRecordAsHeader().parse(reader);
                    RecordSchema lookupRecordSchema = null;
                    for (final CSVRecord record : records) {
                        final String key = record.get(lookupKeyColumn);

                        if (StringUtils.isBlank(key)) {
                            throw new IllegalStateException("Empty lookup key encountered in: " + csvFile);
                        } else if (!ignoreDuplicates && cache.containsKey(key)) {
                            throw new IllegalStateException("Duplicate lookup key encountered: " + key + " in " + csvFile);
                        } else if (ignoreDuplicates && cache.containsKey(key)) {
                            logger.warn("Duplicate lookup key encountered: {} in {}", new Object[]{key, csvFile});
                        }

                        // Put each key/value pair (except the lookup) into the properties
                        final Map<String, Object> properties = new HashMap<>();
                        record.toMap().forEach((k, v) -> {
                            if (!lookupKeyColumn.equals(k)) {
                                properties.put(k, v);
                            }
                        });

                        if (lookupRecordSchema == null) {
                            List<RecordField> recordFields = new ArrayList<>(properties.size());
                            properties.forEach((k, v) -> recordFields.add(new RecordField(k, RecordFieldType.STRING.getDataType())));
                            lookupRecordSchema = new SimpleRecordSchema(recordFields);
                        }

                        cache.put(key, new MapRecord(lookupRecordSchema, properties));
                    }
                }
            }

            this.cache = cache;

            if (cache.isEmpty()) {
                logger.warn("Lookup table is empty after reading file: " + csvFile);
            }
        } finally {
            lock.unlock();
        }
    }
}
 

public static Map<String, CorrectedInstance> loadCorrectedInstancesFromCSV()
            throws IOException
    {
        Map<String, CorrectedInstance> result = new TreeMap<>();
        // read corrections
        List<String> fileNames = Arrays.asList("mturk/annotation-task/97-post-validation.csv",
                "mturk/annotation-task/97-post-validation2.csv");
        for (String fileName : fileNames) {
            CSVParser csvParser = CSVParser
                    .parse(new File(fileName), Charset.forName("utf-8"), CSVFormat.RFC4180);

            Iterator<CSVRecord> iterator = csvParser.iterator();

            while (iterator.hasNext()) {
                CSVRecord firstLine = iterator.next();
                CSVRecord secondLine = iterator.next();
                CSVRecord thirdLine = iterator.next();

                String id = firstLine.get(0);
                boolean skipRecord = "x".equals(firstLine.get(1)) || firstLine.get(1).isEmpty();

                if (!skipRecord) {
                    int correctLabel = Integer.valueOf(firstLine.get(1));

                    //                String[] split = secondLine.get(2).split("\\W", 2);
                    //                System.out.println(Arrays.toString(split));
                    int secondLineLabel = Integer.valueOf(secondLine.get(2).split("\\W", 2)[0]);
                    String secondLineText = secondLine.get(2).split("\\W", 2)[1];

                    int thirdLineLabel = Integer.valueOf(thirdLine.get(2).split("\\W", 2)[0]);
                    String thirdLineText = thirdLine.get(2).split("\\W", 2)[1];

                    System.out.println(correctLabel);
                    System.out.println(secondLineLabel + ", " + secondLineText);
                    System.out.println(thirdLineLabel + ", " + thirdLineText);

                    String originalWarrant;
                    String alternativeWarrant;
                    if (correctLabel == secondLineLabel) {
                        originalWarrant = secondLineText;
                        alternativeWarrant = thirdLineText;
                    }
                    else {
                        originalWarrant = thirdLineText;
                        alternativeWarrant = secondLineText;
                    }

                    CorrectedInstance correctedInstance = new CorrectedInstance(originalWarrant,
                            alternativeWarrant);
//                    System.out.println(correctedInstance);

                    result.put(id, correctedInstance);
                }
            }

            System.out.println(result.size());
        }
        return result;
    }
 

public void parseCSV(String csv) {
	try {
		Reader in = new StringReader(csv);
		CSVFormat format = CSVFormat.EXCEL.withHeader();
		CSVParser parser = format.parse(in);
		this.firstColIsIndex = false;
		for (CSVRecord record : parser) {
			if ( !firstColIsIndex && Character.isAlphabetic(record.get(0).charAt(0)) ) {
				// latch if we see alpha not number
				firstColIsIndex = true;
			}
			List<String> row = new ArrayList<>();
			for (int i = 0; i<record.size(); i++) {
				String v = record.get(i);
				boolean isInt = false;
				try {
					Integer.parseInt(v);
					isInt = true;
				}
				catch (NumberFormatException nfe) {
					isInt = false;
				}
				if ( !isInt && !NumberUtils.isDigits(v) && NumberUtils.isCreatable(v) ) {
					v = String.format("%.4f",Precision.round(Double.valueOf(v), 4));
				}
				else {
					v = abbrevString(v, 25);
				}
				row.add(v);
			}
			rows.add(row);
		}
		Set<String> colNames = parser.getHeaderMap().keySet();
		if ( !firstColIsIndex ) {
			colNames.remove(""); // remove index column name
		}
		this.colNames.addAll(colNames);
	}
	catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 

@Override
public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException {

    try {
        final RecordSchema schema = getSchema();

        final List<RecordField> recordFields = getRecordFields();
        final int numFieldNames = recordFields.size();
        for (final CSVRecord csvRecord : csvParser) {
            final Map<String, Object> values = new LinkedHashMap<>(recordFields.size() * 2);
            for (int i = 0; i < csvRecord.size(); i++) {
                final String rawValue = csvRecord.get(i);

                final String rawFieldName;
                final DataType dataType;
                if (i >= numFieldNames) {
                    if (!dropUnknownFields) {
                        values.put("unknown_field_index_" + i, rawValue);
                    }

                    continue;
                } else {
                    final RecordField recordField = recordFields.get(i);
                    rawFieldName = recordField.getFieldName();
                    dataType = recordField.getDataType();
                }


                final Object value;
                if (coerceTypes) {
                    value = convert(rawValue, dataType, rawFieldName);
                } else {
                    // The CSV Reader is going to return all fields as Strings, because CSV doesn't have any way to
                    // dictate a field type. As a result, we will use the schema that we have to attempt to convert
                    // the value into the desired type if it's a simple type.
                    value = convertSimpleIfPossible(rawValue, dataType, rawFieldName);
                }

                values.put(rawFieldName, value);
            }

            return new MapRecord(schema, values, coerceTypes, dropUnknownFields);
        }
    } catch (Exception e) {
        throw new MalformedRecordException("Error while getting next record. Root cause: " +  Throwables.getRootCause(e), e);
    }

    return null;
}
 

public void importData(Context context, DBHelper db, InputStream input, ImportExportProgressUpdater updater) throws IOException, FormatException, InterruptedException
{
    InputStreamReader reader = new InputStreamReader(input, Charsets.UTF_8);
    final CSVParser parser = new CSVParser(reader, CSVFormat.RFC4180.withHeader());

    SQLiteDatabase database = db.getWritableDatabase();
    database.beginTransaction();

    try
    {
        for (CSVRecord record : parser)
        {
            String type = record.get(DBHelper.TransactionDbIds.TYPE);
            if(type.equals("BUDGET"))
            {
                importBudget(database, db, record);
            }
            else
            {
                importTransaction(context, database, db, record);
            }

            updater.update();

            if(Thread.currentThread().isInterrupted())
            {
                throw new InterruptedException();
            }
        }

        // Do not close the parser, as it will close the input stream;
        // Closing the input stream is the responsibility of the caller.
        database.setTransactionSuccessful();
    }
    catch(IllegalArgumentException|IllegalStateException e)
    {
        throw new FormatException("Issue parsing CSV data", e);
    }
    finally
    {
        database.endTransaction();
        database.close();
    }
}