com.opencsv.enums.CSVReaderNullFieldIndicator Java Examples

public SpoolDirCsvSourceConnectorConfig(final boolean isTask, Map<String, ?> settings) {
  super(isTask, true, config(), settings);
  this.skipLines = this.getInt(SpoolDirCsvSourceConnectorConfig.CSV_SKIP_LINES_CONF);
  this.separatorChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_SEPARATOR_CHAR_CONF);
  this.quoteChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_QUOTE_CHAR_CONF);
  this.escapeChar = this.getChar(SpoolDirCsvSourceConnectorConfig.CSV_ESCAPE_CHAR_CONF);
  this.ignoreLeadingWhitespace = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_IGNORE_LEADING_WHITESPACE_CONF);
  this.ignoreQuotations = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_IGNORE_QUOTATIONS_CONF);
  this.strictQuotes = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_STRICT_QUOTES_CONF);
  this.keepCarriageReturn = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_KEEP_CARRIAGE_RETURN_CONF);
  this.verifyReader = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_VERIFY_READER_CONF);
  this.nullFieldIndicator = ConfigUtils.getEnum(CSVReaderNullFieldIndicator.class, this, SpoolDirCsvSourceConnectorConfig.CSV_NULL_FIELD_INDICATOR_CONF);
  this.firstRowAsHeader = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_FIRST_ROW_AS_HEADER_CONF);

  String charsetName = this.getString(SpoolDirCsvSourceConnectorConfig.CSV_CHARSET_CONF);
  this.charset = Charset.forName(charsetName);

  this.caseSensitiveFieldNames = this.getBoolean(SpoolDirCsvSourceConnectorConfig.CSV_CASE_SENSITIVE_FIELD_NAMES_CONF);
  this.useRFC4180Parser = this.getBoolean(CSV_USE_RFC_4180_PARSER_CONF);
}
 

public CSVReader createCSVReader() throws IOException {
	Path path = Paths.get(this.datasetPath + this.datasetName + this.datasetEnding);
	
	CSVParser parser = new CSVParserBuilder()
			.withSeparator(this.valueSeparator)
			.withQuoteChar(this.valueQuote)
			.withEscapeChar(this.valueEscape)
			.withStrictQuotes(this.valueStrictQuotes)
			.withIgnoreLeadingWhiteSpace(this.valueIgnoreLeadingWhitespace)
			.withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS)
			.build();
	
	BufferedReader buffer = Files.newBufferedReader(path, this.charset);
	CSVReader reader = new CSVReaderBuilder(buffer).withCSVParser(parser).build();
	
	if (this.fileHasHeader)
		reader.readNext();
	
	return reader;
}
 

/**
 * Expects filenames in the map
 */
public static Map<En, List<String[]>> start2(Map<String, Reader> map, AqlOptions op,
		Schema<Ty, En, Sym, Fk, Att> sch, boolean omitCheck) throws Exception {
	Character sepChar = (Character) op.getOrDefault(AqlOption.csv_field_delim_char);
	Character quoteChar = (Character) op.getOrDefault(AqlOption.csv_quote_char);
	Character escapeChar = (Character) op.getOrDefault(AqlOption.csv_escape_char);

	final CSVParser parser = new CSVParserBuilder().withSeparator(sepChar).withQuoteChar(quoteChar)
			.withEscapeChar(escapeChar).withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();

	Map<En, List<String[]>> ret = new THashMap<>();
	for (String k : map.keySet()) {
		if (!omitCheck) {
			if (!sch.ens.contains(En.En(k))) {
				throw new RuntimeException("Not an entity: " + k);
			}
		}
		Reader r = map.get(k);

		// File file = new File(map.get(k));
		// BufferedReader fileReader = new BufferedReader(r);
		// String s;
		/// while ((s = fileReader.readLine()) != null) {
		// System.out.println(s);
		// }

		final CSVReader reader = new CSVReaderBuilder(r).withCSVParser(parser)
				.withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();

		List<String[]> rows = reader.readAll();
		
		ret.put(En.En(k), rows);
		reader.close();
		r.close();
	}

	if (!omitCheck) {
		for (En en : sch.ens) {
			if (!ret.containsKey(en)) {
				ret.put(en,
						new LinkedList<>(Collections.singletonList(Util
								.union(sch.attsFrom(en).stream().map(Object::toString).collect(Collectors.toList()),
										sch.fksFrom(en).stream().map(Object::toString).collect(Collectors.toList()))
								.toArray(new String[0]))));
			}
		}
	}
	return ret;
}
 

@Override
protected Schema<Ty, En, Sym, Fk, Att> eval0(AqlEnv env, boolean isCompileTime) {
	AqlOptions op = new AqlOptions(options, env.defaults);
	SqlTypeSide ts = SqlTypeSide.SqlTypeSide(op);
	
	Character sepChar = (Character) op.getOrDefault(AqlOption.csv_field_delim_char);
	Character quoteChar = (Character) op.getOrDefault(AqlOption.csv_quote_char);
	Character escapeChar = (Character) op.getOrDefault(AqlOption.csv_escape_char);
	String colSep = (String) op.getOrDefault(AqlOption.import_col_seperator);
	String pre = (String) op.getOrDefault(AqlOption.csv_import_prefix);
	String ext = (String) op.getOrDefault(AqlOption.csv_file_extension);

	boolean prepend = (boolean)	op.getOrDefault(AqlOption.csv_prepend_entity);

	final CSVParser parser = new CSVParserBuilder().withSeparator(sepChar).withQuoteChar(quoteChar)
			.withEscapeChar(escapeChar).withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();

	File f = new File(str);
	if (!f.exists()) {
		throw new RuntimeException("Does not exist: " + str);			
	} else if (!f.isDirectory()) {
		throw new RuntimeException("Not a directory: " + str);
	}
	Collage<Ty, En, Sym, Fk, Att, Void, Void> col = new CCollage<>();
	File[] files = f.listFiles();
	Ty vc = Ty.Ty("Varchar");
	for (File xx : files) {
		try {
			Reader r = new FileReader(xx);
			final CSVReader reader = new CSVReaderBuilder(r).withCSVParser(parser)
					.withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();

			String[] rows = reader.readNext();
			if (rows == null) {
				reader.close();
				r.close();
				throw new RuntimeException("No header in: " + xx.getPath());
			}				
			
			String en = xx.getName().replaceAll("[\uFEFF-\uFFFF]", "").trim();
			if (en.endsWith("." + ext)) {
				en = en.substring(0, en.length() - (ext.length() + 1));
			}
			xx.list(new FilenameFilter() {
				@Override
				public boolean accept(File dir, String name) {
					return name.endsWith("." + ext);
				}					
			});
			En e = En.En(en);
			col.getEns().add(e);
			for (String c0 : rows) {
				String c = c0.replace("\t"," ").replaceAll("[\uFEFF-\uFFFF]", "").trim();
				String d = (prepend ? en + colSep : "").trim() + c;
				Att att = Att.Att(En.En(en), d.trim());
				col.atts().put(att, new Pair<>(e, vc));
			}
			reader.close();
			r.close();		
		} catch (Exception ex) {
			ex.printStackTrace();
			throw new RuntimeException(ex.getMessage());
		}
		
	}

	return new Schema<>(ts, col, op);
	
}