org.apache.flink.types.parser.FieldParser Java Examples

private void initializeParsers() {
    Class<?>[] fieldClasses = extractTypeClasses(fieldTypes);

    // instantiate the parsers
    FieldParser<?>[] parsers = new FieldParser<?>[fieldClasses.length];

    for (int i = 0; i < fieldClasses.length; i++) {
        if (fieldClasses[i] != null) {
            Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldClasses[i]);
            if (parserType == null) {
                throw new RuntimeException("No parser available for type '" + fieldClasses[i].getName() + "'.");
            }

            FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);
            p.setCharset(charset);
            parsers[i] = p;
        }
    }
    this.fieldParsers = parsers;
    this.holders = new Object[fieldTypes.length];
    for (int i = 0; i < fieldTypes.length; i++) {
        holders[i] = fieldParsers[i].createValue();
    }
}
 

public KvParser(String[] fieldNames, TypeInformation[] fieldTypes, String colDelimiter, String valDelimiter) {
    this.fieldNames = fieldNames;
    Preconditions.checkArgument(fieldNames.length == fieldTypes.length);
    this.isString = new boolean[fieldNames.length];
    this.parsers = new FieldParser[fieldNames.length];

    for (int i = 0; i < fieldTypes.length; i++) {
        parsers[i] = getFieldParser(fieldTypes[i].getTypeClass());
        isString[i] = fieldTypes[i].equals(Types.STRING);
    }
    this.colDelimiter = colDelimiter;
    this.valDelimiter = valDelimiter;

    keyToFieldIdx = new HashMap<>();
    for (int i = 0; i < fieldNames.length; i++) {
        keyToFieldIdx.put(fieldNames[i], i);
    }
}
 

static Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        return Tuple2.of(true, token);
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(false, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[]{0}, null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 

public ColumnsWriter(TableSchema schema) {
	this.nCols = schema.getFieldNames().length;
	this.colNames = schema.getFieldNames();
	this.isString = new boolean[colNames.length];
	TypeInformation[] fieldTypes = schema.getFieldTypes();

	this.parsers = new FieldParser[fieldTypes.length];

	for (int i = 0; i < fieldTypes.length; i++) {
		parsers[i] = getFieldParser(fieldTypes[i].getTypeClass());
		isString[i] = fieldTypes[i].equals(Types.STRING);
	}

	keyToFieldIdx = new HashMap <>();
	for (int i = 0; i < colNames.length; i++) {
		keyToFieldIdx.put(colNames[i], i);
	}
}
 

static Tuple2 <Boolean, Object> parseField(FieldParser <?> parser, String token, boolean isStringField) {
	if (isStringField) {
		return Tuple2.of(true, token);
	} else {
		if (StringUtils.isNullOrWhitespaceOnly(token)) {
			return Tuple2.of(false, null);
		}
		byte[] bytes = token.getBytes();
		parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[] {0}, null);
		FieldParser.ParseErrorState errorState = parser.getErrorState();
		if (errorState != FieldParser.ParseErrorState.NONE) {
			return Tuple2.of(false, null);
		} else {
			return Tuple2.of(true, parser.getLastResult());
		}
	}
}
 

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

/**
 * The Constructor.
 *
 * @param types      Column types.
 * @param fieldDelim Field delimiter in the text line.
 * @param quoteChar  Quoting character. Contents between a pair of quoting chars are treated as a field, even if
 *                   contains field delimiters. Two consecutive quoting chars represents a real quoting char.
 */
public CsvParser(TypeInformation[] types, String fieldDelim, @Nullable Character quoteChar) {
    this.fieldDelim = fieldDelim;
    this.lenFieldDelim = this.fieldDelim.length();
    this.quoteChar = quoteChar;
    this.enableQuote = quoteChar != null;
    this.parsers = new FieldParser[types.length];
    this.isString = new boolean[types.length];

    if (enableQuote) {
        this.quoteString = quoteChar.toString();
        this.escapedQuote = this.quoteString + this.quoteString;
    }

    for (int i = 0; i < types.length; i++) {
        Class typeClazz = types[i].getTypeClass();
        Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
        if (parserType == null) {
            throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
        }
        parsers[i] = InstantiationUtil.instantiate(parserType, FieldParser.class);
        isString[i] = types[i].equals(Types.STRING);
    }
}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();

	stringParser.enableQuotedStringParsing((byte) '"');

	Map<String, StringParser.ParseErrorState> failures = new HashMap<>();
	failures.put("\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING);
	failures.put("\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING);

	for (Map.Entry<String, StringParser.ParseErrorState> failure : failures.entrySet()) {
		int result = stringParser.parseField(
			failure.getKey().getBytes(ConfigConstants.DEFAULT_CHARSET),
			0,
			failure.getKey().length(),
			new byte[]{(byte) '|'},
			null);
		assertEquals(-1, result);
		assertEquals(failure.getValue(), stringParser.getErrorState());
	}
}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();
	stringParser.enableQuotedStringParsing((byte) '"');

	Object[][] failures = {
			{"\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING},
			{"\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING}
	};

	for (Object[] failure : failures) {
		String input = (String) failure[0];

		int result = stringParser.parseField(input.getBytes(ConfigConstants.DEFAULT_CHARSET), 0,
			input.length(), new byte[]{'|'}, null);

		assertThat(result, is(-1));
		assertThat(stringParser.getErrorState(), is(failure[1]));
	}

}
 

private Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        if (!enableQuote || token.charAt(0) != quoteChar) {
            return Tuple2.of(true, token);
        }
        String content;
        if (token.endsWith(quoteChar.toString())) {
            content = token.substring(1, token.length() - 1);
        } else {
            content = token.substring(1, token.length());
        }
        return Tuple2.of(true, content.replace(escapedQuote, quoteString));
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(true, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, fieldDelim.getBytes(), null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	@SuppressWarnings("unchecked")
	FieldParser<Object>[] fieldParsers = (FieldParser<Object>[]) getFieldParsers();

	// create the value holders
	this.parsedValues = new Object[fieldParsers.length];
	for (int i = 0; i < fieldParsers.length; i++) {
		this.parsedValues[i] = fieldParsers[i].createValue();
	}

	// left to right evaluation makes access [0] okay
	// this marker is used to fasten up readRecord, so that it doesn't have to check each call if the line ending is set to default
	if (this.getDelimiter().length == 1 && this.getDelimiter()[0] == '\n') {
		this.lineDelimiterIsLinebreak = true;
	}

	this.commentCount = 0;
	this.invalidLineCount = 0;
}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();
	stringParser.enableQuotedStringParsing((byte) '"');

	Object[][] failures = {
			{"\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING},
			{"\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING}
	};

	for (Object[] failure : failures) {
		String input = (String) failure[0];

		int result = stringParser.parseField(input.getBytes(ConfigConstants.DEFAULT_CHARSET), 0,
			input.length(), new byte[]{'|'}, null);

		assertThat(result, is(-1));
		assertThat(stringParser.getErrorState(), is(failure[1]));
	}

}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();

	stringParser.enableQuotedStringParsing((byte) '"');

	Map<String, StringParser.ParseErrorState> failures = new HashMap<>();
	failures.put("\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING);
	failures.put("\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING);

	for (Map.Entry<String, StringParser.ParseErrorState> failure : failures.entrySet()) {
		int result = stringParser.parseField(
			failure.getKey().getBytes(ConfigConstants.DEFAULT_CHARSET),
			0,
			failure.getKey().length(),
			new byte[]{(byte) '|'},
			null);
		assertEquals(-1, result);
		assertEquals(failure.getValue(), stringParser.getErrorState());
	}
}
 

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();

	stringParser.enableQuotedStringParsing((byte) '"');

	Map<String, StringParser.ParseErrorState> failures = new HashMap<>();
	failures.put("\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING);
	failures.put("\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING);

	for (Map.Entry<String, StringParser.ParseErrorState> failure : failures.entrySet()) {
		int result = stringParser.parseField(
			failure.getKey().getBytes(ConfigConstants.DEFAULT_CHARSET),
			0,
			failure.getKey().length(),
			new byte[]{(byte) '|'},
			null);
		assertEquals(-1, result);
		assertEquals(failure.getValue(), stringParser.getErrorState());
	}
}
 

@Test
public void testParseStringErrors() throws Exception {
	StringParser stringParser = new StringParser();
	stringParser.enableQuotedStringParsing((byte) '"');

	Object[][] failures = {
			{"\"string\" trailing", FieldParser.ParseErrorState.UNQUOTED_CHARS_AFTER_QUOTED_STRING},
			{"\"unterminated ", FieldParser.ParseErrorState.UNTERMINATED_QUOTED_STRING}
	};

	for (Object[] failure : failures) {
		String input = (String) failure[0];

		int result = stringParser.parseField(input.getBytes(ConfigConstants.DEFAULT_CHARSET), 0,
			input.length(), new byte[]{'|'}, null);

		assertThat(result, is(-1));
		assertThat(stringParser.getErrorState(), is(failure[1]));
	}

}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	@SuppressWarnings("unchecked")
	FieldParser<Object>[] fieldParsers = (FieldParser<Object>[]) getFieldParsers();

	//throw exception if no field parsers are available
	if (fieldParsers.length == 0) {
		throw new IOException("CsvInputFormat.open(FileInputSplit split) - no field parsers to parse input");
	}

	// create the value holders
	this.parsedValues = new Object[fieldParsers.length];
	for (int i = 0; i < fieldParsers.length; i++) {
		this.parsedValues[i] = fieldParsers[i].createValue();
	}

	// left to right evaluation makes access [0] okay
	// this marker is used to fasten up readRecord, so that it doesn't have to check each call if the line ending is set to default
	if (this.getDelimiter().length == 1 && this.getDelimiter()[0] == '\n') {
		this.lineDelimiterIsLinebreak = true;
	}

	this.commentCount = 0;
	this.invalidLineCount = 0;
}
 

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

@Override
public void open() {
	this.formatReader = initFormatReader(super.getDataSchema(), params).f0;
	this.isString = new boolean[fieldTypes.length];
	this.parsers = new FieldParser[fieldTypes.length];

	for (int i = 0; i < fieldTypes.length; i++) {
		parsers[i] = ColumnsWriter.getFieldParser(fieldTypes[i].getTypeClass());
		isString[i] = fieldTypes[i].equals(Types.STRING);
	}
}
 

static FieldParser <?> getFieldParser(Class typeClazz) {
	Class <? extends FieldParser <?>> parserType = FieldParser.getParserForType(typeClazz);
	if (parserType == null) {
		throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
	}
	return InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}
 

public JsonParser(String[] fieldNames, TypeInformation[] fieldTypes) {
    this.fieldNames = fieldNames;
    Preconditions.checkArgument(fieldNames.length == fieldTypes.length);
    this.isString = new boolean[fieldNames.length];
    this.parsers = new FieldParser[fieldNames.length];

    for (int i = 0; i < fieldTypes.length; i++) {
        parsers[i] = getFieldParser(fieldTypes[i].getTypeClass());
        isString[i] = fieldTypes[i].equals(Types.STRING);
    }
}
 

static FieldParser<?> getFieldParser(Class typeClazz) {
    Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
    if (parserType == null) {
        throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
    }
    return InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}
 

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}
 

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}