Java Code Examples for org.apache.flink.types.parser.FieldParser#getParserForType()

The following examples show how to use org.apache.flink.types.parser.FieldParser#getParserForType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: GenericCsvInputFormat.java From Flink-CEPplus with Apache License 2.0

6 votes

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}

Example 2

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

6 votes

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}

Example 3

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

6 votes

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}

Example 4

Source File: GenericCsvInputFormat.java From Alink with Apache License 2.0

6 votes

private void initializeParsers() {
    Class<?>[] fieldClasses = extractTypeClasses(fieldTypes);

    // instantiate the parsers
    FieldParser<?>[] parsers = new FieldParser<?>[fieldClasses.length];

    for (int i = 0; i < fieldClasses.length; i++) {
        if (fieldClasses[i] != null) {
            Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldClasses[i]);
            if (parserType == null) {
                throw new RuntimeException("No parser available for type '" + fieldClasses[i].getName() + "'.");
            }

            FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);
            p.setCharset(charset);
            parsers[i] = p;
        }
    }
    this.fieldParsers = parsers;
    this.holders = new Object[fieldTypes.length];
    for (int i = 0; i < fieldTypes.length; i++) {
        holders[i] = fieldParsers[i].createValue();
    }
}

Example 5

Source File: CsvParser.java From Alink with Apache License 2.0

6 votes

/**
 * The Constructor.
 *
 * @param types      Column types.
 * @param fieldDelim Field delimiter in the text line.
 * @param quoteChar  Quoting character. Contents between a pair of quoting chars are treated as a field, even if
 *                   contains field delimiters. Two consecutive quoting chars represents a real quoting char.
 */
public CsvParser(TypeInformation[] types, String fieldDelim, @Nullable Character quoteChar) {
    this.fieldDelim = fieldDelim;
    this.lenFieldDelim = this.fieldDelim.length();
    this.quoteChar = quoteChar;
    this.enableQuote = quoteChar != null;
    this.parsers = new FieldParser[types.length];
    this.isString = new boolean[types.length];

    if (enableQuote) {
        this.quoteString = quoteChar.toString();
        this.escapedQuote = this.quoteString + this.quoteString;
    }

    for (int i = 0; i < types.length; i++) {
        Class typeClazz = types[i].getTypeClass();
        Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
        if (parserType == null) {
            throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
        }
        parsers[i] = InstantiationUtil.instantiate(parserType, FieldParser.class);
        isString[i] = types[i].equals(Types.STRING);
    }
}

Example 6

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}

Example 7

Source File: PrimitiveInputFormat.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}

Example 8

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}

Example 9

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}

Example 10

Source File: PrimitiveInputFormat.java From flink with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}

Example 11

Source File: ColumnsWriter.java From Alink with Apache License 2.0

5 votes

static FieldParser <?> getFieldParser(Class typeClazz) {
	Class <? extends FieldParser <?>> parserType = FieldParser.getParserForType(typeClazz);
	if (parserType == null) {
		throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
	}
	return InstantiationUtil.instantiate(parserType, FieldParser.class);
}

Example 12

Source File: StringParsers.java From Alink with Apache License 2.0

5 votes

static FieldParser<?> getFieldParser(Class typeClazz) {
    Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
    if (parserType == null) {
        throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
    }
    return InstantiationUtil.instantiate(parserType, FieldParser.class);
}

Example 13

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}

Example 14

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}

Example 15

Source File: GenericCsvInputFormat.java From flink with Apache License 2.0

5 votes

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}

Example 16

Source File: PrimitiveInputFormat.java From flink with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}

Example 17

Source File: GenericCsvInputFormat.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}

Example 18

Source File: GenericCsvInputFormat.java From Flink-CEPplus with Apache License 2.0

5 votes

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}

Example 19

Source File: GenericCsvInputFormat.java From Flink-CEPplus with Apache License 2.0

5 votes

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}