Java Code Examples for org.apache.flink.types.Value

private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 

@Override
public TypePairComparator<Record, Record> createComparator12(
		TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos1, pos2, types1);
}
 

public <T extends Value> void registerAggregationConvergenceCriterion(
		String name, Aggregator<T> aggregator, ConvergenceCriterion<T> convergenceCheck)
{
	if (name == null || aggregator == null || convergenceCheck == null) {
		throw new IllegalArgumentException("Name, aggregator, or convergence criterion must not be null");
	}
	
	Aggregator<?> genAgg = aggregator;
	
	Aggregator<?> previous = this.registry.get(name);
	if (previous != null && previous != genAgg) {
		throw new RuntimeException("An aggregator is already registered under the given name.");
	}
	
	this.registry.put(name, genAgg);
	this.convergenceCriterion = convergenceCheck;
	this.convergenceCriterionAggregatorName = name;
}
 

/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 

@Test
public void testReadTooShortInputLenient() throws IOException {
	try {
		final String fileContent = "666|777|888|999|555\n111|222|333|444\n666|777|888|999|555";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);
		format.setLenient(true);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(5);
		
		assertNotNull(format.nextRecord(values));	// line okay
		assertNull(format.nextRecord(values));	// line too short
		assertNotNull(format.nextRecord(values));	// line okay
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 

/**
 * Copy constructor.
 * 
 * @param toCopy Comparator to copy.
 */
private RecordComparator(RecordComparator toCopy) {
	this.keyFields = toCopy.keyFields;
	this.keyHolders = new Value[toCopy.keyHolders.length];
	this.transientKeyHolders = new Value[toCopy.keyHolders.length];
	
	try {
		for (int i = 0; i < this.keyHolders.length; i++) {
			this.keyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
			this.transientKeyHolders[i] = toCopy.keyHolders[i].getClass().newInstance();
		}
	} catch (Exception ex) {
		// this should never happen, because the classes have been instantiated before. Report for debugging.
		throw new RuntimeException("Could not instantiate key classes when duplicating RecordComparator.", ex);
	}
	
	this.normalizedKeyLengths = toCopy.normalizedKeyLengths;
	this.numLeadingNormalizableKeys = toCopy.numLeadingNormalizableKeys;
	this.normalizableKeyPrefixLen = toCopy.normalizableKeyPrefixLen;
	this.ascending = toCopy.ascending;
	
	this.temp1 = new Record();
	this.temp2 = new Record();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testValueSupertypeException() {
	RichMapFunction<?, ?> function = new RichMapFunction<StringValue, Value>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Value map(StringValue value) throws Exception {
			return null;
		}
	};

	TypeInformation<?> ti =TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}), "name", true);
	Assert.assertTrue(ti instanceof MissingTypeInfo);
	
	try {
		TypeExtractor.getMapReturnTypes(function, (TypeInformation)TypeInformation.of(new TypeHint<StringValue>(){}));
		Assert.fail("Expected an exception");
	}
	catch (InvalidTypesException e) {
		// expected
	}
}
 

@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();

	this.memManager = MemoryManagerBuilder.newBuilder().setMemorySize(32 * 1024 * 1024).build();
	this.ioManager = new IOManagerAsync();
}
 

/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 

/**
 * Produce a {@code ValueArray} for the given {@code Value} type.
 *
 * @param cls {@code Value} class
 * @return {@code ValueArray} for given {@code Value} class
 */
@SuppressWarnings("unchecked")
public static <T> ValueArray<T> createValueArray(Class<? extends Value> cls) {
	if (ByteValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ByteValueArray();
	} else if (CharValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new CharValueArray();
	} else if (DoubleValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new DoubleValueArray();
	} else if (FloatValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new FloatValueArray();
	} else if (IntValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new IntValueArray();
	} else if (LongValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new LongValueArray();
	} else if (NullValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new NullValueArray();
	} else if (ShortValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new ShortValueArray();
	} else if (StringValue.class.isAssignableFrom(cls)) {
		return (ValueArray<T>) new StringValueArray();
	} else {
		throw new IllegalArgumentException("Unable to create unbounded ValueArray for type " + cls);
	}
}
 

public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 

public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 

protected IterationEventWithAggregators(Map<String, Aggregator<?>> aggregators) {
	int num = aggregators.size();
	if (num == 0) {
		this.aggNames = NO_STRINGS;
		this.aggregates = NO_VALUES;
	} else {
		this.aggNames = new String[num];
		this.aggregates = new Value[num];

		int i = 0;
		for (Map.Entry<String, Aggregator<?>> entry : aggregators.entrySet()) {
			this.aggNames[i] = entry.getKey();
			this.aggregates[i] = entry.getValue().getAggregate();
			i++;
		}
	}
}
 

@Before
public void setup()
{
	final int[] keyPos = new int[] {0};
	@SuppressWarnings("unchecked")
	final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
	
	this.recordBuildSideAccesssor = RecordSerializer.get();
	this.recordProbeSideAccesssor = RecordSerializer.get();
	this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
	this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
	this.pactRecordComparator = new RecordPairComparatorFirstInt();
	
	this.pairBuildSideAccesssor = new IntPairSerializer();
	this.pairProbeSideAccesssor = new IntPairSerializer();
	this.pairBuildSideComparator = new IntPairComparator();
	this.pairProbeSideComparator = new IntPairComparator();
	this.pairComparator = new IntPairPairComparator();
	
	this.memManager = new MemoryManager(32 * 1024 * 1024,1);
	this.ioManager = new IOManagerAsync();
}
 

@Override
public TypePairComparator<Record, Record> createComparator21(
	TypeComparator<Record> comparator1,	TypeComparator<Record> comparator2)
{
	if (!(comparator1 instanceof RecordComparator && comparator2 instanceof RecordComparator)) {
		throw new IllegalArgumentException("Cannot instantiate pair comparator from the given comparators.");
	}
	final RecordComparator prc1 = (RecordComparator) comparator1;
	final RecordComparator prc2 = (RecordComparator) comparator2;
	
	final int[] pos1 = prc1.getKeyPositions();
	final int[] pos2 = prc2.getKeyPositions();
	
	final Class<? extends Value>[] types1 = prc1.getKeyTypes();
	final Class<? extends Value>[] types2 = prc2.getKeyTypes();
	
	checkComparators(pos1, pos2, types1, types2);
	
	return new RecordPairComparator(pos2, pos1, types1);
}
 

public RecordPairComparator(int[] keyFieldsReference, int[] keyFieldsCandidate, Class<? extends Value>[] keyTypes) {
	if (keyFieldsReference.length != keyFieldsCandidate.length || keyFieldsCandidate.length != keyTypes.length) {
		throw new IllegalArgumentException(
			"The arrays describing the key positions and types must be of the same length.");
	}
	this.keyFields1 = keyFieldsReference;
	this.keyFields2 = keyFieldsCandidate;
	
	// instantiate fields to extract keys into
	this.keyHolders1 = new Value[keyTypes.length];
	this.keyHolders2 = new Value[keyTypes.length];
	
	for (int i = 0; i < keyTypes.length; i++) {
		if (keyTypes[i] == null) {
			throw new NullPointerException("Key type " + i + " is null.");
		}
		this.keyHolders1[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
		this.keyHolders2[i] = InstantiationUtil.instantiate(keyTypes[i], Value.class);
	}
}
 

public RecordComparatorFactory(int[] positions, Class<? extends Value>[] types, boolean[] sortDirections) {
	if (positions == null || types == null) {
		throw new NullPointerException();
	}
	if (positions.length != types.length) {
		throw new IllegalArgumentException();
	}
	
	this.positions = positions;
	this.types = types;
	
	if (sortDirections == null) {
		this.sortDirections = new boolean[positions.length];
		Arrays.fill(this.sortDirections, true);
	} else if (sortDirections.length != positions.length) {
		throw new IllegalArgumentException();
	} else {
		this.sortDirections = sortDirections;
	}
}
 

private void onWorkerDoneEvent(WorkerDoneEvent workerDoneEvent) {
	if (this.endOfSuperstep) {
		throw new RuntimeException("Encountered WorderDoneEvent when still in End-of-Superstep status.");
	}

	workerDoneEventCounter++;

	String[] aggNames = workerDoneEvent.getAggregatorNames();
	Value[] aggregates = workerDoneEvent.getAggregates(userCodeClassLoader);

	if (aggNames.length != aggregates.length) {
		throw new RuntimeException("Inconsistent WorkerDoneEvent received!");
	}

	for (int i = 0; i < aggNames.length; i++) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> aggregator = (Aggregator<Value>) this.aggregators.get(aggNames[i]);
		aggregator.aggregate(aggregates[i]);
	}

	if (workerDoneEventCounter % numberOfEventsUntilEndOfSuperstep == 0) {
		endOfSuperstep = true;
		Thread.currentThread().interrupt();
	}
}
 

@PublicEvolving
public ValueTypeInfo(Class<T> type) {
	this.type = checkNotNull(type);

	checkArgument(
		Value.class.isAssignableFrom(type) || type.equals(Value.class),
		"ValueTypeInfo can only be used for subclasses of %s", Value.class.getName());
}
 

public CollectionExecutor(ExecutionConfig executionConfig) {
	this.executionConfig = executionConfig;
	
	this.intermediateResults = new HashMap<Operator<?>, List<?>>();
	this.accumulators = new HashMap<String, Accumulator<?,?>>();
	this.previousAggregates = new HashMap<String, Value>();
	this.aggregators = new HashMap<String, Aggregator<?>>();
	this.cachedFiles = new HashMap<String, Future<Path>>();
	this.userCodeClassLoader = Thread.currentThread().getContextClassLoader();
}
 

@PublicEvolving
public ValueTypeInfo(Class<T> type) {
	this.type = checkNotNull(type);

	checkArgument(
		Value.class.isAssignableFrom(type) || type.equals(Value.class),
		"ValueTypeInfo can only be used for subclasses of %s", Value.class.getName());
}
 

@PublicEvolving
static <X extends Value> TypeInformation<X> getValueTypeInfo(Class<X> typeClass) {
	if (Value.class.isAssignableFrom(typeClass) && !typeClass.equals(Value.class)) {
		return new ValueTypeInfo<X>(typeClass);
	}
	else {
		throw new InvalidTypesException("The given class is no subclass of " + Value.class.getName());
	}
}
 

public Collection<AggregatorWithName<?>> getAllRegisteredAggregators() {
	ArrayList<AggregatorWithName<?>> list = new ArrayList<AggregatorWithName<?>>(this.registry.size());
	
	for (Map.Entry<String, Aggregator<?>> entry : this.registry.entrySet()) {
		@SuppressWarnings("unchecked")
		Aggregator<Value> valAgg = (Aggregator<Value>) entry.getValue();
		list.add(new AggregatorWithName<>(entry.getKey(), valAgg));
	}
	return list;
}
 

@Test
public void readWithEmptyField() {
	try {
		final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
		final FileInputSplit split = createTempFile(fileContent);
	
		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(StringValue.class, StringValue.class, StringValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = new Value[] { new StringValue(), new StringValue(), new StringValue()};
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("abc", ((StringValue) values[0]).getValue());
		assertEquals("def", ((StringValue) values[1]).getValue());
		assertEquals("ghijk", ((StringValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("abc", ((StringValue) values[0]).getValue());
		assertEquals("", ((StringValue) values[1]).getValue());
		assertEquals("hhg", ((StringValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals("", ((StringValue) values[0]).getValue());
		assertEquals("", ((StringValue) values[1]).getValue());
		assertEquals("", ((StringValue) values[2]).getValue());
		
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testReadNoPosAllGzip() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempGzipFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testCheckRuntimeContextAccess() {
	final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
	final TaskInfo taskInfo = new TaskInfo("test name", 3, 1, 3, 0);
	
	inputFormat.setRuntimeContext(new RuntimeUDFContext(
			taskInfo, getClass().getClassLoader(), new ExecutionConfig(),
			new HashMap<String, Future<Path>>(),
			new HashMap<String, Accumulator<?, ?>>(),
			new UnregisteredMetricsGroup()));

	assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
	assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
}
 

@Test
public void testReadNoPosAllDeflate() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempDeflateFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testReadNoPosAllGzip() throws IOException {
	try {
		final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
		final FileInputSplit split = createTempGzipFile(fileContent);

		final Configuration parameters = new Configuration();

		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, IntValue.class, IntValue.class, IntValue.class, IntValue.class);

		format.configure(parameters);
		format.open(split);

		Value[] values = createIntValues(5);

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(222, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		assertEquals(444, ((IntValue) values[3]).getValue());
		assertEquals(555, ((IntValue) values[4]).getValue());

		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(666, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		assertEquals(999, ((IntValue) values[3]).getValue());
		assertEquals(000, ((IntValue) values[4]).getValue());

		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
	}
}
 

@Test
public void testSparseParse() {
	try {
		final String fileContent =
				"111|222|333|444|555|666|777|888|999|000|\n"+
				"000|999|888|777|666|555|444|333|222|111|";
		final FileInputSplit split = createTempFile(fileContent);	
	
		final Configuration parameters = new Configuration();
		
		format.setFieldDelimiter("|");
		format.setFieldTypesGeneric(IntValue.class, null, null, IntValue.class, null, null, null, IntValue.class);
		
		format.configure(parameters);
		format.open(split);
		
		Value[] values = createIntValues(3);
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(111, ((IntValue) values[0]).getValue());
		assertEquals(444, ((IntValue) values[1]).getValue());
		assertEquals(888, ((IntValue) values[2]).getValue());
		
		values = format.nextRecord(values);
		assertNotNull(values);
		assertEquals(000, ((IntValue) values[0]).getValue());
		assertEquals(777, ((IntValue) values[1]).getValue());
		assertEquals(333, ((IntValue) values[2]).getValue());
		
		assertNull(format.nextRecord(values));
		assertTrue(format.reachedEnd());
	}
	catch (Exception ex) {
		System.err.println(ex.getMessage());
		ex.printStackTrace();
		fail("Test erroneous");
	}
}