Java Code Examples for org.apache.flink.types.Row#getField()

The following examples show how to use org.apache.flink.types.Row#getField() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private String createKey(Row row) {
	final StringBuilder builder = new StringBuilder();
	for (int i = 0; i < keyFieldIndices.length; i++) {
		final int keyFieldIndex = keyFieldIndices[i];
		if (i > 0) {
			builder.append(keyDelimiter);
		}
		final Object value = row.getField(keyFieldIndex);
		if (value == null) {
			builder.append(keyNullLiteral);
		} else {
			builder.append(value.toString());
		}
	}
	return builder.toString();
}
 
Example 2
Source Project: flink   File: DeeplyEqualsChecker.java    License: Apache License 2.0 6 votes vote down vote up
private boolean deepEqualsRow(Row row1, Row row2) {
	int arity = row1.getArity();

	if (row1.getArity() != row2.getArity()) {
		return false;
	}

	for (int i = 0; i < arity; i++) {
		Object copiedValue = row1.getField(i);
		Object element = row2.getField(i);
		if (!deepEquals(copiedValue, element)) {
			return false;
		}
	}

	return true;
}
 
Example 3
Source Project: flink   File: RowComparator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int hash(Row record) {
	int code = 0;
	int i = 0;

	try {
		for (; i < keyPositions.length; i++) {
			code *= TupleComparatorBase.HASH_SALT[i & 0x1F];
			Object element = record.getField(keyPositions[i]); // element can be null
			code += comparators[i].hash(element);
		}
	} catch (IndexOutOfBoundsException e) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i]);
	}

	return code;
}
 
Example 4
Source Project: flink   File: RowSmallIntWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void doWrite(Row value, int ordinal) {
	if (value.getField(ordinal) == null) {
		((SmallIntVector) getValueVector()).setNull(getCount());
	} else {
		((SmallIntVector) getValueVector()).setSafe(getCount(), (short) value.getField(ordinal));
	}
}
 
Example 5
/**
 * 获取自定义索引名
 *
 * @param row
 * @return
 */
private String getIndex(Row row) {
    if (fieldIndex == null) {
        return this.index;
    }
    return (String) row.getField(fieldIndex);
}
 
Example 6
/**
 * 获取自定义索引名
 *
 * @param row
 * @return
 */
private String getIndex(Row row) {
    if (fieldIndex == null) {
        return this.index;
    }
    return (String) row.getField(fieldIndex);
}
 
Example 7
Source Project: flink   File: RowComparator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setReference(Row toCompare) {
	int i = 0;
	try {
		for (; i < keyPositions.length; i++) {
			TypeComparator<Object> comparator = comparators[i];
			Object element = toCompare.getField(keyPositions[i]);
			comparator.setReference(element);   // element can be null
		}
	} catch (IndexOutOfBoundsException e) {
		throw new KeyFieldOutOfBoundsException(keyPositions[i]);
	}
}
 
Example 8
@Override
public void writeRecord(Row record) throws IOException {
	// TODO Auto-generated method stub
	if (record != null) {
		Object[] simpleRow = new Object[record.getArity()];
		for (int i = 0; i < simpleRow.length; i++) {
			simpleRow[i] = record.getField(i);
		}
		this.writeRow(converter.getSpreadSheetCellDAOfromSimpleDataType(simpleRow, this.defaultSheetName,
				this.rowNumSimple));
	}
	this.rowNumSimple++;
}
 
Example 9
@Test
public void testReadWithProjection() throws IOException {
	rowOrcInputFormat = new OrcRowInputFormat(getPath(TEST_FILE_NESTED), TEST_SCHEMA_NESTED, new Configuration());

	rowOrcInputFormat.selectFields(7, 0, 10, 8);

	FileInputSplit[] splits = rowOrcInputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	rowOrcInputFormat.openInputFormat();
	rowOrcInputFormat.open(splits[0]);

	assertFalse(rowOrcInputFormat.reachedEnd());
	Row row = rowOrcInputFormat.nextRecord(null);

	// validate first row
	assertNotNull(row);
	assertEquals(4, row.getArity());
	// check binary
	assertArrayEquals(new byte[]{0, 1, 2, 3, 4}, (byte[]) row.getField(0));
	// check boolean
	assertEquals(false, row.getField(1));
	// check list
	assertTrue(row.getField(2) instanceof Object[]);
	Object[] list1 = (Object[]) row.getField(2);
	assertEquals(2, list1.length);
	assertEquals(Row.of(3, "good"), list1[0]);
	assertEquals(Row.of(4, "bad"), list1[1]);
	// check string
	assertEquals("hi", row.getField(3));

	// check that there is a second row with four fields
	assertFalse(rowOrcInputFormat.reachedEnd());
	row = rowOrcInputFormat.nextRecord(null);
	assertNotNull(row);
	assertEquals(4, row.getArity());
	assertTrue(rowOrcInputFormat.reachedEnd());
}
 
Example 10
Source Project: Alink   File: ChiSquareTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compare(Row o1, Row o2) {
    double d1 = (double) o1.getField(1);
    double d2 = (double) o2.getField(1);

    return Double.compare(d1, d2);
}
 
Example 11
Source Project: Alink   File: GlmUtil.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Row map(Row row) throws Exception {
    for (int i = 0; i < features.length; i++) {
        features[i] = (double) row.getField(i);
    }
    features = familyLink.calcWeightAndLabel(this.coefficients, this.intercept, features);

    Row outRow = new Row(features.length);
    for (int i = 0; i < features.length; i++) {
        outRow.setField(i, features[i]);
    }
    return outRow;
}
 
Example 12
Row map(Row row){
	for (int i = 0; i < paramsBuilder.selectedCols.length; i++) {
		int colIdxInData = selectedColIndicesInData[i];
		Object val = row.getField(colIdxInData);
		int foundIndex = discretizers[i].findIndex(val);
		predictIndices[i] = (long) foundIndex;
		if (!discretizers[i].isValid(foundIndex)) {
			switch (paramsBuilder.handleInvalidStrategy) {
				case KEEP:
					break;
				case SKIP:
					predictIndices[i] = null;
					break;
				case ERROR:
					throw new RuntimeException("Unseen token: " + val);
				default:
					throw new IllegalArgumentException("Invalid handle invalid strategy.");
			}
		}
	}

	return paramsBuilder.outputColsHelper.getResultRow(
		row,
		setResultRow(
			predictIndices,
			paramsBuilder.encode,
			dropIndex,
			vectorSize,
			paramsBuilder.dropLast,
			assembledVectorSize)
	);
}
 
Example 13
Source Project: flink   File: RowFloatWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void doWrite(Row value, int ordinal) {
	if (value.getField(ordinal) == null) {
		((Float4Vector) getValueVector()).setNull(getCount());
	} else {
		((Float4Vector) getValueVector()).setSafe(getCount(), (float) value.getField(ordinal));
	}
}
 
Example 14
Source Project: Alink   File: JsonReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
boolean read(Row row, Map <String, String> out) {
	String line = (String) row.getField(jsonColIndex);

	Map map = JsonConverter.fromJson(line, Map.class);

	map.forEach((key, value) -> {
		out.put(key.toString(), value.toString());

	});
	return true;
}
 
Example 15
Source Project: Alink   File: AlsModelDataConverter.java    License: Apache License 2.0 5 votes vote down vote up
public static long getVertexId(Row modelRow) {
    if (modelRow.getField(0) != null) {
        return (Long) modelRow.getField(0);
    } else {
        return (Long) modelRow.getField(1);
    }
}
 
Example 16
default BaseSample strInfo(Row row, int strIdx, int... keepIdxs){
    String str = (String) row.getField(strIdx);
    row = TableUtil.getRow(row, keepIdxs);
    return strInfo(str, row);
}
 
Example 17
default BaseSample textInfo(Row row, int strIdx, int... keepIdxs){
    String str = (String) row.getField(strIdx);
    row = TableUtil.getRow(row, keepIdxs);
    return textInfo(str, row);
}
 
Example 18
Source Project: flink   File: TypeExtractor.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "unchecked", "rawtypes" })
private <X> TypeInformation<X> privateGetForObject(X value) {
	checkNotNull(value);

	// check if type information can be produced using a factory
	final ArrayList<Type> typeHierarchy = new ArrayList<>();
	typeHierarchy.add(value.getClass());
	final TypeInformation<X> typeFromFactory = createTypeInfoFromFactory(value.getClass(), typeHierarchy, null, null);
	if (typeFromFactory != null) {
		return typeFromFactory;
	}

	// check if we can extract the types from tuples, otherwise work with the class
	if (value instanceof Tuple) {
		Tuple t = (Tuple) value;
		int numFields = t.getArity();
		if(numFields != countFieldsInClass(value.getClass())) {
			// not a tuple since it has more fields.
			return analyzePojo((Class<X>) value.getClass(), new ArrayList<Type>(), null, null, null); // we immediately call analyze Pojo here, because
			// there is currently no other type that can handle such a class.
		}

		TypeInformation<?>[] infos = new TypeInformation[numFields];
		for (int i = 0; i < numFields; i++) {
			Object field = t.getField(i);

			if (field == null) {
				throw new InvalidTypesException("Automatic type extraction is not possible on candidates with null values. "
						+ "Please specify the types directly.");
			}

			infos[i] = privateGetForObject(field);
		}
		return new TupleTypeInfo(value.getClass(), infos);
	}
	else if (value instanceof Row) {
		Row row = (Row) value;
		int arity = row.getArity();
		for (int i = 0; i < arity; i++) {
			if (row.getField(i) == null) {
				LOG.warn("Cannot extract type of Row field, because of Row field[" + i + "] is null. " +
					"Should define RowTypeInfo explicitly.");
				return privateGetForClass((Class<X>) value.getClass(), new ArrayList<Type>());
			}
		}
		TypeInformation<?>[] typeArray = new TypeInformation<?>[arity];
		for (int i = 0; i < arity; i++) {
			typeArray[i] = TypeExtractor.getForObject(row.getField(i));
		}
		return (TypeInformation<X>) new RowTypeInfo(typeArray);
	}
	else {
		return privateGetForClass((Class<X>) value.getClass(), new ArrayList<Type>());
	}
}
 
Example 19
@Override
protected boolean sendValues(Iterable<Row> values, long checkpointId, long timestamp) throws Exception {
	final AtomicInteger updatesCount = new AtomicInteger(0);
	final AtomicInteger updatesConfirmed = new AtomicInteger(0);

	final AtomicReference<Throwable> exception = new AtomicReference<>();

	FutureCallback<ResultSet> callback = new FutureCallback<ResultSet>() {
		@Override
		public void onSuccess(ResultSet resultSet) {
			updatesConfirmed.incrementAndGet();
			if (updatesCount.get() > 0) { // only set if all updates have been sent
				if (updatesCount.get() == updatesConfirmed.get()) {
					synchronized (updatesConfirmed) {
						updatesConfirmed.notifyAll();
					}
				}
			}
		}

		@Override
		public void onFailure(Throwable throwable) {
			if (exception.compareAndSet(null, throwable)) {
				LOG.error("Error while sending value.", throwable);
				synchronized (updatesConfirmed) {
					updatesConfirmed.notifyAll();
				}
			}
		}
	};

	//set values for prepared statement
	int updatesSent = 0;
	for (Row value : values) {
		for (int x = 0; x < arity; x++) {
			fields[x] = value.getField(x);
		}
		//insert values and send to cassandra
		BoundStatement s = preparedStatement.bind(fields);
		s.setDefaultTimestamp(timestamp);
		ResultSetFuture result = session.executeAsync(s);
		updatesSent++;
		if (result != null) {
			//add callback to detect errors
			Futures.addCallback(result, callback);
		}
	}
	updatesCount.set(updatesSent);

	synchronized (updatesConfirmed) {
		while (exception.get() == null && updatesSent != updatesConfirmed.get()) {
			updatesConfirmed.wait();
		}
	}

	if (exception.get() != null) {
		LOG.warn("Sending a value failed.", exception.get());
		return false;
	} else {
		return true;
	}
}
 
Example 20
Source Project: iceberg   File: FlinkParquetReaders.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected Object getField(Row row, int pos) {
  return row.getField(pos);
}