Java Code Examples for org.apache.flink.types.Row#setField()

The following examples show how to use org.apache.flink.types.Row#setField() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DenseVectorSummary.java    From Alink with Apache License 2.0 6 votes vote down vote up
@Override
public String toString() {
    String[] outColNames = new String[]{"id", "count",
        "sum", "mean", "variance", "standardDeviation", "min", "max", "normL1", "normL2"};

    List<Row> data = new ArrayList<>();

    for (int i = 0; i < vectorSize(); i++) {
        Row row = new Row(outColNames.length);

        row.setField(0, i);
        row.setField(1, count);
        row.setField(2, sum(i));
        row.setField(3, mean(i));
        row.setField(4, variance(i));
        row.setField(5, standardDeviation(i));
        row.setField(6, min(i));
        row.setField(7, max(i));
        row.setField(8, normL1(i));
        row.setField(9, normL2(i));

        data.add(row);
    }

    return TableUtil.format(outColNames, data);
}
 
Example 2
Source File: RowSerializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public Row deserialize(DataInputView source) throws IOException {
	int len = fieldSerializers.length;

	Row result = new Row(len);

	// read null mask
	readIntoNullMask(len, source, nullMask);

	for (int i = 0; i < len; i++) {
		if (nullMask[i]) {
			result.setField(i, null);
		}
		else {
			result.setField(i, fieldSerializers[i].deserialize(source));
		}
	}

	return result;
}
 
Example 3
Source File: JsonRowSerializationSchemaTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNestedSchema() {
	final TypeInformation<Row> rowSchema = Types.ROW_NAMED(
		new String[] {"f1", "f2", "f3"},
		Types.INT, Types.BOOLEAN, Types.ROW(Types.INT, Types.DOUBLE));

	final Row row = new Row(3);
	row.setField(0, 42);
	row.setField(1, false);
	final Row nested = new Row(2);
	nested.setField(0, 22);
	nested.setField(1, 2.3);
	row.setField(2, nested);

	final JsonRowSerializationSchema serializationSchema = new JsonRowSerializationSchema.Builder(rowSchema)
		.build();
	final JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(rowSchema)
		.build();

	assertThat(row, whenSerializedWith(serializationSchema)
		.andDeserializedWith(deserializationSchema)
		.equalsTo(row));
}
 
Example 4
Source File: SparseVectorSummary.java    From Alink with Apache License 2.0 6 votes vote down vote up
@Override
public String toString() {
    List<Row> data = new ArrayList<>();

    for (int i = 0; i < vectorSize(); i++) {
        Row row = new Row(OUT_COL_NAMES.length);

        row.setField(0, i);
        row.setField(1, count);
        row.setField(2, sum(i));
        row.setField(3, mean(i));
        row.setField(4, variance(i));
        row.setField(5, standardDeviation(i));
        row.setField(6, min(i));
        row.setField(7, max(i));
        row.setField(8, normL1(i));
        row.setField(9, normL2(i));

        data.add(row);
    }

    return TableUtil.format(OUT_COL_NAMES, data);
}
 
Example 5
Source File: OutputColsHelper.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Merge the input row and the output row.
 *
 * @param input  The input row being predicted or transformed.
 * @param output The output row of the prediction/transformation operator.
 * @return The result row, which is a combination of preserved columns and the operator's
 * output columns.
 */
public Row getResultRow(Row input, Row output) {
	int numOutputs = outputColsPosInResult.length;
	if (output.getArity() != numOutputs) {
		throw new IllegalArgumentException("Invalid output size");
	}
	int resultLength = reservedCols.length + outputColNames.length;
	Row result = new Row(resultLength);
	for (int i = 0; i < reservedCols.length; i++) {
		result.setField(reservedColsPosInResult[i], input.getField(reservedCols[i]));
	}
	for (int i = 0; i < numOutputs; i++) {
		result.setField(outputColsPosInResult[i], output.getField(i));
	}
	return result;
}
 
Example 6
Source File: PulsarDeserializer.java    From pulsar-flink with Apache License 2.0 6 votes vote down vote up
private void writeMetadataFields(Message<?> message, Row row) {
    int metaStartIdx = row.getArity() - 5;

    if (message.hasKey()) {
        row.setField(metaStartIdx, message.getKeyBytes());
    } else {
        row.setField(metaStartIdx, null);
    }

    row.setField(metaStartIdx + 1, message.getTopicName());
    row.setField(metaStartIdx + 2, message.getMessageId().toByteArray());
    row.setField(metaStartIdx + 3, new Timestamp(message.getPublishTime()));

    if (message.getEventTime() > 0L) {
        row.setField(metaStartIdx + 4, new Timestamp(message.getEventTime()));
    } else {
        row.setField(metaStartIdx + 4, null);
    }
}
 
Example 7
Source File: FlinkPravegaTableITCase.java    From flink-connectors with Apache License 2.0 5 votes vote down vote up
static List<Row> getExpectedResultsAppend() {
    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.s");
    String[][] data = {
            { "Bob",   "2018-08-02 08:25:00.0", "1" },
            { "Chris", "2018-08-02 08:25:00.0", "1" },
            { "David", "2018-08-02 08:30:00.0", "1" },
            { "Gary",  "2018-08-02 09:35:00.0", "2" },
            { "Mary",  "2018-08-02 09:35:00.0", "2" },
            { "Nina",  "2018-08-02 09:40:00.0", "1" },
            { "Peter", "2018-08-02 09:45:00.0", "1" },
            { "Tony",  "2018-08-02 09:45:00.0", "1" },
            // The watermark is not passing the 10:45, so the window will not be fired and compute the count,
            // thus we don't have this line for output.
            //{ "Tony",  "2018-08-02 10:45:00.0", "1" },

    };
    List<Row> expectedResults = new ArrayList<>();
    for (int i = 0; i < data.length; i++) {
        Row row = new Row(3);
        row.setField(0, data[i][0]);
        row.setField(1, LocalDateTime.parse(data[i][1], formatter));
        row.setField(2, Long.valueOf(data[i][2]));
        expectedResults.add(row);
    }

    return expectedResults;
}
 
Example 8
Source File: LegacyRowSerializerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLargeRowSerializer() {
	RowTypeInfo typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO);

	Row row = new Row(13);
	row.setField(0, 2);
	row.setField(1, null);
	row.setField(3, null);
	row.setField(4, null);
	row.setField(5, null);
	row.setField(6, null);
	row.setField(7, null);
	row.setField(8, null);
	row.setField(9, null);
	row.setField(10, null);
	row.setField(11, null);
	row.setField(12, "Test");

	TypeSerializer<Row> serializer = typeInfo.createLegacySerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, row);
	testInstance.testAll();
}
 
Example 9
Source File: TransactionRowInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Row nextRecord(Row reuse) {
	Transaction transaction = transactions.next();
	reuse.setField(0, transaction.getAccountId());
	reuse.setField(1, new Timestamp(transaction.getTimestamp()));
	reuse.setField(2, transaction.getAmount());

	return reuse;
}
 
Example 10
Source File: HiveDB.java    From Alink with Apache License 2.0 5 votes vote down vote up
@Override
public Row map(BaseRow baseRow) throws Exception {
    Row row = new Row(baseRow.getArity());
    for (int i = 0; i < baseRow.getArity(); i++) {
        Object o = TypeGetterSetters.get(baseRow, i, dataTypes[i].getLogicalType());
        if (o != null && o instanceof BinaryString) {
            o = ((BinaryString) o).toString();
        }
        row.setField(i, o);
    }
    return row;
}
 
Example 11
Source File: RowComparatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static Row createRow(Object f0, Object f1, Object f2, Object f3, Object f4) {
	Row row = new Row(5);
	row.setField(0, f0);
	row.setField(1, f1);
	row.setField(2, f2);
	row.setField(3, f3);
	row.setField(4, f4);
	return row;
}
 
Example 12
Source File: RowUtil.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * merge obj and row, return a new row.
 */
public static Row merge(Object obj, Row rec1) {
    int n1 = rec1.getArity();
    Row ret = new Row(n1 + 1);
    ret.setField(0, obj);
    for (int i = 0; i < n1; ++i) {
        ret.setField(i + 1, rec1.getField(i));
    }
    return ret;
}
 
Example 13
Source File: TreeModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
@Override
public void reduce(Iterable <Row> input, Collector <Row> output) throws Exception {
	HashMap <Integer, Integer> finalMap = new HashMap <>(1000);

	for (Row value : input) {
		if ((Long)value.getField(0) == 0L || value.getField(2) != null) {
			continue;
		}
		String nodeDetail = (String) value.getField(1);
		TreeModelDataConverter.NodeSerializable lc = JsonConverter.fromJson(
			nodeDetail, TreeModelDataConverter.NodeSerializable.class);
		if (lc != null && lc.node != null) {
			int featureId = lc.node.getFeatureIndex();
			if (featureId < 0) {
				continue;
			}
			if (!finalMap.containsKey(featureId)) {
				finalMap.put(featureId, 1);
			} else {
				finalMap.put(featureId, finalMap.get(featureId) + 1);
			}
			//System.out.println("lc " + );
		}
	}
	for (HashMap.Entry <Integer, Integer> index : finalMap.entrySet()) {
		//System.out.println("key "+index.getKey()+" value "+index.getValue());
		Row row = new Row(2);
		row.setField(0, featureColNames[index.getKey()]);
		row.setField(1, (long) index.getValue());
		output.collect(row);
		//System.out.println("row "+row);
	}
}
 
Example 14
Source File: CassandraConnectorITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCassandraTableSink() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

	DataStreamSource<Row> source = env.fromCollection(rowCollection);

	tEnv.registerDataStreamInternal("testFlinkTable", source);
	tEnv.registerTableSink(
		"cassandraTable",
		new String[]{"f0", "f1", "f2"},
		new TypeInformation[]{Types.STRING, Types.INT, Types.INT},
		new CassandraAppendTableSink(builder, injectTableName(INSERT_DATA_QUERY)));

	tEnv.sqlQuery("select * from testFlinkTable").insertInto("cassandraTable");

	env.execute();
	ResultSet rs = session.execute(injectTableName(SELECT_DATA_QUERY));

	// validate that all input was correctly written to Cassandra
	List<Row> input = new ArrayList<>(rowCollection);
	List<com.datastax.driver.core.Row> output = rs.all();
	for (com.datastax.driver.core.Row o : output) {
		Row cmp = new Row(3);
		cmp.setField(0, o.getString(0));
		cmp.setField(1, o.getInt(2));
		cmp.setField(2, o.getInt(1));
		Assert.assertTrue("Row " + cmp + " was written to Cassandra but not in input.", input.remove(cmp));
	}
	Assert.assertTrue("The input data was not completely written to Cassandra", input.isEmpty());
}
 
Example 15
Source File: JdbcOutputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testExceptionOnClose() {
	String expectedMsg = "Writing records to JDBC failed.";
	try {
		jdbcOutputFormat = JdbcOutputFormat.buildJdbcOutputFormat()
			.setDrivername(DERBY_EBOOKSHOP_DB.getDriverClass())
			.setDBUrl(DERBY_EBOOKSHOP_DB.getUrl())
			.setQuery(String.format(INSERT_TEMPLATE, OUTPUT_TABLE))
			.setSqlTypes(new int[]{
				Types.INTEGER,
				Types.VARCHAR,
				Types.VARCHAR,
				Types.DOUBLE,
				Types.INTEGER})
			.finish();
		setRuntimeContext(jdbcOutputFormat, true);
		jdbcOutputFormat.open(0, 1);

		TestEntry entry = TEST_DATA[0];
		Row row = new Row(5);
		row.setField(0, entry.id);
		row.setField(1, entry.title);
		row.setField(2, entry.author);
		row.setField(3, entry.price);
		row.setField(4, entry.qty);
		jdbcOutputFormat.writeRecord(row);
		jdbcOutputFormat.writeRecord(row); // writing the same record twice must yield a unique key violation.

		jdbcOutputFormat.close();
	} catch (Exception e) {
		assertTrue(findThrowable(e, RuntimeException.class).isPresent());
		assertTrue(findThrowableWithMessage(e, expectedMsg).isPresent());
	}
}
 
Example 16
Source File: JsonRowSerializationSchemaTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testSchema() throws IOException {
	final TypeInformation<Row> rowSchema = JsonRowSchemaConverter.convert(
		"{" +
		"    type: 'object'," +
		"    properties: {" +
		"         id: { type: 'integer' }," +
		"         idNumber: { type: 'number' }," +
		"         idOrNull: { type: ['integer', 'null'] }," +
		"         name: { type: 'string' }," +
		"         date: { type: 'string', format: 'date' }," +
		"         time: { type: 'string', format: 'time' }," +
		"         timestamp: { type: 'string', format: 'date-time' }," +
		"         bytes: { type: 'string', contentEncoding: 'base64' }," +
		"         numbers: { type: 'array', items: { type: 'integer' } }," +
		"         strings: { type: 'array', items: { type: 'string' } }," +
		"         nested: { " +
		"             type: 'object'," +
		"             properties: { " +
		"                 booleanField: { type: 'boolean' }," +
		"                 decimalField: { type: 'number' }" +
		"             }" +
		"         }" +
		"    }" +
		"}");

	final Row row = new Row(11);
	row.setField(0, BigDecimal.valueOf(-333));
	row.setField(1, BigDecimal.valueOf(12.2222));
	row.setField(2, null);
	row.setField(3, "");
	row.setField(4, Date.valueOf("1990-10-14"));
	row.setField(5, Time.valueOf("12:12:43"));
	row.setField(6, Timestamp.valueOf("1990-10-14 12:12:43"));

	final byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	row.setField(7, bytes);
	final BigDecimal[] numbers = new BigDecimal[] {
		BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)};
	row.setField(8, numbers);
	final String[] strings = new String[] {"one", "two", "three"};
	row.setField(9, strings);
	final Row nestedRow = new Row(2);
	nestedRow.setField(0, true);
	nestedRow.setField(1, BigDecimal.valueOf(12));
	row.setField(10, nestedRow);

	final Row resultRow = serializeAndDeserialize(rowSchema, row);
	assertEquals(row, resultRow);
}
 
Example 17
Source File: JsonRowDeserializationSchemaTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testSchemaDeserialization() throws Exception {
	final BigDecimal id = BigDecimal.valueOf(1238123899121L);
	final String name = "asdlkjasjkdla998y1122";
	final byte[] bytes = new byte[1024];
	ThreadLocalRandom.current().nextBytes(bytes);
	final BigDecimal[] numbers = new BigDecimal[] {
		BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)};
	final String[] strings = new String[] {"one", "two", "three"};

	final ObjectMapper objectMapper = new ObjectMapper();

	// Root
	ObjectNode root = objectMapper.createObjectNode();
	root.put("id", id.longValue());
	root.putNull("idOrNull");
	root.put("name", name);
	root.put("date", "1990-10-14");
	root.put("time", "12:12:43Z");
	root.put("timestamp", "1990-10-14T12:12:43Z");
	root.put("bytes", bytes);
	root.putArray("numbers").add(1).add(2).add(3);
	root.putArray("strings").add("one").add("two").add("three");
	root.putObject("nested").put("booleanField", true).put("decimalField", 12);

	final byte[] serializedJson = objectMapper.writeValueAsBytes(root);

	JsonRowDeserializationSchema deserializationSchema = new JsonRowDeserializationSchema.Builder(
		"{" +
		"    type: 'object'," +
		"    properties: {" +
		"         id: { type: 'integer' }," +
		"         idOrNull: { type: ['integer', 'null'] }," +
		"         name: { type: 'string' }," +
		"         date: { type: 'string', format: 'date' }," +
		"         time: { type: 'string', format: 'time' }," +
		"         timestamp: { type: 'string', format: 'date-time' }," +
		"         bytes: { type: 'string', contentEncoding: 'base64' }," +
		"         numbers: { type: 'array', items: { type: 'integer' } }," +
		"         strings: { type: 'array', items: { type: 'string' } }," +
		"         nested: { " +
		"             type: 'object'," +
		"             properties: { " +
		"                 booleanField: { type: 'boolean' }," +
		"                 decimalField: { type: 'number' }" +
		"             }" +
		"         }" +
		"    }" +
		"}").build();

	final Row expected = new Row(10);
	expected.setField(0, id);
	expected.setField(1, null);
	expected.setField(2, name);
	expected.setField(3, Date.valueOf("1990-10-14"));
	expected.setField(4, Time.valueOf("12:12:43"));
	expected.setField(5, Timestamp.valueOf("1990-10-14 12:12:43"));
	expected.setField(6, bytes);
	expected.setField(7, numbers);
	expected.setField(8, strings);
	final Row nestedRow = new Row(2);
	nestedRow.setField(0, true);
	nestedRow.setField(1, BigDecimal.valueOf(12));
	expected.setField(9, nestedRow);

	assertThat(serializedJson, whenDeserializedWith(deserializationSchema).equalsTo(expected));
}
 
Example 18
Source File: AvroTestUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests all Avro data types as well as nested types for a specific record.
 */
public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getSpecificTestData() {
	final Address addr = Address.newBuilder()
		.setNum(42)
		.setStreet("Main Street 42")
		.setCity("Test City")
		.setState("Test State")
		.setZip("12345")
		.build();

	final Row rowAddr = new Row(5);
	rowAddr.setField(0, 42);
	rowAddr.setField(1, "Main Street 42");
	rowAddr.setField(2, "Test City");
	rowAddr.setField(3, "Test State");
	rowAddr.setField(4, "12345");

	final User user = User.newBuilder()
		.setName("Charlie")
		.setFavoriteNumber(null)
		.setFavoriteColor("blue")
		.setTypeLongTest(1337L)
		.setTypeDoubleTest(1.337d)
		.setTypeNullTest(null)
		.setTypeBoolTest(false)
		.setTypeArrayString(Arrays.asList("hello", "world"))
		.setTypeArrayBoolean(Arrays.asList(true, true, false))
		.setTypeNullableArray(null)
		.setTypeEnum(Colors.RED)
		.setTypeMap(Collections.singletonMap("test", 12L))
		.setTypeFixed(new Fixed16(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}))
		.setTypeUnion(12.0)
		.setTypeNested(addr)
		.setTypeBytes(ByteBuffer.allocate(10))
		.setTypeDate(LocalDate.parse("2014-03-01"))
		.setTypeTimeMillis(LocalTime.parse("12:12:12"))
		.setTypeTimeMicros(123456)
		.setTypeTimestampMillis(DateTime.parse("2014-03-01T12:12:12.321Z"))
		.setTypeTimestampMicros(123456L)
		// byte array must contain the two's-complement representation of the
		// unscaled integer value in big-endian byte order
		.setTypeDecimalBytes(ByteBuffer.wrap(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()))
		// array of length n can store at most
		// Math.floor(Math.log10(Math.pow(2, 8 * n - 1) - 1))
		// base-10 digits of precision
		.setTypeDecimalFixed(new Fixed2(BigDecimal.valueOf(2000, 2).unscaledValue().toByteArray()))
		.build();

	final Row rowUser = new Row(23);
	rowUser.setField(0, "Charlie");
	rowUser.setField(1, null);
	rowUser.setField(2, "blue");
	rowUser.setField(3, 1337L);
	rowUser.setField(4, 1.337d);
	rowUser.setField(5, null);
	rowUser.setField(6, false);
	rowUser.setField(7, new String[]{"hello", "world"});
	rowUser.setField(8, new Boolean[]{true, true, false});
	rowUser.setField(9, null);
	rowUser.setField(10, "RED");
	rowUser.setField(11, Collections.singletonMap("test", 12L));
	rowUser.setField(12, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16});
	rowUser.setField(13, 12.0);
	rowUser.setField(14, rowAddr);
	rowUser.setField(15, new byte[10]);
	rowUser.setField(16, Date.valueOf("2014-03-01"));
	rowUser.setField(17, Time.valueOf("12:12:12"));
	rowUser.setField(18, 123456);
	rowUser.setField(19, Timestamp.valueOf("2014-03-01 12:12:12.321"));
	rowUser.setField(20, 123456L);
	rowUser.setField(21, BigDecimal.valueOf(2000, 2));
	rowUser.setField(22, BigDecimal.valueOf(2000, 2));

	final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
	t.f0 = User.class;
	t.f1 = user;
	t.f2 = rowUser;

	return t;
}
 
Example 19
Source File: TestUtil.java    From flink with Apache License 2.0 4 votes vote down vote up
public static Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> getNestedRecordTestData() {
	final Bar bar = Bar.newBuilder()
		.setSpam(1L).build();

	final ArrayItem arrayItem = ArrayItem.newBuilder()
		.setType("color")
		.setValue(1L).build();

	final MapItem mapItem = MapItem.newBuilder()
		.setType("map")
		.setValue("hashMap").build();

	List<ArrayItem> nestedArray = new ArrayList<>();
	nestedArray.add(arrayItem);

	Map<CharSequence, MapItem> nestedMap = new HashMap<>();
	nestedMap.put("mapItem", mapItem);

	List<Long> longArray = new ArrayList<>();
	longArray.add(1L);

	List<CharSequence> stringArray = new ArrayList<>();
	stringArray.add("String");

	Long[] primitiveLongArray = {1L};
	String[] primitiveStringArray = {"String"};

	final NestedRecord nestedRecord = NestedRecord.newBuilder()
		.setBar(bar)
		.setNestedArray(nestedArray)
		.setStrArray(stringArray)
		.setNestedMap(nestedMap)
		.setArr(longArray).build();

	final Row barRow = new Row(1);
	barRow.setField(0, 1L);

	final Row arrayItemRow = new Row(2);
	arrayItemRow.setField(0, "color");
	arrayItemRow.setField(1, 1L);

	final Row mapItemRow = new Row(2);
	mapItemRow.setField(0, "map");
	mapItemRow.setField(1, "hashMap");

	Row[] nestedRowArray = {arrayItemRow};
	Map<String, Row> nestedRowMap = new HashMap<>();
	nestedRowMap.put("mapItem", mapItemRow);

	final Row nestedRow = new Row(7);
	nestedRow.setField(2, barRow);
	nestedRow.setField(3, primitiveLongArray);
	nestedRow.setField(4, primitiveStringArray);
	nestedRow.setField(5, nestedRowMap);
	nestedRow.setField(6, nestedRowArray);

	final Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> t = new Tuple3<>();
	t.f0 = NestedRecord.class;
	t.f1 = nestedRecord;
	t.f2 = nestedRow;

	return t;
}
 
Example 20
Source File: OneVsRestModelMapper.java    From Alink with Apache License 2.0 4 votes vote down vote up
@Override
public void loadModel(List<Row> modelRows) {
    Params meta = extractMeta(modelRows);
    int numClasses = meta.get(ModelParamName.NUM_CLASSES);
    String labelsStr = meta.get(ModelParamName.LABELS);
    String labelTypeName = meta.get(ModelParamName.LABEL_TYPE_NAME);
    String binClsClassName = meta.get(ModelParamName.BIN_CLS_CLASS_NAME);
    String[] modelColNames = meta.get(ModelParamName.MODEL_COL_NAMES);
    Integer[] modelColTypesInt = meta.get(ModelParamName.MODEL_COL_TYPES);
    TypeInformation[] modelColTypes = new TypeInformation[modelColTypesInt.length];
    for (int i = 0; i < modelColTypesInt.length; i++) {
        modelColTypes[i] = JdbcTypeConverter.getFlinkType(modelColTypesInt[i]);
    }
    this.predictors = new ArrayList<>(numClasses);
    this.labels = gson.fromJson(labelsStr, ArrayList.class);
    recoverLabelType(labels, labelTypeName);

    try {
        for (int i = 0; i < numClasses; i++) {
            List<Row> rows = new ArrayList<Row>();
            for (Row row : modelRows) {
                if (row.getField(2) == null) {
                    continue;
                }
                long id = (Long) row.getField(2);
                if ((long) (i) == id) {
                    Row subRow = new Row(row.getArity() - 4);
                    for (int j = 0; j < subRow.getArity(); j++) {
                        subRow.setField(j, row.getField(3 + j));
                    }
                    rows.add(subRow);
                }
            }
            TableSchema schema = new TableSchema(modelColNames, modelColTypes);
            RichModelMapper predictor = createModelPredictor(binClsClassName, schema, getDataSchema(), binClsPredParams,
                rows);
            this.predictors.add(predictor);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}