Java Code Examples for org.apache.flink.api.java.tuple.Tuple2#of()

The following examples show how to use org.apache.flink.api.java.tuple.Tuple2#of() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StringParsers.java    From Alink with Apache License 2.0 6 votes vote down vote up
static Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        return Tuple2.of(true, token);
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(false, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[]{0}, null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 
Example 2
Source File: MetricUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static Tuple2<TaskManagerMetricGroup, MetricGroup> instantiateTaskManagerMetricGroup(
		MetricRegistry metricRegistry,
		String hostName,
		ResourceID resourceID,
		Optional<Time> systemResourceProbeInterval) {
	final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(
		metricRegistry,
		hostName,
		resourceID.toString());

	MetricGroup statusGroup = taskManagerMetricGroup.addGroup(METRIC_GROUP_STATUS_NAME);

	// Initialize the TM metrics
	instantiateStatusMetrics(statusGroup);

	if (systemResourceProbeInterval.isPresent()) {
		instantiateSystemMetrics(taskManagerMetricGroup, systemResourceProbeInterval.get());
	}
	return Tuple2.of(taskManagerMetricGroup, statusGroup);
}
 
Example 3
Source File: BlockingGrpcPubSubSubscriber.java    From flink with Apache License 2.0 6 votes vote down vote up
private Tuple2<List<String>, List<String>> splitAckIds(List<String> ackIds) {
	final int maxPayload = 500 * 1024; //little below 512k bytes to be on the safe side
	final int fixedOverheadPerCall = 100;
	final int overheadPerId = 3;

	int totalBytes = fixedOverheadPerCall;

	for (int i = 0; i < ackIds.size(); i++) {
		totalBytes += ackIds.get(i).length() + overheadPerId;
		if (totalBytes > maxPayload) {
			return Tuple2.of(ackIds.subList(0, i), ackIds.subList(i, ackIds.size()));
		}
	}

	return Tuple2.of(ackIds, emptyList());
}
 
Example 4
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the JobManager's hostname and port extracted from the given
 * {@link Configuration}.
 *
 * @param configuration Configuration to extract the JobManager's address from
 * @return The JobManager's hostname and port
 * @throws ConfigurationException if the JobManager's address cannot be extracted from the configuration
 */
public static Tuple2<String, Integer> getJobManagerAddress(Configuration configuration) throws ConfigurationException {

	final String hostname = configuration.getString(JobManagerOptions.ADDRESS);
	final int port = configuration.getInteger(JobManagerOptions.PORT);

	if (hostname == null) {
		throw new ConfigurationException("Config parameter '" + JobManagerOptions.ADDRESS +
			"' is missing (hostname/address of JobManager to connect to).");
	}

	if (port <= 0 || port >= 65536) {
		throw new ConfigurationException("Invalid value for '" + JobManagerOptions.PORT +
			"' (port of the JobManager actor system) : " + port +
			".  it must be greater than 0 and less than 65536.");
	}

	return Tuple2.of(hostname, port);
}
 
Example 5
Source File: IntervalJoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private StreamRecord<Tuple2<TestElem, TestElem>> streamRecordOf(
	long lhsTs,
	long rhsTs
) {
	TestElem lhs = new TestElem(lhsTs, "lhs");
	TestElem rhs = new TestElem(rhsTs, "rhs");

	long ts = Math.max(lhsTs, rhsTs);
	return new StreamRecord<>(Tuple2.of(lhs, rhs), ts);
}
 
Example 6
Source File: LdaModelDataConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
* Serialize the model data to "Tuple2<Params, Iterable<String>>".
*
* @param modelData The model data to serialize.
* @return The serialization result.
*/
  @Override
  public Tuple2<Params, Iterable<String>> serializeModel(LdaModelData modelData) {
      if (modelData.gamma != null) {
          return Tuple2.of(BuildMeta(modelData), serializeMatrix(Tuple2.of(modelData.gamma, modelData.list)));
      } else {
          return Tuple2.of(BuildMeta(modelData), serializeMatrix(Tuple2.of(modelData.wordTopicCounts, modelData.list)));
      }
  }
 
Example 7
Source File: IsotonicRegressionConverter.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Serialize the model data to "Tuple2<Params, Iterable<String>>".
 *
 * @param modelData The model data to serialize.
 * @return The serialization result.
 */
@Override
public Tuple2<Params, Iterable<String>> serializeModel(IsotonicRegressionModelData modelData) {
	Double[] boundaries = modelData.boundaries;
	Double[] values = modelData.values;
	Params meta = modelData.meta;
	List <String> data = new ArrayList <>();
	data.add(JsonConverter.toJson(boundaries));
	data.add(JsonConverter.toJson(values));
	return Tuple2.of(meta, data);
}
 
Example 8
Source File: AkkaRpcService.java    From flink with Apache License 2.0 5 votes vote down vote up
private Tuple2<String, String> extractAddressHostname(ActorRef actorRef) {
	final String actorAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef);
	final String hostname;
	Option<String> host = actorRef.path().address().host();
	if (host.isEmpty()) {
		hostname = "localhost";
	} else {
		hostname = host.get();
	}

	return Tuple2.of(actorAddress, hostname);
}
 
Example 9
Source File: AkkaRpcService.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private Tuple2<String, String> extractAddressHostname(ActorRef actorRef) {
	final String actorAddress = AkkaUtils.getAkkaURL(actorSystem, actorRef);
	final String hostname;
	Option<String> host = actorRef.path().address().host();
	if (host.isEmpty()) {
		hostname = "localhost";
	} else {
		hostname = host.get();
	}

	return Tuple2.of(actorAddress, hostname);
}
 
Example 10
Source File: GmmModelMapper.java    From Alink with Apache License 2.0 5 votes vote down vote up
@Override
protected Tuple2<Object, String> predictResultDetail(Row row) throws Exception {
    Vector sample = VectorUtil.getVector(row.getField(vectorColIdx));

    int k = modelData.k;
    double probSum = 0.;
    for (int i = 0; i < k; i++) {
        double density = this.multivariateGaussians[i].pdf(sample);
        double p = modelData.data.get(i).weight * density;
        prob[i] = p;
        probSum += p;
    }
    for (int i = 0; i < k; i++) {
        prob[i] /= probSum;
    }

    int maxIndex = 0;
    double maxProb = prob[0];

    for (int i = 1; i < k; i++) {
        if (prob[i] > maxProb) {
            maxProb = prob[i];
            maxIndex = i;
        }
    }

    return Tuple2.of((long) maxIndex, new DenseVector(prob).toString());
}
 
Example 11
Source File: SqlCommandParserTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private Tuple2<Boolean, SqlCommandCall> parseSqlAndCheckException(TestItem item) {
	SqlCommandCall call = null;
	Throwable actualException = null;
	try {
		call = SqlCommandParser.parse(parser, item.sql);
	} catch (Throwable e) {
		actualException = e;
	}

	if (item.expectedException == null && actualException == null) {
		return Tuple2.of(true, call);
	} else if (item.expectedException == null) {
		actualException.printStackTrace();
		fail("Failed to run sql: " + item.sql);
	} else if (actualException == null) {
		fail("the excepted exception: '" + item.expectedException + "' does not occur.\n" +
				"test statement: " + item.sql);
	} else {
		assertTrue(actualException.getClass().isAssignableFrom(item.expectedException));
		boolean hasExpectedExceptionMsg = false;
		while (actualException != null) {
			if (actualException.getMessage().contains(item.expectedExceptionMsg)) {
				hasExpectedExceptionMsg = true;
				break;
			}
			actualException = actualException.getCause();
		}
		if (!hasExpectedExceptionMsg) {
			fail("the excepted exception message: '" + item.expectedExceptionMsg + "' does not occur.\n" +
					"test statement: " + item.sql);
		}
	}
	return Tuple2.of(false, null);
}
 
Example 12
Source File: KMeansUtil.java    From Alink with Apache License 2.0 5 votes vote down vote up
/**
 * Find the closest cluster index.
 *
 * @param sample         query sample.
 * @param centroids      centroids.
 * @param k              cluster number.
 * @param distance       FastDistance.
 * @param distanceMatrix Preallocated distance matrix.
 * @return the closest cluster index and distance.
 */
public static Tuple2<Integer, Double> getClosestClusterIndex(FastDistanceVectorData sample,
                                                             FastDistanceMatrixData centroids,
                                                             int k,
                                                             FastDistance distance,
                                                             DenseMatrix distanceMatrix) {
    getClusterDistances(sample, centroids, distance, distanceMatrix);
    double[] data = distanceMatrix.getData();
    int index = getMinPointIndex(data, k);
    return Tuple2.of(index, data[index]);
}
 
Example 13
Source File: BinaryClassMetrics.java    From Alink with Apache License 2.0 4 votes vote down vote up
public Tuple2<double[], double[]> getF1ByThreshold() {
    return Tuple2.of(getParams().get(THRESHOLD_ARRAY), getParams().get(F1_ARRAY));
}
 
Example 14
Source File: TtlMapStateVerifier.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
@Nonnull
public Tuple2<String, String> generateRandomUpdate() {
	return Tuple2.of(KEYS.get(RANDOM.nextInt(KEYS.size())), randomString());
}
 
Example 15
Source File: BaseTuning.java    From Alink with Apache License 2.0 4 votes vote down vote up
protected Tuple2<Pipeline, Report> findBestCV(BatchOperator<?> in, int k, PipelineCandidatesBase candidates) {
	Preconditions.checkArgument(k > 1, "numFolds could be greater than 1.");
	DataSet<Tuple2<Integer, Row>> splitData = split(in, k);

	int nIter = candidates.size();
	Double bestAvg = null;
	Pipeline best = null;

	ArrayList<Double> experienceScores = new ArrayList<>(nIter);
	List<Report.ReportElement> reportElements = new ArrayList<>();
	for (int i = 0; i < nIter; i++) {
		Tuple2<Pipeline, List<Tuple3<Integer, ParamInfo, Object>>> cur;
		try {
			cur = candidates.get(i, experienceScores);
		} catch (CloneNotSupportedException e) {
			throw new RuntimeException(e);
		}

		double avg = kFoldCv(splitData, cur.f0, in.getSchema(), k);

		experienceScores.add(i, avg);

		if (Double.isNaN(avg)) {
			System.out.println(String.format("BestCV, i: %d, best: %f, avg: %f",
				i, bestAvg, avg));
			reportElements.add(
				new Report.ReportElement(
					cur.f0,
					cur.f1,
					avg
				)
			);
			continue;
		}

		reportElements.add(
			new Report.ReportElement(
				cur.f0,
				cur.f1,
				avg
			)
		);

		if (bestAvg == null) {
			bestAvg = avg;
			best = cur.f0;
		} else if ((tuningEvaluator.isLargerBetter() && bestAvg < avg)
			|| (!tuningEvaluator.isLargerBetter() && bestAvg > avg)) {
			bestAvg = avg;
			best = cur.f0;
		}

		System.out.println(String.format("BestCV, i: %d, best: %f, avg: %f",
			i, bestAvg, avg));

	}

	if (best == null) {
		throw new RuntimeException("Can not find a best model.");
	}

	return Tuple2.of(best, new Report(reportElements));
}
 
Example 16
Source File: AggregatingSubtasksMetricsHandlerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected Tuple2<String, List<String>> getFilter() {
	return Tuple2.of("subtasks", Arrays.asList("1", "3"));
}
 
Example 17
Source File: KeyedStateRow.java    From bravo with Apache License 2.0 4 votes vote down vote up
public Tuple2<Integer, String> getKeyGroupAndStateName(int maxParallelism) throws IOException {
	return Tuple2.of(getKeyGroup(maxParallelism), getStateName());
}
 
Example 18
Source File: GeoUtils.java    From infoworld-post with Apache License 2.0 4 votes vote down vote up
public Tuple2<Float, Float> eval(int cellId) {
    return Tuple2.of(
            GeoUtils.getGridCellCenterLon(cellId),
            GeoUtils.getGridCellCenterLat(cellId)
    );
}
 
Example 19
Source File: BinaryClassMetrics.java    From Alink with Apache License 2.0 4 votes vote down vote up
public Tuple2<double[], double[]> getRecallPrecisionCurve() {
    double[][] curve = getParams().get(RECALL_PRECISION_CURVE);
    return Tuple2.of(curve[0], curve[1]);
}
 
Example 20
Source File: KMeansUtil.java    From Alink with Apache License 2.0 3 votes vote down vote up
/**
 * Find the closest cluster index.
 *
 * @param trainModelData trainModel
 * @param sample         query sample
 * @param distance       ContinuousDistance
 * @return the index and distance.
 */
public static Tuple2<Integer, Double> getClosestClusterIndex(KMeansTrainModelData trainModelData,
                                                             Vector sample,
                                                             ContinuousDistance distance) {
    double[] distances = getClusterDistances(trainModelData, sample, distance);
    int index = getMinPointIndex(distances, trainModelData.params.k);
    return Tuple2.of(index, distances[index]);
}