org.apache.flink.util.StringUtils Java Examples

The following examples show how to use org.apache.flink.util.StringUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: alibaba-flink-connectors   Author: alibaba   File: DatahubOutputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	if (null == datahubClientProvider) {
		if (!StringUtils.isNullOrWhitespaceOnly(accessId) && !StringUtils.isNullOrWhitespaceOnly(accessKey)) {
			datahubClientProvider = new DatahubClientProvider(endPoint, accessId, accessKey);
		} else {
			datahubClientProvider = new DatahubClientProvider(endPoint, properties);
		}
	}

	client = datahubClientProvider.getClient();

	for (ShardEntry shardEntry : client.listShard(projectName, topicName).getShards()) {
		if (shardEntry.getState() == ShardState.ACTIVE) {
			shardIdList.add(shardEntry.getShardId());
		}
	}
	shardNum = shardIdList.size();

	outTps = MetricUtils.registerOutTps(getRuntimeContext());
	outBps = MetricUtils.registerOutBps(getRuntimeContext(), "datahub");
	latencyGauge = MetricUtils.registerOutLatency(getRuntimeContext());
	scheduleFlusher();

	recordResolver.open();
}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void dropDatabase(String databaseName, boolean ignoreIfNotExists)
		throws DatabaseNotExistException, DatabaseNotEmptyException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName));

	if (databases.containsKey(databaseName)) {

		// Make sure the database is empty
		if (isDatabaseEmpty(databaseName)) {
			databases.remove(databaseName);
		} else {
			throw new DatabaseNotEmptyException(getName(), databaseName);
		}
	} else if (!ignoreIfNotExists) {
		throw new DatabaseNotExistException(getName(), databaseName);
	}
}
 
Example #3
Source Project: Alink   Author: alibaba   File: CsvParser.java    License: Apache License 2.0 6 votes vote down vote up
private Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        if (!enableQuote || token.charAt(0) != quoteChar) {
            return Tuple2.of(true, token);
        }
        String content;
        if (token.endsWith(quoteChar.toString())) {
            content = token.substring(1, token.length() - 1);
        } else {
            content = token.substring(1, token.length());
        }
        return Tuple2.of(true, content.replace(escapedQuote, quoteString));
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(true, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, fieldDelim.getBytes(), null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: StringSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMixedValues() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			null
		};
		
		testSerialization(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: Tuple21.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17, f18, f19, f20),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ "," + StringUtils.arrayAwareToString(this.f14)
		+ "," + StringUtils.arrayAwareToString(this.f15)
		+ "," + StringUtils.arrayAwareToString(this.f16)
		+ "," + StringUtils.arrayAwareToString(this.f17)
		+ "," + StringUtils.arrayAwareToString(this.f18)
		+ "," + StringUtils.arrayAwareToString(this.f19)
		+ "," + StringUtils.arrayAwareToString(this.f20)
		+ ")";
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: Tuple13.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ ")";
}
 
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: Tuple17.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ "," + StringUtils.arrayAwareToString(this.f14)
		+ "," + StringUtils.arrayAwareToString(this.f15)
		+ "," + StringUtils.arrayAwareToString(this.f16)
		+ ")";
}
 
Example #8
Source Project: flink   Author: apache   File: Tuple18.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ "," + StringUtils.arrayAwareToString(this.f14)
		+ "," + StringUtils.arrayAwareToString(this.f15)
		+ "," + StringUtils.arrayAwareToString(this.f16)
		+ "," + StringUtils.arrayAwareToString(this.f17)
		+ ")";
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: CheckpointCoordinator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds the given master hook to the checkpoint coordinator. This method does nothing, if
 * the checkpoint coordinator already contained a hook with the same ID (as defined via
 * {@link MasterTriggerRestoreHook#getIdentifier()}).
 *
 * @param hook The hook to add.
 * @return True, if the hook was added, false if the checkpoint coordinator already
 *         contained a hook with the same ID.
 */
public boolean addMasterHook(MasterTriggerRestoreHook<?> hook) {
	checkNotNull(hook);

	final String id = hook.getIdentifier();
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(id), "The hook has a null or empty id");

	synchronized (lock) {
		if (!masterHooks.containsKey(id)) {
			masterHooks.put(id, hook);
			return true;
		}
		else {
			return false;
		}
	}
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: Tuple21.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14, f15, f16, f17, f18, f19, f20),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ "," + StringUtils.arrayAwareToString(this.f14)
		+ "," + StringUtils.arrayAwareToString(this.f15)
		+ "," + StringUtils.arrayAwareToString(this.f16)
		+ "," + StringUtils.arrayAwareToString(this.f17)
		+ "," + StringUtils.arrayAwareToString(this.f18)
		+ "," + StringUtils.arrayAwareToString(this.f19)
		+ "," + StringUtils.arrayAwareToString(this.f20)
		+ ")";
}
 
Example #11
Source Project: Flink-CEPplus   Author: ljygz   File: Tuple15.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13, f14),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ "," + StringUtils.arrayAwareToString(this.f14)
		+ ")";
}
 
Example #12
Source Project: flink   Author: apache   File: Tuple14.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ "," + StringUtils.arrayAwareToString(this.f12)
		+ "," + StringUtils.arrayAwareToString(this.f13)
		+ ")";
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: BufferSpiller.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new {@link BufferSpiller}, spilling to one of the I/O manager's temp directories.
 *
 * @param ioManager The I/O manager for access to the temp directories.
 * @param pageSize The page size used to re-create spilled buffers.
 * @param maxBufferedBytes The maximum bytes to be buffered before the checkpoint aborts.
 * @param taskName The task name for logging.
 * @throws IOException Thrown if the temp files for spilling cannot be initialized.
 */
public BufferSpiller(IOManager ioManager, int pageSize, long maxBufferedBytes, String taskName) throws IOException {
	super(maxBufferedBytes, taskName);
	this.pageSize = pageSize;

	this.readBuffer = ByteBuffer.allocateDirect(READ_BUFFER_SIZE);
	this.readBuffer.order(ByteOrder.LITTLE_ENDIAN);

	this.headBuffer = ByteBuffer.allocateDirect(16);
	this.headBuffer.order(ByteOrder.LITTLE_ENDIAN);

	File[] tempDirs = ioManager.getSpillingDirectories();
	this.tempDir = tempDirs[DIRECTORY_INDEX.getAndIncrement() % tempDirs.length];

	byte[] rndBytes = new byte[32];
	ThreadLocalRandom.current().nextBytes(rndBytes);
	this.spillFilePrefix = StringUtils.byteToHexString(rndBytes) + '.';

	// prepare for first contents
	createSpillingChannel();
}
 
Example #14
Source Project: Flink-CEPplus   Author: ljygz   File: Tuple12.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ "," + StringUtils.arrayAwareToString(this.f11)
		+ ")";
}
 
Example #15
Source Project: flink   Author: apache   File: Tuple11.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a string representation of the tuple in the form
 * (f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10),
 * where the individual fields are the value returned by calling {@link Object#toString} on that field.
 * @return The string representation of the tuple.
 */
@Override
public String toString() {
	return "(" + StringUtils.arrayAwareToString(this.f0)
		+ "," + StringUtils.arrayAwareToString(this.f1)
		+ "," + StringUtils.arrayAwareToString(this.f2)
		+ "," + StringUtils.arrayAwareToString(this.f3)
		+ "," + StringUtils.arrayAwareToString(this.f4)
		+ "," + StringUtils.arrayAwareToString(this.f5)
		+ "," + StringUtils.arrayAwareToString(this.f6)
		+ "," + StringUtils.arrayAwareToString(this.f7)
		+ "," + StringUtils.arrayAwareToString(this.f8)
		+ "," + StringUtils.arrayAwareToString(this.f9)
		+ "," + StringUtils.arrayAwareToString(this.f10)
		+ ")";
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractGenericArraySerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleTypesObjects() {
	SimpleTypes a = new SimpleTypes();
	SimpleTypes b = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes c = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes d = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes e = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes f = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes g = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	
	runTests(new SimpleTypes[] {a, b, c}, new SimpleTypes[] {d, e, f, g});
}
 
Example #17
Source Project: Flink-CEPplus   Author: ljygz   File: TupleSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTuple1String() {
	Random rnd = new Random(68761564135413L);
	
	@SuppressWarnings({"unchecked", "rawtypes"})
	Tuple1<String>[] testTuples = new Tuple1[] {
		new Tuple1<String>(StringUtils.getRandomString(rnd, 10, 100)),
		new Tuple1<String>("abc"),
		new Tuple1<String>(""),
		new Tuple1<String>(StringUtils.getRandomString(rnd, 30, 170)),
		new Tuple1<String>(StringUtils.getRandomString(rnd, 15, 50)),
		new Tuple1<String>("")
	};
	
	runTests(-1, testTuples);
}
 
Example #18
Source Project: flink   Author: apache   File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 
Example #19
Source Project: flink   Author: flink-tpc-ds   File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Deletes the YARN application files, e.g., Flink binaries, libraries, etc., from the remote
 * filesystem.
 *
 * @param env The environment variables.
 */
public static void deleteApplicationFiles(final Map<String, String> env) {
	final String applicationFilesDir = env.get(YarnConfigKeys.FLINK_YARN_FILES);
	if (!StringUtils.isNullOrWhitespaceOnly(applicationFilesDir)) {
		final org.apache.flink.core.fs.Path path = new org.apache.flink.core.fs.Path(applicationFilesDir);
		try {
			final org.apache.flink.core.fs.FileSystem fileSystem = path.getFileSystem();
			if (!fileSystem.delete(path, true)) {
				LOG.error("Deleting yarn application files under {} was unsuccessful.", applicationFilesDir);
			}
		} catch (final IOException e) {
			LOG.error("Could not properly delete yarn application files directory {}.", applicationFilesDir, e);
		}
	} else {
		LOG.debug("No yarn application files directory set. Therefore, cannot clean up the data.");
	}
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractGenericTypeSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleTypesObjects() {
	SimpleTypes a = new SimpleTypes();
	SimpleTypes b = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes c = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes d = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes e = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes f = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());
	SimpleTypes g = new SimpleTypes(rnd.nextInt(), rnd.nextLong(), (byte) rnd.nextInt(),
			StringUtils.getRandomString(rnd, 10, 100), (short) rnd.nextInt(), rnd.nextDouble());

	runTests(a, b, c, d, e, f, g);
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: StringValueSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLongValues() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2)
		};
		
		testSerialization(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: StringValueSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMixedValues() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			""
		};
		
		testSerialization(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: StringSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLongValues() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2)
		};
		
		testSerialization(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: StringSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMixedValues() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			null
		};
		
		testSerialization(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: StringSerializationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBinaryCopyOfLongStrings() {
	try {
		String[] testStrings = new String[] {
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			null,
			"",
			StringUtils.getRandomString(rnd, 10000, 1024 * 1024 * 2),
			"",
			null
		};
		
		testCopy(testStrings);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Exception in test: " + e.getMessage());
	}
}
 
Example #26
Source Project: flink   Author: apache   File: UnresolvedIdentifier.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Constructs an {@link UnresolvedIdentifier} from an array of identifier segments.
 * The length of the path must be between 1 (only object name) and 3 (fully qualified
 * identifier with catalog, database and object name).
 *
 * @param path array of identifier segments
 * @return an identifier that must be resolved before accessing an object from a catalog manager
 */
public static UnresolvedIdentifier of(String... path) {
	if (path == null) {
		throw new ValidationException("Object identifier can not be null!");
	}
	if (path.length < 1 || path.length > 3) {
		throw new ValidationException("Object identifier must consist of 1 to 3 parts.");
	}
	if (Arrays.stream(path).anyMatch(StringUtils::isNullOrWhitespaceOnly)) {
		throw new ValidationException("Parts of the object identifier are null or whitespace-only.");
	}

	if (path.length == 3) {
		return new UnresolvedIdentifier(path[0], path[1], path[2]);
	} else if (path.length == 2) {
		return new UnresolvedIdentifier(null, path[0], path[1]);
	} else {
		return new UnresolvedIdentifier(null, null, path[0]);
	}
}
 
Example #27
Source Project: flink   Author: apache   File: SqlToOperationConverter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Converts language string to the FunctionLanguage.
 *
 * @param languageString  the language string from SQL parser
 * @return supported FunctionLanguage otherwise raise UnsupportedOperationException.
 * @throws UnsupportedOperationException if the languageString is not parsable or language is not supported
 */
private FunctionLanguage parseLanguage(String languageString) {
	if (StringUtils.isNullOrWhitespaceOnly(languageString)) {
		return FunctionLanguage.JAVA;
	}

	FunctionLanguage language;
	try {
		language = FunctionLanguage.valueOf(languageString);
	} catch (IllegalArgumentException e) {
		throw new UnsupportedOperationException(
			String.format("Unrecognized function language string %s", languageString), e);
	}

	return language;
}
 
Example #28
Source Project: flink   Author: apache   File: ClusterDataSetListResponseBodyTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFrom() {
	final Map<IntermediateDataSetID, DataSetMetaInfo> originalDataSets = new HashMap<>();
	originalDataSets.put(new IntermediateDataSetID(), DataSetMetaInfo.withNumRegisteredPartitions(1, 2));
	originalDataSets.put(new IntermediateDataSetID(), DataSetMetaInfo.withNumRegisteredPartitions(2, 2));

	List<ClusterDataSetEntry> convertedDataSets = ClusterDataSetListResponseBody.from(originalDataSets).getDataSets();
	assertThat(convertedDataSets, hasSize(2));
	for (ClusterDataSetEntry convertedDataSet : convertedDataSets) {
		IntermediateDataSetID id = new IntermediateDataSetID(new AbstractID(StringUtils.hexStringToByte(convertedDataSet.getDataSetId())));

		DataSetMetaInfo dataSetMetaInfo = originalDataSets.get(id);

		assertThat(convertedDataSet.isComplete(), is(dataSetMetaInfo.getNumRegisteredPartitions().orElse(0) == dataSetMetaInfo.getNumTotalPartitions()));
	}
}
 
Example #29
Source Project: flink   Author: apache   File: GenericInMemoryCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void alterDatabase(String databaseName, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
		throws DatabaseNotExistException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName));
	checkNotNull(newDatabase);

	CatalogDatabase existingDatabase = databases.get(databaseName);

	if (existingDatabase != null) {
		if (existingDatabase.getClass() != newDatabase.getClass()) {
			throw new CatalogException(
				String.format("Database types don't match. Existing database is '%s' and new database is '%s'.",
					existingDatabase.getClass().getName(), newDatabase.getClass().getName())
			);
		}

		databases.put(databaseName, newDatabase.copy());
	} else if (!ignoreIfNotExists) {
		throw new DatabaseNotExistException(getName(), databaseName);
	}
}
 
Example #30
Source Project: Alink   Author: alibaba   File: StringParsers.java    License: Apache License 2.0 6 votes vote down vote up
static Tuple2<Boolean, Object> parseField(FieldParser<?> parser, String token, boolean isStringField) {
    if (isStringField) {
        return Tuple2.of(true, token);
    } else {
        if (StringUtils.isNullOrWhitespaceOnly(token)) {
            return Tuple2.of(false, null);
        }
        byte[] bytes = token.getBytes();
        parser.resetErrorStateAndParse(bytes, 0, bytes.length, new byte[]{0}, null);
        FieldParser.ParseErrorState errorState = parser.getErrorState();
        if (errorState != FieldParser.ParseErrorState.NONE) {
            return Tuple2.of(false, null);
        } else {
            return Tuple2.of(true, parser.getLastResult());
        }
    }
}