org.apache.flink.configuration.OptimizerOptions Java Examples

The following examples show how to use org.apache.flink.configuration.OptimizerOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: DelimitedInputFormatSamplingTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void initialize() {
	try {
		testTempFolder = tempFolder.newFolder();
		// make sure we do 4 samples
		CONFIG = TestConfigUtils.loadGlobalConf(
			new String[] { OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key(),
							OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key() },
			new String[] { "4", "4" },
			testTempFolder);


	} catch (Throwable t) {
		Assert.fail("Could not load the global configuration.");
	}
}

Example #2

Source File: DelimitedInputFormatSamplingTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #3

Source File: DelimitedInputFormatSamplingTest.java From flink with Apache License 2.0

6 votes

@BeforeClass
public static void initialize() {
	try {
		testTempFolder = tempFolder.newFolder();
		// make sure we do 4 samples
		CONFIG = TestConfigUtils.loadGlobalConf(
			new String[] { OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key(),
							OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key() },
			new String[] { "4", "4" },
			testTempFolder);


	} catch (Throwable t) {
		Assert.fail("Could not load the global configuration.");
	}
}

Example #4

Source File: DelimitedInputFormatSamplingTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #5

Source File: DelimitedInputFormatSamplingTest.java From flink with Apache License 2.0

6 votes

@BeforeClass
public static void initialize() {
	try {
		testTempFolder = tempFolder.newFolder();
		// make sure we do 4 samples
		CONFIG = TestConfigUtils.loadGlobalConf(
			new String[] { OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key(),
							OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key() },
			new String[] { "4", "4" },
			testTempFolder);


	} catch (Throwable t) {
		Assert.fail("Could not load the global configuration.");
	}
}

Example #6

Source File: DelimitedInputFormatSamplingTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSamplingOverlyLongRecord() {
	try {
		final String tempFile = TestFileUtils.createTempFile(2 * OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue());
		final Configuration conf = new Configuration();
		
		final TestDelimitedInputFormat format = new TestDelimitedInputFormat(CONFIG);
		format.setFilePath(tempFile);
		format.configure(conf);
		
		Assert.assertNull("Expected exception due to overly long record.", format.getStatistics(null));
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example #7

Source File: DelimitedInputFormat.java From Flink-CEPplus with Apache License 2.0

5 votes

protected static void loadConfigParameters(Configuration parameters) {
	int maxSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES);
	int minSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES);
	
	if (maxSamples < 0) {
		LOG.error("Invalid default maximum number of line samples: " + maxSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key());
		maxSamples = OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.defaultValue();
	}
	if (minSamples < 0) {
		LOG.error("Invalid default minimum number of line samples: " + minSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key());
		minSamples = OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.defaultValue();
	}
	
	DEFAULT_MAX_NUM_SAMPLES = maxSamples;
	
	if (minSamples > maxSamples) {
		LOG.error("Default minimum number of line samples cannot be greater the default maximum number " +
				"of line samples: min=" + minSamples + ", max=" + maxSamples + ". Defaulting minimum to maximum.");
		DEFAULT_MIN_NUM_SAMPLES = maxSamples;
	} else {
		DEFAULT_MIN_NUM_SAMPLES = minSamples;
	}
	
	int maxLen = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN);
	if (maxLen <= 0) {
		maxLen = OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue();
		LOG.error("Invalid value for the maximum sample record length. Using default value of " + maxLen + '.');
	} else if (maxLen < DEFAULT_READ_BUFFER_SIZE) {
		maxLen = DEFAULT_READ_BUFFER_SIZE;
		LOG.warn("Increasing maximum sample record length to size of the read buffer (" + maxLen + ").");
	}
	MAX_SAMPLE_LEN = maxLen;
}

Example #8

Source File: DelimitedInputFormat.java From flink with Apache License 2.0

5 votes

protected static void loadConfigParameters(Configuration parameters) {
	int maxSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES);
	int minSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES);
	
	if (maxSamples < 0) {
		LOG.error("Invalid default maximum number of line samples: " + maxSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key());
		maxSamples = OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.defaultValue();
	}
	if (minSamples < 0) {
		LOG.error("Invalid default minimum number of line samples: " + minSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key());
		minSamples = OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.defaultValue();
	}
	
	DEFAULT_MAX_NUM_SAMPLES = maxSamples;
	
	if (minSamples > maxSamples) {
		LOG.error("Default minimum number of line samples cannot be greater the default maximum number " +
				"of line samples: min=" + minSamples + ", max=" + maxSamples + ". Defaulting minimum to maximum.");
		DEFAULT_MIN_NUM_SAMPLES = maxSamples;
	} else {
		DEFAULT_MIN_NUM_SAMPLES = minSamples;
	}
	
	int maxLen = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN);
	if (maxLen <= 0) {
		maxLen = OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue();
		LOG.error("Invalid value for the maximum sample record length. Using default value of " + maxLen + '.');
	} else if (maxLen < DEFAULT_READ_BUFFER_SIZE) {
		maxLen = DEFAULT_READ_BUFFER_SIZE;
		LOG.warn("Increasing maximum sample record length to size of the read buffer (" + maxLen + ").");
	}
	MAX_SAMPLE_LEN = maxLen;
}

Example #9

Source File: DelimitedInputFormat.java From flink with Apache License 2.0

5 votes

protected static void loadConfigParameters(Configuration parameters) {
	int maxSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES);
	int minSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES);
	
	if (maxSamples < 0) {
		LOG.error("Invalid default maximum number of line samples: " + maxSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key());
		maxSamples = OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.defaultValue();
	}
	if (minSamples < 0) {
		LOG.error("Invalid default minimum number of line samples: " + minSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key());
		minSamples = OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.defaultValue();
	}
	
	DEFAULT_MAX_NUM_SAMPLES = maxSamples;
	
	if (minSamples > maxSamples) {
		LOG.error("Default minimum number of line samples cannot be greater the default maximum number " +
				"of line samples: min=" + minSamples + ", max=" + maxSamples + ". Defaulting minimum to maximum.");
		DEFAULT_MIN_NUM_SAMPLES = maxSamples;
	} else {
		DEFAULT_MIN_NUM_SAMPLES = minSamples;
	}
	
	int maxLen = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN);
	if (maxLen <= 0) {
		maxLen = OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue();
		LOG.error("Invalid value for the maximum sample record length. Using default value of " + maxLen + '.');
	} else if (maxLen < DEFAULT_READ_BUFFER_SIZE) {
		maxLen = DEFAULT_READ_BUFFER_SIZE;
		LOG.warn("Increasing maximum sample record length to size of the read buffer (" + maxLen + ").");
	}
	MAX_SAMPLE_LEN = maxLen;
}