Java Code Examples for org.apache.hadoop.conf.Configuration#setDouble()

The following examples show how to use org.apache.hadoop.conf.Configuration#setDouble() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: Bats   File: YarnAppLauncherImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void setConfiguration(Configuration conf, String property, Object value)
{
  if (value instanceof Integer) {
    conf.setInt(property, (Integer)value);
  } else if (value instanceof Boolean) {
    conf.setBoolean(property, (Boolean)value);
  } else if (value instanceof Long) {
    conf.setLong(property, (Long)value);
  } else if (value instanceof Float) {
    conf.setFloat(property, (Float)value);
  } else if (value instanceof Double) {
    conf.setDouble(property, (Double)value);
  } else {
    conf.set(property, value.toString());
  }
}
 
Example 2
Source Project: datawave   File: ConfigurationHelper.java    License: Apache License 2.0 6 votes vote down vote up
public static void set(final Configuration conf, final String name, final Object value) {
    if (null == value) {
        return;
    }
    
    if (value instanceof String) {
        conf.set(name, (String) value);
    } else if (value instanceof Integer) {
        conf.setInt(name, ((Integer) value).intValue());
    } else if (value instanceof Long) {
        conf.setLong(name, ((Long) value).longValue());
    } else if (value instanceof Float) {
        conf.setFloat(name, ((Float) value).floatValue());
    } else if (value instanceof Double) {
        conf.setDouble(name, ((Double) value).doubleValue());
    } else if (value instanceof Boolean) {
        conf.setBoolean(name, ((Boolean) value).booleanValue());
    } else {
        throw (new IllegalArgumentException(value.getClass().getSimpleName() + " is unhandled."));
    }
}
 
Example 3
private void setConfiguration(Configuration conf, String property, Object value)
{
  if (value instanceof Integer) {
    conf.setInt(property, (Integer)value);
  } else if (value instanceof Boolean) {
    conf.setBoolean(property, (Boolean)value);
  } else if (value instanceof Long) {
    conf.setLong(property, (Long)value);
  } else if (value instanceof Float) {
    conf.setFloat(property, (Float)value);
  } else if (value instanceof Double) {
    conf.setDouble(property, (Double)value);
  } else {
    conf.set(property, value.toString());
  }
}
 
Example 4
Source Project: hbase   File: PerformanceEvaluation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Note that all subclasses of this class must provide a public constructor
 * that has the exact same list of arguments.
 */
TestBase(final Configuration conf, final TestOptions options, final Status status) {
  this.conf = conf;
  this.receiverHost = this.conf == null? null: SpanReceiverHost.getInstance(conf);
  this.opts = options;
  this.status = status;
  this.testName = this.getClass().getSimpleName();
  if (options.traceRate >= 1.0) {
    this.traceSampler = Sampler.ALWAYS;
  } else if (options.traceRate > 0.0) {
    conf.setDouble("hbase.sampler.fraction", options.traceRate);
    this.traceSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(conf));
  } else {
    this.traceSampler = Sampler.NEVER;
  }
  everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate));
  if (options.isValueZipf()) {
    this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.2);
  }
  LOG.info("Sampling 1 every " + everyN + " out of " + opts.perClientRunRows + " total rows.");
}
 
Example 5
private static void initConf(Configuration conf) {

    conf.setInt("hfile.format.version", 3);
    conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 0);
    conf.setInt("hbase.client.retries.number", 100);
    conf.setInt("hbase.hregion.max.filesize", 200000000);
    conf.setInt("hbase.hregion.memstore.flush.size", 800000);
    conf.setInt("hbase.hstore.blockingStoreFiles", 150);
    conf.setInt("hbase.hstore.compaction.throughput.lower.bound", 52428800);
    conf.setInt("hbase.hstore.compaction.throughput.higher.bound", 2 * 52428800);
    conf.setDouble("hbase.mob.compaction.fault.probability", failureProb);
    conf.set(MobStoreEngine.MOB_COMPACTOR_CLASS_KEY,
      FaultyMobStoreCompactor.class.getName());
    conf.setBoolean("hbase.table.sanity.checks", false);
    conf.setLong(MobConstants.MIN_AGE_TO_ARCHIVE_KEY, 20000);

  }
 
Example 6
Source Project: laser   File: LrIterationDriver.java    License: Apache License 2.0 5 votes vote down vote up
public static int run(String collection, Path input, Path output,
		Float regularizationFactor, Boolean addIntercept,
		Configuration baseConf) throws IOException, ClassNotFoundException,
		InterruptedException {
	Configuration conf = new Configuration(baseConf);
	if (null != addIntercept) {
		conf.setBoolean("lr.iteration.add.intercept", addIntercept);
	}
	if (null != regularizationFactor) {
		conf.setDouble("lr.iteration.regulariztion.factor",
				regularizationFactor);
	}

	conf.set("com.b5m.laser.msgpack.output.method", "update_online_model");

	Job job = Job.getInstance(conf);
	job.setJarByClass(LrIterationDriver.class);
	job.setJobName("logistic regression");

	FileInputFormat.setInputPaths(job, input);

	job.setOutputFormatClass(MsgpackOutputFormat.class);
	job.setOutputKeyClass(String.class);
	job.setOutputValueClass(LaserOnlineModel.class);

	LrIterationInputFormat.setNumMapTasks(job, 120);
	job.setInputFormatClass(LrIterationInputFormat.class);
	job.setMapperClass(LrIterationMapper.class);
	job.setNumReduceTasks(0);

	boolean succeeded = job.waitForCompletion(true);
	if (!succeeded) {
		throw new IllegalStateException("Job:logistic regression,  Failed!");
	}
	return 0;
}
 
Example 7
Source Project: geowave   File: BasicParameterHelper.java    License: Apache License 2.0 5 votes vote down vote up
private static final void setParameter(
    final Configuration config,
    final Class<?> scope,
    final Object val,
    final ParameterEnum configItem) {
  if (val != null) {
    if (val instanceof Long) {
      config.setLong(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Long) val));
    } else if (val instanceof Double) {
      config.setDouble(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Double) val));
    } else if (val instanceof Boolean) {
      config.setBoolean(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Boolean) val));
    } else if (val instanceof Integer) {
      config.setInt(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Integer) val));
    } else if (val instanceof Class) {
      config.setClass(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ((Class) val),
          ((Class) val));
    } else if (val instanceof byte[]) {
      config.set(
          GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()),
          ByteArrayUtils.byteArrayToString((byte[]) val));
    } else {
      config.set(GeoWaveConfiguratorBase.enumToConfKey(scope, configItem.self()), val.toString());
    }
  }
}
 
Example 8
Source Project: incubator-tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
private Configuration getConf() {
  Configuration conf = new Configuration(false);
  conf.set("test1", "value1");
  conf.setBoolean("test2", true);
  conf.setDouble("test3", 1.2345);
  conf.setInt("test4", 34567);
  conf.setLong("test5", 1234567890L);
  conf.setStrings("test6", "S1", "S2", "S3");
  return conf;
}
 
Example 9
@Test(timeout = 5000)
public void testReserveFractionConfigured() {
  Configuration conf = new Configuration(this.conf);
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_RESERVE_FRACTION, 0.5d);
  MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
  
  dist.setJvmMemory(10000l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
  
  // Third request - output
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
  dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
  
  dist.makeInitialAllocations();
  
  // Total available: 50% of 10K = 7000
  // 3 requests - 10K, 10K, 5K
  // Scale down to - 2000, 2000, 1000
  assertEquals(2000, e1Callback.assigned);
  assertEquals(2000, e2Callback.assigned);
  assertEquals(1000, e3Callback.assigned);
}
 
Example 10
Source Project: tez   File: TestTezUtils.java    License: Apache License 2.0 5 votes vote down vote up
private Configuration getConf() {
  Configuration conf = new Configuration(false);
  conf.set("test1", "value1");
  conf.setBoolean("test2", true);
  conf.setDouble("test3", 1.2345);
  conf.setInt("test4", 34567);
  conf.setLong("test5", 1234567890L);
  conf.setStrings("test6", "S1", "S2", "S3");
  return conf;
}
 
Example 11
Source Project: tez   File: TestSpeculation.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets default conf.
 */
@Before
public void setDefaultConf() {
  try {
    defaultConf = new Configuration(false);
    defaultConf.set("fs.defaultFS", "file:///");
    defaultConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    defaultConf.setBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, true);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 1);
    defaultConf.setFloat(
        ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 1);
    localFs = FileSystem.getLocal(defaultConf);
    String stagingDir =
        "target" + Path.SEPARATOR + TestSpeculation.class.getName()
            + "-tmpDir";
    defaultConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir);
    defaultConf.setClass(TezConfiguration.TEZ_AM_TASK_ESTIMATOR_CLASS,
        estimatorClass,
        TaskRuntimeEstimator.class);
    defaultConf.setInt(TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, 20);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, 0.2);
    defaultConf.setDouble(TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, 0.25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, 25);
    defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, 50);
    defaultConf.setInt(TezConfiguration.TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS, 2);
  } catch (IOException e) {
    throw new RuntimeException("init failure", e);
  }
}
 
Example 12
Source Project: tez   File: TestMemoryDistributor.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testReserveFractionConfigured() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d);
  MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
  
  dist.setJvmMemory(10000l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
  
  // Third request - output
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
  dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
  
  dist.makeInitialAllocations();
  
  // Total available: 50% of 10K = 7000
  // 3 requests - 10K, 10K, 5K
  // Scale down to - 2000, 2000, 1000
  assertEquals(2000, e1Callback.assigned);
  assertEquals(2000, e2Callback.assigned);
  assertEquals(1000, e3Callback.assigned);
}
 
Example 13
Source Project: hbase   File: TestTablesWithQuotas.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testInsufficientlyReportedTableFiltering() throws Exception {
  final Map<TableName,Integer> reportedRegions = new HashMap<>();
  final Map<TableName,Integer> actualRegions = new HashMap<>();
  final Configuration conf = HBaseConfiguration.create();
  conf.setDouble(QuotaObserverChore.QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY, 0.95);

  TableName tooFewRegionsTable = TableName.valueOf("tn1");
  TableName sufficientRegionsTable = TableName.valueOf("tn2");
  TableName tooFewRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
  TableName sufficientRegionsNamespaceTable = TableName.valueOf("ns1", "tn2");
  final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) {
    @Override
    Configuration getConfiguration() {
      return conf;
    }

    @Override
    int getNumRegions(TableName tableName) {
      return actualRegions.get(tableName);
    }

    @Override
    int getNumReportedRegions(TableName table, QuotaSnapshotStore<TableName> tableStore) {
      return reportedRegions.get(table);
    }
  };
  tables.addTableQuotaTable(tooFewRegionsTable);
  tables.addTableQuotaTable(sufficientRegionsTable);
  tables.addNamespaceQuotaTable(tooFewRegionsNamespaceTable);
  tables.addNamespaceQuotaTable(sufficientRegionsNamespaceTable);

  reportedRegions.put(tooFewRegionsTable, 5);
  actualRegions.put(tooFewRegionsTable, 10);
  reportedRegions.put(sufficientRegionsTable, 19);
  actualRegions.put(sufficientRegionsTable, 20);
  reportedRegions.put(tooFewRegionsNamespaceTable, 9);
  actualRegions.put(tooFewRegionsNamespaceTable, 10);
  reportedRegions.put(sufficientRegionsNamespaceTable, 98);
  actualRegions.put(sufficientRegionsNamespaceTable, 100);

  // Unused argument
  tables.filterInsufficientlyReportedTables(null);
  Set<TableName> filteredTablesWithTableQuotas = tables.getTableQuotaTables();
  assertEquals(Collections.singleton(sufficientRegionsTable), filteredTablesWithTableQuotas);
  Set<TableName> filteredTablesWithNamespaceQutoas = tables.getNamespaceQuotaTables();
  assertEquals(Collections.singleton(sufficientRegionsNamespaceTable), filteredTablesWithNamespaceQutoas);
}
 
Example 14
Source Project: hbase   File: TestHStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * If there are two running InMemoryFlushRunnable, the later InMemoryFlushRunnable
 * may change the versionedList. And the first InMemoryFlushRunnable will use the chagned
 * versionedList to remove the corresponding segments.
 * In short, there will be some segements which isn't in merge are removed.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRunDoubleMemStoreCompactors() throws IOException, InterruptedException {
  int flushSize = 500;
  Configuration conf = HBaseConfiguration.create();
  conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStoreWithCustomCompactor.class.getName());
  conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.25);
  MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.set(0);
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(flushSize));
  // Set the lower threshold to invoke the "MERGE" policy
  conf.set(MemStoreCompactionStrategy.COMPACTING_MEMSTORE_THRESHOLD_KEY, String.valueOf(0));
  init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family)
      .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build());
  byte[] value = Bytes.toBytes("thisisavarylargevalue");
  MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing();
  long ts = EnvironmentEdgeManager.currentTime();
  long seqId = 100;
  // older data whihc shouldn't be "seen" by client
  store.add(createCell(qf1, ts, seqId, value), memStoreSizing);
  store.add(createCell(qf2, ts, seqId, value), memStoreSizing);
  store.add(createCell(qf3, ts, seqId, value), memStoreSizing);
  assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY);
  storeFlushCtx.prepare();
  // This shouldn't invoke another in-memory flush because the first compactor thread
  // hasn't accomplished the in-memory compaction.
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing);
  assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  //okay. Let the compaction be completed
  MyMemStoreCompactor.START_COMPACTOR_LATCH.countDown();
  CompactingMemStore mem = (CompactingMemStore) ((HStore)store).memstore;
  while (mem.isMemStoreFlushingInMemory()) {
    TimeUnit.SECONDS.sleep(1);
  }
  // This should invoke another in-memory flush.
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing);
  assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get());
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
    String.valueOf(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE));
  storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
  storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
}
 
Example 15
@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScaling() {
  Configuration conf = new Configuration(this.conf);
  conf.setStrings(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_WEIGHTED_RATIOS,
      generateWeightStrings(2, 3, 6, 1, 1));
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezJobConfig.TEZ_RUNTIME_SCALE_TASK_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(ShuffledMergedInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(ShuffledUnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  TezOutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OnFileSortedOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 7000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Scale down to - 3000, 1000, 500, 1500
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(500, e3Callback.assigned);
  assertEquals(1500, e4Callback.assigned);
}
 
Example 16
Source Project: tez   File: TestFaultTolerance.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * In unpartitioned cartesian product, failure fraction should be #unique failure/#consumer that
 * depends on the src task. Here we test a 2x2 cartesian product and let 4th destination task fail.
 * The failure fraction limit is configured to be 0.25. So the failure fraction should be 1/2,
 * not 1/4.
 * @throws Exception
 */
@Test
public void testCartesianProduct() throws Exception {
  Configuration dagConf = new Configuration();
  dagConf.setDouble(TezConfiguration.TEZ_TASK_MAX_ALLOWED_OUTPUT_FAILURES_FRACTION, 0.25);
  DAG dag = DAG.create("dag");

  Configuration vertexConf = new Configuration();
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), 3);
  vertexConf.setInt(TestProcessor.getVertexConfName(
    TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3"), 5);
  UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(vertexConf);
  ProcessorDescriptor processorDescriptor =
    ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(vertexPayload);
  Vertex v1 = Vertex.create("v1", processorDescriptor, 2);
  Vertex v2 = Vertex.create("v2", processorDescriptor, 2);
  Vertex v3 = Vertex.create("v3", processorDescriptor);

  String[] sourceVertices = {"v1", "v2"};
  CartesianProductConfig cartesianProductConfig =
    new CartesianProductConfig(Arrays.asList(sourceVertices));
  TezConfiguration tezConf = new TezConfiguration();
  tezConf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS, 1);
  tezConf.setBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, false);
  UserPayload cartesianProductPayload =
    cartesianProductConfig.toUserPayload(tezConf);

  v3.setVertexManagerPlugin(
    VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName())
      .setUserPayload(cartesianProductPayload));

  EdgeManagerPluginDescriptor edgeManagerPluginDescriptor =
    EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName())
      .setUserPayload(cartesianProductPayload);

  Configuration inputConf = new Configuration();
  inputConf.setBoolean(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), 3);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), 0);
  inputConf.setInt(TestInput.getVertexConfName(
    TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0);
  UserPayload inputPayload = TezUtils.createUserPayloadFromConf(inputConf);
  EdgeProperty edgeProperty =
    EdgeProperty.create(edgeManagerPluginDescriptor, DataMovementType.CUSTOM,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(null),
      TestInput.getInputDesc(inputPayload));
  Edge e1 = Edge.create(v1, v3, edgeProperty);
  Edge e2 = Edge.create(v2, v3, edgeProperty);
  dag.addVertex(v1).addVertex(v2).addVertex(v3);
  dag.addEdge(e1).addEdge(e2);

  // run dag
  runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
 
Example 17
@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScaling() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 7000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Scale down to - 3000, 1000, 500, 1500
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(500, e3Callback.assigned);
  assertEquals(1500, e4Callback.assigned);
}
 
Example 18
@Test(timeout = 5000)
public void testWeightedScalingNonConcurrent() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
  System.err.println(Joiner.on(",").join(conf.getStringCollection(
      TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput)
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  // Fifth request - Processor
  MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
  ProcessorContext e5ProcContext = createTestProcessortContext();
  ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
  dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);

  dist.makeInitialAllocations();

  // Total available: 80% of 10K = 8000
  // 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
  // Overlap input and output memory
  assertEquals(5250, e1Callback.assigned);
  assertEquals(1750, e2Callback.assigned);
  assertEquals(2333, e3Callback.assigned);
  assertEquals(4666, e4Callback.assigned);
  assertEquals(1000, e5Callback.assigned);
}
 
Example 19
@Test(timeout = 5000)
public void testAdditionalReserveFractionWeightedScalingNonConcurrent() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 2, 3, 6, 1, 1));
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.025d);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX, 0.2d);

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput [weight 6]
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput [weight 2]
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput) [weight 1]
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput [weight 3]
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  dist.makeInitialAllocations();

  // Total available: 60% of 10K = 6000
  // 4 requests (weight) - 10K (6), 10K(2), 10K(1), 10K(3)
  // Overlap input and output memory
  assertEquals(4500, e1Callback.assigned);
  assertEquals(1500, e2Callback.assigned);
  assertEquals(1500, e3Callback.assigned);
  assertEquals(4500, e4Callback.assigned);
}
 
Example 20
@Test(timeout = 5000)
public void testWeightedScalingNonConcurrentInputsDisabled() throws TezException {
  Configuration conf = new Configuration(this.conf);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
  conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, false);
  conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
  conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS,
      WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
  System.err.println(Joiner.on(",").join(conf.getStringCollection(
      TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));

  MemoryDistributor dist = new MemoryDistributor(2, 2, conf);

  dist.setJvmMemory(10000l);

  // First request - ScatterGatherShuffleInput
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
  dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);

  // Second request - BroadcastInput
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
  dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

  // Third request - randomOutput (simulates MROutput)
  MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
  OutputContext e3OutputContext1 = createTestOutputContext();
  OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
  dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

  // Fourth request - OnFileSortedOutput
  MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
  OutputContext e4OutputContext2 = createTestOutputContext();
  OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
  dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

  // Fifth request - Processor
  MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
  ProcessorContext e5ProcContext = createTestProcessortContext();
  ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
  dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);

  dist.makeInitialAllocations();

  // Total available: 80% of 10K = 8000
  // 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
  // Overlap input and output memory
  assertEquals(3000, e1Callback.assigned);
  assertEquals(1000, e2Callback.assigned);
  assertEquals(2333, e3Callback.assigned);
  assertEquals(4666, e4Callback.assigned);
  assertEquals(1000, e5Callback.assigned);
}