Java Code Examples for org.apache.hadoop.conf.Configuration.setInt()

The following are Jave code examples for showing how to use setInt() of the org.apache.hadoop.conf.Configuration class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: hadoop-oss   File: HAServiceTarget.java   View Source Code Vote up 9 votes
private HAServiceProtocol getProxyForAddress(Configuration conf,
    int timeoutMs, InetSocketAddress addr) throws IOException {
  Configuration confCopy = new Configuration(conf);
  // Lower the timeout so we quickly fail to connect
  confCopy.setInt(
      CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
  SocketFactory factory = NetUtils.getDefaultSocketFactory(confCopy);
  return new HAServiceProtocolClientSideTranslatorPB(
      addr,
      confCopy, factory, timeoutMs);
}
 
Example 2
Project: hadoop   File: TestBootstrapStandbyWithBKJM.java   View Source Code Vote up 6 votes
@Before
public void setUp() throws Exception {
  Configuration conf = new Configuration();
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 5);
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, BKJMUtil
      .createJournalURI("/bootstrapStandby").toString());
  BKJMUtil.addJournalManagerDefinition(conf);
  conf.setBoolean(DFSConfigKeys.DFS_IMAGE_COMPRESS_KEY, true);
  conf.set(DFSConfigKeys.DFS_IMAGE_COMPRESSION_CODEC_KEY,
      SlowCodec.class.getCanonicalName());
  CompressionCodecFactory.setCodecClasses(conf,
      ImmutableList.<Class> of(SlowCodec.class));
  MiniDFSNNTopology topology = new MiniDFSNNTopology()
      .addNameservice(new MiniDFSNNTopology.NSConf("ns1").addNN(
          new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10001)).addNN(
          new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10002)));
  cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology)
      .numDataNodes(1).manageNameDfsSharedDirs(false).build();
  cluster.waitActive();
}
 
Example 3
Project: hadoop   File: TestRMWebServicesDelegationTokens.java   View Source Code Vote up 6 votes
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rm = new MockRM(rmconf);
  bind(ResourceManager.class).toInstance(rm);
  if (isKerberosAuth == true) {
    filter("/*").through(TestKerberosAuthFilter.class);
  } else {
    filter("/*").through(TestSimpleAuthFilter.class);
  }
  serve("/*").with(GuiceContainer.class);
}
 
Example 4
Project: ditb   File: TestSnapshotFromMaster.java   View Source Code Vote up 6 votes
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 2);
  conf.setInt("hbase.hstore.compactionThreshold", 5);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
  conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
  conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, "");
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());

}
 
Example 5
Project: aliyun-maxcompute-data-collectors   File: ExportJobBase.java   View Source Code Vote up 6 votes
@Override
protected void propagateOptionsToJob(Job job) {
  super.propagateOptionsToJob(job);
  Configuration conf = job.getConfiguration();

  // This is export job where re-trying failed mapper mostly don't make sense. By
  // default we will force MR to run only one attempt per mapper. User or connector
  // developer can override this behavior by setting SQOOP_EXPORT_MAP_TASK_MAX_ATTEMTPS:
  //
  // * Positive number - we will allow specified number of attempts
  // * Negative number - we will default to Hadoop's default number of attempts
  //
  // This is important for most connectors as they are directly committing data to
  // final table and hence re-running one mapper will lead to a misleading errors
  // of inserting duplicate rows.
  int sqoopMaxAttempts = conf.getInt(SQOOP_EXPORT_MAP_TASK_MAX_ATTEMTPS, 1);
  if (sqoopMaxAttempts > 1) {
    conf.setInt(HADOOP_MAP_TASK_MAX_ATTEMTPS, sqoopMaxAttempts);
  }
}
 
Example 6
Project: hadoop   File: TestStandbyCheckpoints.java   View Source Code Vote up 5 votes
@SuppressWarnings("rawtypes")
@Before
public void setupCluster() throws Exception {
  Configuration conf = setupCommonConfig();

  // Dial down the retention of extra edits and checkpoints. This is to
  // help catch regressions of HDFS-4238 (SBN should not purge shared edits)
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0);

  int retryCount = 0;
  while (true) {
    try {
      int basePort = 10060 + random.nextInt(100) * 2;
      MiniDFSNNTopology topology = new MiniDFSNNTopology()
          .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
              .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(basePort))
              .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(basePort + 1)));

      cluster = new MiniDFSCluster.Builder(conf)
          .nnTopology(topology)
          .numDataNodes(1)
          .build();
      cluster.waitActive();

      nn0 = cluster.getNameNode(0);
      nn1 = cluster.getNameNode(1);
      fs = HATestUtil.configureFailoverFs(cluster, conf);

      cluster.transitionToActive(0);
      ++retryCount;
      break;
    } catch (BindException e) {
      LOG.info("Set up MiniDFSCluster failed due to port conflicts, retry "
          + retryCount + " times");
    }
  }
}
 
Example 7
Project: hadoop   File: TestDatanodeRestart.java   View Source Code Vote up 5 votes
public void testRbwReplicas() throws IOException {
  Configuration conf = new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024L);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY, 512);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();
  try {
    testRbwReplicas(cluster, false);
    testRbwReplicas(cluster, true);
  } finally {
    cluster.shutdown();
  }
}
 
Example 8
Project: hadoop   File: TestLineRecordReader.java   View Source Code Vote up 5 votes
@Test
public void testStripBOM() throws IOException {
  // the test data contains a BOM at the start of the file
  // confirm the BOM is skipped by LineRecordReader
  String UTF8_BOM = "\uFEFF";
  URL testFileUrl = getClass().getClassLoader().getResource("testBOM.txt");
  assertNotNull("Cannot find testBOM.txt", testFileUrl);
  File testFile = new File(testFileUrl.getFile());
  Path testFilePath = new Path(testFile.getAbsolutePath());
  long testFileSize = testFile.length();
  Configuration conf = new Configuration();
  conf.setInt(org.apache.hadoop.mapreduce.lib.input.
      LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);

  TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());

  // read the data and check whether BOM is skipped
  FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
      (String[])null);
  LineRecordReader reader = new LineRecordReader();
  reader.initialize(split, context);
  int numRecords = 0;
  boolean firstLine = true;
  boolean skipBOM = true;
  while (reader.nextKeyValue()) {
    if (firstLine) {
      firstLine = false;
      if (reader.getCurrentValue().toString().startsWith(UTF8_BOM)) {
        skipBOM = false;
      }
    }
    ++numRecords;
  }
  reader.close();

  assertTrue("BOM is not skipped", skipBOM);
}
 
Example 9
Project: ytk-learn   File: HadoopTrainWorker.java   View Source Code Vote up 5 votes
public HadoopTrainWorker(Configuration conf,
                         String modelName,
                         String configPath,
                         String configFile,
                         String pyTransformScript,
                         boolean needPyTransform,
                         String loginName,
                         String hostName,
                         int hostPort,
                         int slaveNum,
                         int threadNum,
                         String hadoopQueueName,
                         String hadoopReduceMemory,
                         String user) throws Exception {
    super(modelName, configPath, configFile, pyTransformScript, needPyTransform, loginName, hostName, hostPort, threadNum);
    this.conf = conf;
    this.slaveNum = slaveNum;
    this.hadoopQueueName = hadoopQueueName;
    this.hadoopReduceMemory = hadoopReduceMemory;
    this.user = user;

    conf.set("mapreduce.task.timeout", "720000000");
    conf.set("modelName", modelName);
    conf.set("configFile", configFile);
    conf.set("pyTransformScript", pyTransformScript);
    conf.setBoolean("needPyTransform", needPyTransform);
    conf.set("loginName", loginName);
    conf.set("hostName", hostName);
    conf.setInt("hostPort", hostPort);
    conf.setInt("slaveNum", slaveNum);
    conf.setInt("threadNum", threadNum);

    conf.set("mapreduce.job.queuename", hadoopQueueName);
    conf.set("mapreduce.reduce.memory.mb", hadoopReduceMemory);
    conf.set("mapreduce.reduce.java.opts", "-Xmx" + ((int)((Integer.parseInt(hadoopReduceMemory) * 0.9))) + "m");
    conf.set("yarn.app.mapreduce.am.resource.mb", "" + threadNum);
    conf.set("mapreduce.reduce.cpu.vcores", "" + threadNum);
}
 
Example 10
Project: circus-train   File: AwsS3ClientFactoryTest.java   View Source Code Vote up 5 votes
@Test
public void typical() {
  Configuration conf = new Configuration();
  conf.set(ConfigurationVariable.REGION.getName(), "eu-west-1");
  conf.setInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), 7);
  conf.setLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), 333L);
  AmazonS3 client = factory.newInstance(conf);
  assertThat(client, is(instanceOf(AmazonS3Client.class)));
  assertThat(client.getRegion(), is(Region.EU_Ireland));
}
 
Example 11
Project: hadoop   File: TestWasbFsck.java   View Source Code Vote up 5 votes
private void runFsck(String command) throws Exception {
  Configuration conf = fs.getConf();
  // Set the dangling cutoff to zero, so every temp blob is considered
  // dangling.
  conf.setInt(NativeAzureFileSystem.AZURE_TEMP_EXPIRY_PROPERTY_NAME, 0);
  WasbFsck fsck = new WasbFsck(conf);
  fsck.setMockFileSystemForTesting(fs);
  fsck.run(new String[] { AzureBlobStorageTestAccount.MOCK_WASB_URI, command });
}
 
Example 12
Project: hadoop-oss   File: TestProtoBufRpc.java   View Source Code Vote up 5 votes
@Before
public void setUp() throws IOException { // Setup server for both protocols
  conf = new Configuration();
  conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, 1024);
  conf.setBoolean(CommonConfigurationKeys.IPC_SERVER_LOG_SLOW_RPC, true);
  // Set RPC engine to protobuf RPC engine
  RPC.setProtocolEngine(conf, TestRpcService.class, ProtobufRpcEngine.class);
  RPC.setProtocolEngine(conf, TestRpcService2.class, ProtobufRpcEngine.class);

  // Create server side implementation
  PBServerImpl serverImpl = new PBServerImpl();
  BlockingService service = TestProtobufRpcProto
      .newReflectiveBlockingService(serverImpl);

  // Get RPC server for server side implementation
  server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
      .setInstance(service).setBindAddress(ADDRESS).setPort(PORT).build();
  addr = NetUtils.getConnectAddress(server);

  // now the second protocol
  PBServer2Impl server2Impl = new PBServer2Impl();
  BlockingService service2 = TestProtobufRpc2Proto
      .newReflectiveBlockingService(server2Impl);

  server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService2.class,
      service2);
  server.start();
}
 
Example 13
Project: ditb   File: TestMasterCoprocessorExceptionWithAbort.java   View Source Code Vote up 5 votes
@BeforeClass
public static void setupBeforeClass() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      BuggyMasterObserver.class.getName());
  conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);  // Fail fast
  UTIL.startMiniCluster();
}
 
Example 14
Project: ditb   File: TestStripeCompactionPolicy.java   View Source Code Vote up 5 votes
private static StripeCompactionPolicy createPolicy(Configuration conf,
    long splitSize, float splitCount, int initialCount, boolean hasTtl) throws Exception {
  conf.setLong(StripeStoreConfig.SIZE_TO_SPLIT_KEY, splitSize);
  conf.setFloat(StripeStoreConfig.SPLIT_PARTS_KEY, splitCount);
  conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, initialCount);
  StoreConfigInformation sci = mock(StoreConfigInformation.class);
  when(sci.getStoreFileTtl()).thenReturn(hasTtl ? defaultTtl : Long.MAX_VALUE);
  StripeStoreConfig ssc = new StripeStoreConfig(conf, sci);
  return new StripeCompactionPolicy(conf, sci, ssc);
}
 
Example 15
Project: hadoop   File: TestFileCreationClient.java   View Source Code Vote up 4 votes
/** Test lease recovery Triggered by DFSClient. */
@Test
public void testClientTriggeredLeaseRecovery() throws Exception {
  final int REPLICATION = 3;
  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 1);
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION).build();

  try {
    final FileSystem fs = cluster.getFileSystem();
    final Path dir = new Path("/wrwelkj");
    
    SlowWriter[] slowwriters = new SlowWriter[10];
    for(int i = 0; i < slowwriters.length; i++) {
      slowwriters[i] = new SlowWriter(fs, new Path(dir, "file" + i));
    }

    try {
      for(int i = 0; i < slowwriters.length; i++) {
        slowwriters[i].start();
      }

      Thread.sleep(1000);                       // let writers get started

      //stop a datanode, it should have least recover.
      cluster.stopDataNode(AppendTestUtil.nextInt(REPLICATION));
      
      //let the slow writer writes a few more seconds
      System.out.println("Wait a few seconds");
      Thread.sleep(5000);
    }
    finally {
      for(int i = 0; i < slowwriters.length; i++) {
        if (slowwriters[i] != null) {
          slowwriters[i].running = false;
          slowwriters[i].interrupt();
        }
      }
      for(int i = 0; i < slowwriters.length; i++) {
        if (slowwriters[i] != null) {
          slowwriters[i].join();
        }
      }
    }

    //Verify the file
    System.out.println("Verify the file");
    for(int i = 0; i < slowwriters.length; i++) {
      System.out.println(slowwriters[i].filepath + ": length="
          + fs.getFileStatus(slowwriters[i].filepath).getLen());
      FSDataInputStream in = null;
      try {
        in = fs.open(slowwriters[i].filepath);
        for(int j = 0, x; (x = in.read()) != -1; j++) {
          assertEquals(j, x);
        }
      }
      finally {
        IOUtils.closeStream(in);
      }
    }
  } finally {
    if (cluster != null) {cluster.shutdown();}
  }
}
 
Example 16
Project: hadoop   File: TestTaskAttempt.java   View Source Code Vote up 4 votes
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1, 0)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
 
Example 17
Project: ditb   File: HFilePerformanceEvaluation.java   View Source Code Vote up 4 votes
/**
 * Add any supported codec or cipher to test the HFile read/write performance. 
 * Specify "none" to disable codec or cipher or both.  
 * @throws Exception
 */
private void runBenchmarks() throws Exception {
  final Configuration conf = new Configuration();
  final FileSystem fs = FileSystem.get(conf);
  final Path mf = fs.makeQualified(new Path("performanceevaluation.mapfile"));
  
  // codec=none cipher=none
  runWriteBenchmark(conf, fs, mf, "none", "none");
  runReadBenchmark(conf, fs, mf, "none", "none");
  
  // codec=gz cipher=none
  runWriteBenchmark(conf, fs, mf, "gz", "none");
  runReadBenchmark(conf, fs, mf, "gz", "none");

  // Add configuration for AES cipher
  final Configuration aesconf = new Configuration();
  aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
  aesconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
  aesconf.setInt("hfile.format.version", 3);
  final FileSystem aesfs = FileSystem.get(aesconf);
  final Path aesmf = aesfs.makeQualified(new Path("performanceevaluation.aes.mapfile"));

  // codec=none cipher=aes
  runWriteBenchmark(aesconf, aesfs, aesmf, "none", "aes");
  runReadBenchmark(aesconf, aesfs, aesmf, "none", "aes");

  // codec=gz cipher=aes
  runWriteBenchmark(aesconf, aesfs, aesmf, "gz", "aes");
  runReadBenchmark(aesconf, aesfs, aesmf, "gz", "aes");

  // cleanup test files
  if (fs.exists(mf)) {
    fs.delete(mf, true);
  }
  if (aesfs.exists(aesmf)) {
    aesfs.delete(aesmf, true);
  }
  
  // Print Result Summary
  LOG.info("\n***************\n" + "Result Summary" + "\n***************\n");
  LOG.info(testSummary.toString());

}
 
Example 18
Project: angel   File: ServerPartitionTest.java   View Source Code Vote up 4 votes
@Test
public void testWriteTo() throws Exception {
  // set basic configuration keys
  conf = new Configuration();
  conf.setBoolean("mapred.mapper.new-api", true);
  conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
  conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());

  // use local deploy mode and dummy dataspliter
  conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
  conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
  conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
  conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
  conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
  conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");

  conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
  conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);

  // get a angel client
  angelClient = AngelClientFactory.get(conf);

  // add matrix
  MatrixContext mMatrix = new MatrixContext();
  mMatrix.setName("w1");
  mMatrix.setRowNum(1);
  mMatrix.setColNum(100000);
  mMatrix.setMaxRowNumInBlock(1);
  mMatrix.setMaxColNumInBlock(50000);
  mMatrix.setRowType(RowType.T_INT_DENSE);
  mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
  mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
  mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
  mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
  angelClient.addMatrix(mMatrix);

  angelClient.startPSServer();
  angelClient.runTask(DummyTask.class);
  Thread.sleep(5000);

  group0Id = new WorkerGroupId(0);
  worker0Id = new WorkerId(group0Id, 0);
  worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
  task0Id = new TaskId(0);
  task1Id = new TaskId(1);
  psId = new ParameterServerId(0);
  psAttempt0Id = new PSAttemptId(psId, 0);

  DataOutputStream out = new DataOutputStream(new FileOutputStream("data"));
  ByteBuf buf = Unpooled.buffer(16);
  buf.writeDouble(0.00);
  buf.writeDouble(1.00);
  buf.writeDouble(-1.00);
  buf.writeDouble(-2.00);
  buf.writeDouble(-5.00);
  buf.writeDouble(-6.00);
  buf.writeDouble(-7.00);
  buf.writeDouble(-8.00);
  serverPartition.getRow(6).update(RowType.T_DOUBLE_DENSE, buf, 8);
  serverPartition.save(out);
  out.close();
  DataInputStream in = new DataInputStream(new FileInputStream("data"));
  PartitionKey partitionKeyNew = new PartitionKey(2, 1, 1, 2, 8, 10);
  ServerPartition serverPartitionNew =
      new ServerPartition(partitionKeyNew, RowType.T_DOUBLE_DENSE);
  serverPartitionNew.init();
  assertNotEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(),
      ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
  serverPartitionNew.load(in);
  in.close();
  assertEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(),
      ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
  angelClient.stop();
}
 
Example 19
Project: aliyun-maxcompute-data-collectors   File: MySQLDumpImportJob.java   View Source Code Vote up 4 votes
/**
 * Configure the inputformat to use for the job.
 */
protected void configureInputFormat(Job job, String tableName,
    String tableClassName, String splitByCol)
    throws ClassNotFoundException, IOException {

  if (null == tableName) {
      LOG.error(
          "mysqldump-based import cannot support free-form query imports.");
      LOG.error("Do not use --direct and --query together for MySQL.");
      throw new IOException("null tableName for MySQLDumpImportJob.");
  }

  ConnManager mgr = getContext().getConnManager();
  String username = options.getUsername();
  if (null == username || username.length() == 0) {
    DBConfiguration.configureDB(job.getConfiguration(),
        mgr.getDriverClass(), options.getConnectString(),
        options.getConnectionParams());
  } else {
    DBConfiguration.configureDB(job.getConfiguration(),
        mgr.getDriverClass(), options.getConnectString(), username,
        options.getPassword(), options.getConnectionParams());
  }

  String [] colNames = options.getColumns();
  if (null == colNames) {
    colNames = mgr.getColumnNames(tableName);
  }

  String [] sqlColNames = null;
  if (null != colNames) {
    sqlColNames = new String[colNames.length];
    for (int i = 0; i < colNames.length; i++) {
      sqlColNames[i] = mgr.escapeColName(colNames[i]);
    }
  }

  // It's ok if the where clause is null in DBInputFormat.setInput.
  String whereClause = options.getWhereClause();

  // We can't set the class properly in here, because we may not have the
  // jar loaded in this JVM. So we start by calling setInput() with
  // DBWritable and then overriding the string manually.

  // Note that mysqldump also does *not* want a quoted table name.
  DataDrivenDBInputFormat.setInput(job, DBWritable.class,
      tableName, whereClause,
      mgr.escapeColName(splitByCol), sqlColNames);

  Configuration conf = job.getConfiguration();
  conf.setInt(MySQLUtils.OUTPUT_FIELD_DELIM_KEY,
      options.getOutputFieldDelim());
  conf.setInt(MySQLUtils.OUTPUT_RECORD_DELIM_KEY,
      options.getOutputRecordDelim());
  conf.setInt(MySQLUtils.OUTPUT_ENCLOSED_BY_KEY,
      options.getOutputEnclosedBy());
  conf.setInt(MySQLUtils.OUTPUT_ESCAPED_BY_KEY,
      options.getOutputEscapedBy());
  conf.setBoolean(MySQLUtils.OUTPUT_ENCLOSE_REQUIRED_KEY,
      options.isOutputEncloseRequired());
  String [] extraArgs = options.getExtraArgs();
  if (null != extraArgs) {
    conf.setStrings(MySQLUtils.EXTRA_ARGS_KEY, extraArgs);
  }

  LOG.debug("Using InputFormat: " + inputFormatClass);
  job.setInputFormatClass(getInputFormatClass());
}
 
Example 20
Project: hadoop   File: Bzip2Factory.java   View Source Code Vote up 4 votes
public static void setWorkFactor(Configuration conf, int workFactor) {
  conf.setInt("bzip2.compress.workfactor", workFactor);
}