Java Code Examples for org.apache.hadoop.conf.Configuration#setStrings()

The following examples show how to use org.apache.hadoop.conf.Configuration#setStrings() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
/**
 * @throws java.lang.Exception
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  ROWS[0] = ROW;
  ROWS[1] = ROW1;
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      MultiRowMutationEndpoint.class.getName());
  conf.setInt("hbase.regionserver.handler.count", 20);
  conf.setInt("hbase.bucketcache.size", 400);
  conf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
  conf.setFloat("hfile.block.cache.size", 0.2f);
  conf.setFloat("hbase.regionserver.global.memstore.size", 0.1f);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);// do not retry
  conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 5000);
  FAMILIES_1[0] = FAMILY;
  TEST_UTIL.startMiniCluster(SLAVES);
}
 
Example 2
private Configuration createConfiguration(TezOutputContext outputContext,
    Class<? extends Writable> keyClass, Class<? extends Writable> valClass,
    boolean shouldCompress, int maxSingleBufferSizeBytes,
    Class<? extends Partitioner> partitionerClass) {
  Configuration conf = new Configuration(false);
  conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs());
  conf.set(TezJobConfig.TEZ_RUNTIME_KEY_CLASS, keyClass.getName());
  conf.set(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS, valClass.getName());
  conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClass.getName());
  if (maxSingleBufferSizeBytes >= 0) {
    conf.setInt(TezJobConfig.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES,
        maxSingleBufferSizeBytes);
  }
  conf.setBoolean(TezJobConfig.TEZ_RUNTIME_COMPRESS, shouldCompress);
  if (shouldCompress) {
    conf.set(TezJobConfig.TEZ_RUNTIME_COMPRESS_CODEC,
        DefaultCodec.class.getName());
  }
  return conf;
}
 
Example 3
Source Project: big-c   File: MiniRPCBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
private void configureSuperUserIPAddresses(Configuration conf,
    String superUserShortName) throws IOException {
  ArrayList<String> ipList = new ArrayList<String>();
  Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
      .getNetworkInterfaces();
  while (netInterfaceList.hasMoreElements()) {
    NetworkInterface inf = netInterfaceList.nextElement();
    Enumeration<InetAddress> addrList = inf.getInetAddresses();
    while (addrList.hasMoreElements()) {
      InetAddress addr = addrList.nextElement();
      ipList.add(addr.getHostAddress());
    }
  }
  StringBuilder builder = new StringBuilder();
  for (String ip : ipList) {
    builder.append(ip);
    builder.append(',');
  }
  builder.append("127.0.1.1,");
  builder.append(InetAddress.getLocalHost().getCanonicalHostName());
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
          getProxySuperuserIpConfKey(superUserShortName), builder.toString());
}
 
Example 4
Source Project: hadoop   File: TestAuxServices.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAuxServiceRecoverySetup() throws IOException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, TEST_DIR.toString());
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      RecoverableServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      RecoverableServiceB.class, Service.class);
  try {
    final AuxServices aux = new AuxServices();
    aux.init(conf);
    Assert.assertEquals(2, aux.getServices().size());
    File auxStorageDir = new File(TEST_DIR,
        AuxServices.STATE_STORE_ROOT_NAME);
    Assert.assertEquals(2, auxStorageDir.listFiles().length);
    aux.close();
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 
Example 5
Source Project: hbase   File: IndexBuilder.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String [] args) throws IOException {
  String tableName = args[0];
  String columnFamily = args[1];
  System.out.println("****" + tableName);
  conf.set(TableInputFormat.SCAN, TableMapReduceUtil.convertScanToString(new Scan()));
  conf.set(TableInputFormat.INPUT_TABLE, tableName);
  conf.set("index.tablename", tableName);
  conf.set("index.familyname", columnFamily);
  String[] fields = new String[args.length - 2];
  System.arraycopy(args, 2, fields, 0, fields.length);
  conf.setStrings("index.fields", fields);
  Job job = new Job(conf, tableName);
  job.setJarByClass(IndexBuilder.class);
  job.setMapperClass(Map.class);
  job.setNumReduceTasks(0);
  job.setInputFormatClass(TableInputFormat.class);
  job.setOutputFormatClass(MultiTableOutputFormat.class);
  return job;
}
 
Example 6
private Configuration createConfiguration(OutputContext outputContext,
    Class<? extends Writable> keyClass, Class<? extends Writable> valClass,
    boolean shouldCompress, int maxSingleBufferSizeBytes,
    Class<? extends Partitioner> partitionerClass) {
  Configuration conf = new Configuration(false);
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs());
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, keyClass.getName());
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, valClass.getName());
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, partitionerClass.getName());
  if (maxSingleBufferSizeBytes >= 0) {
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES,
        maxSingleBufferSizeBytes);
  }
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, shouldCompress);
  if (shouldCompress) {
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC,
        DefaultCodec.class.getName());
  }
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS,
      reportPartitionStats.getType());
  return conf;
}
 
Example 7
Source Project: big-c   File: TestAuxServices.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 
Example 8
private static void configureSuperUserIPAddresses(Configuration conf,
    String superUserShortName) throws IOException {
  ArrayList<String> ipList = new ArrayList<String>();
  Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
      .getNetworkInterfaces();
  while (netInterfaceList.hasMoreElements()) {
    NetworkInterface inf = netInterfaceList.nextElement();
    Enumeration<InetAddress> addrList = inf.getInetAddresses();
    while (addrList.hasMoreElements()) {
      InetAddress addr = addrList.nextElement();
      ipList.add(addr.getHostAddress());
    }
  }
  StringBuilder builder = new StringBuilder();
  for (String ip : ipList) {
    builder.append(ip);
    builder.append(',');
  }
  builder.append("127.0.1.1,");
  builder.append(InetAddress.getLocalHost().getCanonicalHostName());
  LOG.info("Local Ip addresses: " + builder.toString());
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
          getProxySuperuserIpConfKey(superUserShortName),
      builder.toString());
}
 
Example 9
Source Project: hbase   File: TestWALObserver.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
      SampleRegionWALCoprocessor.class.getName());
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      SampleRegionWALCoprocessor.class.getName());
  conf.setInt("dfs.client.block.recovery.retries", 2);

  TEST_UTIL.startMiniCluster(1);
  Path hbaseRootDir = TEST_UTIL.getDFSCluster().getFileSystem()
      .makeQualified(new Path("/hbase"));
  Path hbaseWALRootDir = TEST_UTIL.getDFSCluster().getFileSystem()
          .makeQualified(new Path("/hbaseLogRoot"));
  LOG.info("hbase.rootdir=" + hbaseRootDir);
  CommonFSUtils.setRootDir(conf, hbaseRootDir);
  CommonFSUtils.setWALRootDir(conf, hbaseWALRootDir);
}
 
Example 10
@BeforeClass
public static void setup() throws Exception {
  conf = new Configuration(false);
  final String ourUrl = UserProvider.SCHEME_NAME + ":///";
  conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
  CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
  provider.createCredentialEntry(ServerConfig.
      SENTRY_STORE_JDBC_PASS, passwd);
  provider.flush();

  dataDir = new File(Files.createTempDir(), "sentry_policy_db");
  conf.set(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false");
  conf.set(ServerConfig.SENTRY_STORE_JDBC_URL,
      "jdbc:derby:;databaseName=" + dataDir.getPath() + ";create=true");
  conf.set(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy");
  conf.setStrings(ServerConfig.ADMIN_GROUPS, adminGroups);
  conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING,
      ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
  policyFilePath = new File(dataDir, "local_policy_file.ini");
  conf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE,
      policyFilePath.getPath());
  sentryStore = new SentryStore(conf);
}
 
Example 11
Source Project: phoenix   File: IndexHandlerIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClientWritesWithPriority() throws Exception {
    Configuration conf = new Configuration(UTIL.getConfiguration());
    // add the keys for our rpc factory
    conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
        CountingIndexClientRpcFactory.class.getName());
    // and set the index table as the current table
    conf.setStrings(IndexQosRpcControllerFactory.INDEX_TABLE_NAMES_KEY,
        TestTable.getTableNameString());
    HTable table = new HTable(conf, TestTable.getTableName());

    // do a write to the table
    Put p = new Put(row);
    p.add(family, qual, new byte[] { 1, 0, 1, 0 });
    table.put(p);
    table.flushCommits();

    // check the counts on the rpc controller
    assertEquals("Didn't get the expected number of index priority writes!", 1,
        (int) CountingIndexClientRpcController.priorityCounts
                .get(QueryServicesOptions.DEFAULT_INDEX_MIN_PRIORITY));

    table.close();
}
 
Example 12
Source Project: mnemonic   File: MneConfigHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static void setEntityFactoryProxies(Configuration conf, String prefix, Class<?>[] proxies) {
  List<String> vals = new ArrayList<>();
  for (Class<?> itm : proxies) {
    vals.add(itm.getName());
  }
  conf.setStrings(getConfigName(prefix, ENTITY_FACTORY_PROXIES), vals.toArray(new String[0]));
}
 
Example 13
public static void addHBaseClusterNNHAConfiguration(Configuration conf) {
    String hdfsConfigFile = KylinConfig.getInstanceFromEnv().getHBaseClusterHDFSConfigFile();
    if (hdfsConfigFile == null || hdfsConfigFile.isEmpty()) {
        return;
    }
    Configuration hdfsConf = new Configuration(false);
    hdfsConf.addResource(hdfsConfigFile);
    Collection<String> nameServices = hdfsConf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    Collection<String> mainNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    for (String serviceId : nameServices) {
        mainNameServices.add(serviceId);

        String serviceConfKey = DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + serviceId;
        String proxyConfKey = DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + serviceId;
        conf.set(serviceConfKey, hdfsConf.get(serviceConfKey, ""));
        conf.set(proxyConfKey, hdfsConf.get(proxyConfKey, ""));

        Collection<String> nameNodes = hdfsConf.getTrimmedStringCollection(serviceConfKey);
        for (String nameNode : nameNodes) {
            String rpcConfKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + serviceId + "." + nameNode;
            conf.set(rpcConfKey, hdfsConf.get(rpcConfKey, ""));
        }
    }
    conf.setStrings(DFSConfigKeys.DFS_NAMESERVICES, mainNameServices.toArray(new String[0]));
    // See YARN-3021, instruct RM skip renew token of hbase cluster name services
    conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, nameServices.toArray(new String[0]));
}
 
Example 14
Source Project: datawave   File: DataTypeHelperImpl.java    License: Apache License 2.0 5 votes vote down vote up
/** Sets a property in the config if not already set. */
protected void setIfUnset(final Configuration config, final String key, final String value) {
    final String property = key2property(config, key);
    
    if (config.get(property) == null) {
        config.setStrings(property, value);
        logSetDefault(property, value);
    }
}
 
Example 15
Source Project: hbase   File: TestHFileCleaner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRemovesEmptyDirectories() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  // no cleaner policies = delete all files
  conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
  Server server = new DummyServer();
  Path archivedHfileDir =
    new Path(UTIL.getDataTestDirOnTestFS(), HConstants.HFILE_ARCHIVE_DIRECTORY);

  // setup the cleaner
  FileSystem fs = UTIL.getDFSCluster().getFileSystem();
  HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archivedHfileDir, POOL);

  // make all the directories for archiving files
  Path table = new Path(archivedHfileDir, "table");
  Path region = new Path(table, "regionsomthing");
  Path family = new Path(region, "fam");
  Path file = new Path(family, "file12345");
  fs.mkdirs(family);
  if (!fs.exists(family)) throw new RuntimeException("Couldn't create test family:" + family);
  fs.create(file).close();
  if (!fs.exists(file)) throw new RuntimeException("Test file didn't get created:" + file);

  // run the chore to cleanup the files (and the directories above it)
  cleaner.chore();

  // make sure all the parent directories get removed
  assertFalse("family directory not removed for empty directory", fs.exists(family));
  assertFalse("region directory not removed for empty directory", fs.exists(region));
  assertFalse("table directory not removed for empty directory", fs.exists(table));
  assertTrue("archive directory", fs.exists(archivedHfileDir));
}
 
Example 16
/**
 * @throws Exception If failed.
 */
@Test
public void testMapperException() throws Exception {
    prepareFile("/testFile", 1000);

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(FailMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));

    job.setJarByClass(getClass());

    final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 3),
            createJobInfo(job.getConfiguration(), null));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override public Object call() throws Exception {
            fut.get();

            return null;
        }
    }, IgniteCheckedException.class, null);
}
 
Example 17
Source Project: big-c   File: TestDoAsEffectiveUser.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRealUserGroupAuthorizationFailure() throws IOException {
  final Configuration conf = new Configuration();
  configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
          getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
      "group3");
  Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
      .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
      .setNumHandlers(2).setVerbose(false).build();
  
  refreshConf(conf);

  try {
    server.start();

    final InetSocketAddress addr = NetUtils.getConnectAddress(server);

    UserGroupInformation realUserUgi = UserGroupInformation
        .createRemoteUser(REAL_USER_NAME);

    UserGroupInformation proxyUserUgi = UserGroupInformation
        .createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
    String retVal = proxyUserUgi
        .doAs(new PrivilegedExceptionAction<String>() {
          @Override
          public String run() throws IOException {
            proxy = RPC.getProxy(TestProtocol.class,
                TestProtocol.versionID, addr, conf);
            String ret = proxy.aMethod();
            return ret;
          }
        });

    Assert.fail("The RPC must have failed " + retVal);
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    server.stop();
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
  }
}
 
Example 18
Source Project: rya   File: IndexWritingTool.java    License: Apache License 2.0 4 votes vote down vote up
public void setVarOrders(final String s, final Configuration conf) throws MalformedQueryException {

        final SPARQLParser parser = new SPARQLParser();
        final TupleExpr query = parser.parseQuery(s, null).getTupleExpr();

        final List<String> projList = Lists.newArrayList(((Projection) query).getProjectionElemList().getTargetNames());
        final String projElems = Joiner.on(";").join(projList);
        conf.set("projElems", projElems);

        final Pattern splitPattern1 = Pattern.compile("\n");
        final Pattern splitPattern2 = Pattern.compile(",");
        final String[] lines = splitPattern1.split(s);

        final List<String> varOrders = Lists.newArrayList();
        final List<String> varOrderPos = Lists.newArrayList();

        int orderNum = 0;
        final int projSizeSq = projList.size()*projList.size();

        for (String t : lines) {


            if(orderNum > projSizeSq){
                break;
            }

            String[] order = null;
            if (t.startsWith("#prefix")) {
                t = t.substring(7).trim();
                order = splitPattern2.split(t, projList.size());
            }


            String tempVarOrder = "";
            String tempVarOrderPos = "";

            if (order != null) {
                for (final String u : order) {
                    if (tempVarOrder.length() == 0) {
                        tempVarOrder = u.trim();
                    } else {
                        tempVarOrder = tempVarOrder + ";" + u.trim();
                    }
                    final int pos = projList.indexOf(u.trim());
                    if (pos < 0) {
                        throw new IllegalArgumentException("Invalid variable order!");
                    } else {
                        if (tempVarOrderPos.length() == 0) {
                            tempVarOrderPos = tempVarOrderPos + pos;
                        } else {
                            tempVarOrderPos = tempVarOrderPos + ";" + pos;
                        }
                    }
                }

                varOrders.add(tempVarOrder);
                varOrderPos.add(tempVarOrderPos);
            }

            if(tempVarOrder.length() > 0) {
                orderNum++;
            }

        }

        if(orderNum ==  0) {
            varOrders.add(projElems);
            String tempVarPos = "";

            for(int i = 0; i < projList.size(); i++) {
                if(i == 0) {
                    tempVarPos = Integer.toString(0);
                } else {
                    tempVarPos = tempVarPos + ";" + i;
                }
            }
            varOrderPos.add(tempVarPos);

        }

        final String[] vOrders = varOrders.toArray(new String[varOrders.size()]);
        final String[] vOrderPos = varOrderPos.toArray(new String[varOrderPos.size()]);



        conf.setStrings("varOrders", vOrders);
        conf.setStrings("varOrderPos", vOrderPos);

    }
 
Example 19
Source Project: tez   File: TestMRInputAMSplitGenerator.java    License: Apache License 2.0 4 votes vote down vote up
private void testGroupSplitsAndSortSplits(boolean groupSplitsEnabled,
    boolean sortSplitsEnabled) throws Exception {
  Configuration conf = new Configuration();
  String[] splitLengths = new String[50];
  for (int i = 0; i < splitLengths.length; i++) {
    splitLengths[i] = Integer.toString(1000 * (i + 1));
  }
  conf.setStrings(SPLITS_LENGTHS, splitLengths);
  DataSourceDescriptor dataSource = MRInput.createConfigBuilder(
      conf, InputFormatForTest.class).
      groupSplits(groupSplitsEnabled).sortSplits(sortSplitsEnabled).build();
  UserPayload userPayload = dataSource.getInputDescriptor().getUserPayload();

  InputInitializerContext context =
      new TezTestUtils.TezRootInputInitializerContextForTest(userPayload, new Configuration(false));
  MRInputAMSplitGenerator splitGenerator =
      new MRInputAMSplitGenerator(context);

  List<Event> events = splitGenerator.initialize();

  assertTrue(events.get(0) instanceof InputConfigureVertexTasksEvent);
  boolean shuffled = false;
  InputSplit previousIs = null;
  int numRawInputSplits = 0;
  for (int i = 1; i < events.size(); i++) {
    assertTrue(events.get(i) instanceof InputDataInformationEvent);
    InputDataInformationEvent diEvent = (InputDataInformationEvent) (events.get(i));
    assertNull(diEvent.getDeserializedUserPayload());
    assertNotNull(diEvent.getUserPayload());
    MRSplitProto eventProto = MRSplitProto.parseFrom(ByteString.copyFrom(
        diEvent.getUserPayload()));
    InputSplit is = MRInputUtils.getNewSplitDetailsFromEvent(
        eventProto, new Configuration());
    if (groupSplitsEnabled) {
      numRawInputSplits += ((TezGroupedSplit)is).getGroupedSplits().size();
      for (InputSplit inputSplit : ((TezGroupedSplit)is).getGroupedSplits()) {
        assertTrue(inputSplit instanceof InputSplitForTest);
      }
      assertTrue(((TezGroupedSplit)is).getGroupedSplits().get(0)
          instanceof InputSplitForTest);
    } else {
      numRawInputSplits++;
      assertTrue(is instanceof InputSplitForTest);
    }
    // The splits in the list returned from InputFormat has ascending
    // size in order.
    // If sortSplitsEnabled is true, MRInputAMSplitGenerator will sort the
    // splits in descending order.
    // If sortSplitsEnabled is false, MRInputAMSplitGenerator will shuffle
    // the splits.
    if (previousIs != null) {
      if (sortSplitsEnabled) {
        assertTrue(is.getLength() <= previousIs.getLength());
      } else {
        shuffled |= (is.getLength() > previousIs.getLength());
      }
    }
    previousIs = is;
  }
  assertEquals(splitLengths.length, numRawInputSplits);
  if (!sortSplitsEnabled) {
    assertTrue(shuffled);
  }
}
 
Example 20
Source Project: big-c   File: TestDoAsEffectiveUser.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRealUserIPAuthorizationFailure() throws IOException {
  final Configuration conf = new Configuration();
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
          getProxySuperuserIpConfKey(REAL_USER_SHORT_NAME),
      "20.20.20.20"); //Authorized IP address
  conf.setStrings(DefaultImpersonationProvider.getTestProvider().
          getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
      "group1");
  Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
      .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
      .setNumHandlers(2).setVerbose(false).build();

  refreshConf(conf);
  
  try {
    server.start();

    final InetSocketAddress addr = NetUtils.getConnectAddress(server);

    UserGroupInformation realUserUgi = UserGroupInformation
        .createRemoteUser(REAL_USER_NAME);

    UserGroupInformation proxyUserUgi = UserGroupInformation
        .createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
    String retVal = proxyUserUgi
        .doAs(new PrivilegedExceptionAction<String>() {
          @Override
          public String run() throws IOException {
            proxy = RPC.getProxy(TestProtocol.class,
                TestProtocol.versionID, addr, conf);
            String ret = proxy.aMethod();
            return ret;
          }
        });

    Assert.fail("The RPC must have failed " + retVal);
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    server.stop();
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
  }
}