Java Code Examples for org.apache.hadoop.conf.Configuration#getBoolean()

The following examples show how to use org.apache.hadoop.conf.Configuration#getBoolean() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: rya   File: RyaOutputFormat.java    License: Apache License 2.0 6 votes vote down vote up
private static TemporalIndexer getTemporalIndexer(final Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_TEMPORAL, true)) {
        return null;
    }
    final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer();
    temporal.setConf(conf);
    Connector connector;
    try {
        connector = ConfigUtils.getConnector(conf);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new IOException("Error when attempting to create a connection for writing the temporal index.", e);
    }
    final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    temporal.setConnector(connector);
    temporal.setMultiTableBatchWriter(mtbw);
    temporal.init();
    return temporal;
}
 
Example 2
Source Project: tez   File: RecoveryService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  recoveryPath = appContext.getCurrentRecoveryDir();
  recoveryDirFS = FileSystem.get(recoveryPath.toUri(), conf);
  bufferSize = conf.getInt(TezConfiguration.DAG_RECOVERY_FILE_IO_BUFFER_SIZE,
      TezConfiguration.DAG_RECOVERY_FILE_IO_BUFFER_SIZE_DEFAULT);

  flushInterval = conf.getInt(TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS,
      TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS_DEFAULT);
  maxUnflushedEvents = conf.getInt(TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS,
      TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT);

  drainEventsFlag = conf.getBoolean(
      TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED,
      TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED_DEFAULT);

  LOG.info("RecoveryService initialized with "
    + "recoveryPath=" + recoveryPath
    + ", bufferSize(bytes)=" + bufferSize
    + ", flushInterval(s)=" + flushInterval
    + ", maxUnflushedEvents=" + maxUnflushedEvents);
}
 
Example 3
Source Project: big-c   File: ControlledJob.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Submit this job to mapred. The state becomes RUNNING if submission 
 * is successful, FAILED otherwise.  
 */
protected synchronized void submit() {
  try {
    Configuration conf = job.getConfiguration();
    if (conf.getBoolean(CREATE_DIR, false)) {
      FileSystem fs = FileSystem.get(conf);
      Path inputPaths[] = FileInputFormat.getInputPaths(job);
      for (int i = 0; i < inputPaths.length; i++) {
        if (!fs.exists(inputPaths[i])) {
          try {
            fs.mkdirs(inputPaths[i]);
          } catch (IOException e) {

          }
        }
      }
    }
    job.submit();
    this.state = State.RUNNING;
  } catch (Exception ioe) {
    LOG.info(getJobName()+" got an error while submitting ",ioe);
    this.state = State.FAILED;
    this.message = StringUtils.stringifyException(ioe);
  }
}
 
Example 4
Source Project: big-c   File: NameNode.java    License: Apache License 2.0 5 votes vote down vote up
public static void checkAllowFormat(Configuration conf) throws IOException {
  if (!conf.getBoolean(DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY, 
      DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT)) {
    throw new IOException("The option " + DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY
              + " is set to false for this filesystem, so it "
              + "cannot be formatted. You will need to set "
              + DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY +" parameter "
              + "to true in order to format this filesystem");
  }
}
 
Example 5
Source Project: big-c   File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.shouldExitOnError =
      conf.getBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY,
        Dispatcher.DEFAULT_DISPATCHER_EXIT_ON_ERROR);
  super.serviceInit(conf);
}
 
Example 6
Source Project: ytk-learn   File: HadoopTrainWorker.java    License: MIT License 5 votes vote down vote up
@Override
protected void setup(Context context) {
    Configuration conf = context.getConfiguration();
    workThread = new Thread() {
        @Override
        public void run() {
            HadoopTrainWorkerCore trainWorkerCore = null;
            try {
                trainWorkerCore = new HadoopTrainWorkerCore(
                        conf.get("modelName"),
                        conf.get("configPath"),
                        conf.get("configFile"),
                        conf.get("pyTransformScript"),
                        conf.getBoolean("needPyTransform", false),
                        conf.get("loginName"),
                        conf.get("hostName"),
                        conf.getInt("hostPort", -1),
                        conf.getInt("threadNum", -1)
                        );

                Map<String, Object> customParamsMap = decodeMap(conf.get("customParamsMap"));

                for (Map.Entry<String, Object> entry : customParamsMap.entrySet()) {
                    trainWorkerCore.setCustomParam(entry.getKey(), entry.getValue());
                    LOG.info("hadoop custom params:" + entry.getKey() + "=" + entry.getValue());
                }

                trainWorkerCore.train(Arrays.asList(new ReducerIterator()), null);
            } catch (Exception e) {
                e.printStackTrace();
                System.exit(1);
            }
        }
    };
    workThread.start();
}
 
Example 7
Source Project: hadoop   File: NativeAzureFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(URI uri, Configuration conf)
    throws IOException, IllegalArgumentException {
  // Check authority for the URI to guarantee that it is non-null.
  uri = reconstructAuthorityIfNeeded(uri, conf);
  if (null == uri.getAuthority()) {
    final String errMsg = String
        .format("Cannot initialize WASB file system, URI authority not recognized.");
    throw new IllegalArgumentException(errMsg);
  }
  super.initialize(uri, conf);

  if (store == null) {
    store = createDefaultStore(conf);
  }

  instrumentation = new AzureFileSystemInstrumentation(conf);
  if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
    // Make sure the metrics system is available before interacting with Azure
    AzureFileSystemMetricsSystem.fileSystemStarted();
    metricsSourceName = newMetricsSourceName();
    String sourceDesc = "Azure Storage Volume File System metrics";
    AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc,
      instrumentation);
  }

  store.initialize(uri, conf, instrumentation);
  setConf(conf);
  this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
  this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser()
      .getShortUserName()).makeQualified(getUri(), getWorkingDirectory());
  this.blockSize = conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME,
      MAX_AZURE_BLOCK_SIZE);

  if (LOG.isDebugEnabled()) {
    LOG.debug("NativeAzureFileSystem. Initializing.");
    LOG.debug("  blockSize  = "
        + conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE));
  }
}
 
Example 8
Source Project: hadoop   File: HistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  initializeWebApp(conf);
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.MR_HISTORY_BIND_HOST,
      JHAdminConfig.MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

  server =
      rpc.getServer(HSClientProtocol.class, protocolHandler, address,
          conf, jhsDTSecretManager,
          conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
              JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    server.refreshServiceAcl(conf, new ClientHSPolicyProvider());
  }
  
  server.start();
  this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_BIND_HOST,
                                            JHAdminConfig.MR_HISTORY_ADDRESS,
                                            JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
                                            server.getListenerAddress());
  LOG.info("Instantiated HistoryClientService at " + this.bindAddress);

  super.serviceStart();
}
 
Example 9
private boolean getAllowEmptyValues(RegionCoprocessorEnvironment env, HTableDescriptor htd) {
  String allowEmptyValuesFromTableDesc = htd.getValue(TxConstants.ALLOW_EMPTY_VALUES_KEY);
  Configuration conf = getConfiguration(env);
  boolean allowEmptyValuesFromConfig = (conf != null) ?
    conf.getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY, TxConstants.ALLOW_EMPTY_VALUES_DEFAULT) :
    TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;

  // If the property is not present in the tableDescriptor, get it from the Configuration
  return  (allowEmptyValuesFromTableDesc != null) ?
    Boolean.valueOf(allowEmptyValuesFromTableDesc) : allowEmptyValuesFromConfig;
}
 
Example 10
Source Project: big-c   File: SleepJob.java    License: Apache License 2.0 5 votes vote down vote up
public SleepJob(Configuration conf, long submissionMillis, JobStory jobdesc,
    Path outRoot, UserGroupInformation ugi, int seq, int numLocations,
    String[] hosts) throws IOException {
  super(conf, submissionMillis, jobdesc, outRoot, ugi, seq);
  this.fakeLocations = numLocations;
  this.hosts = hosts.clone();
  this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float) fakeLocations
      / hosts.length, rand.get()) : null;
  this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false);
  mapMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_MAP_TIME, Long.MAX_VALUE);
  reduceMaxSleepTime = conf.getLong(GRIDMIX_SLEEP_MAX_REDUCE_TIME,
      Long.MAX_VALUE);
}
 
Example 11
private boolean getAllowEmptyValues(RegionCoprocessorEnvironment env, TableDescriptor htd) {
  String allowEmptyValuesFromTableDesc = htd.getValue(TxConstants.ALLOW_EMPTY_VALUES_KEY);
  Configuration conf = getConfiguration(env);
  boolean allowEmptyValuesFromConfig =
      (conf != null) ? conf.getBoolean(TxConstants.ALLOW_EMPTY_VALUES_KEY,
        TxConstants.ALLOW_EMPTY_VALUES_DEFAULT) : TxConstants.ALLOW_EMPTY_VALUES_DEFAULT;

  // If the property is not present in the tableDescriptor, get it from the Configuration
  return (allowEmptyValuesFromTableDesc != null) ? Boolean.valueOf(allowEmptyValuesFromTableDesc)
      : allowEmptyValuesFromConfig;
}
 
Example 12
Source Project: hbase   File: BlockCacheFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static BlockCache createBlockCache(Configuration conf) {
  if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
    LOG.warn("The config key {} is deprecated now, instead please use {}. In future release "
        + "we will remove the deprecated config.", DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY,
      BLOCKCACHE_BLOCKSIZE_KEY);
  }
  FirstLevelBlockCache l1Cache = createFirstLevelCache(conf);
  if (l1Cache == null) {
    return null;
  }
  boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
  if (useExternal) {
    BlockCache l2CacheInstance = createExternalBlockcache(conf);
    return l2CacheInstance == null ?
        l1Cache :
        new InclusiveCombinedBlockCache(l1Cache, l2CacheInstance);
  } else {
    // otherwise use the bucket cache.
    BucketCache bucketCache = createBucketCache(conf);
    if (!conf.getBoolean("hbase.bucketcache.combinedcache.enabled", true)) {
      // Non combined mode is off from 2.0
      LOG.warn(
          "From HBase 2.0 onwards only combined mode of LRU cache and bucket cache is available");
    }
    return bucketCache == null ? l1Cache : new CombinedBlockCache(l1Cache, bucketCache);
  }
}
 
Example 13
Source Project: attic-apex-core   File: ACLManager.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean areACLsRequired(Configuration conf)
{
  logger.debug("Check ACLs required");
  if (conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) {
    logger.debug("Admin ACL {}", conf.get(YarnConfiguration.YARN_ADMIN_ACL));
    if (!YarnConfiguration.DEFAULT_YARN_ADMIN_ACL.equals(conf.get(YarnConfiguration.YARN_ADMIN_ACL))) {
      logger.debug("Non default admin ACL");
      return true;
    }
  }
  return false;
}
 
Example 14
Source Project: big-c   File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
private void initDataXceiver(Configuration conf) throws IOException {
  // find free port or use privileged port provided
  TcpPeerServer tcpPeerServer;
  if (secureResources != null) {
    tcpPeerServer = new TcpPeerServer(secureResources);
  } else {
    tcpPeerServer = new TcpPeerServer(dnConf.socketWriteTimeout,
        DataNode.getStreamingAddr(conf));
  }
  tcpPeerServer.setReceiveBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
  streamingAddr = tcpPeerServer.getStreamingAddr();
  LOG.info("Opened streaming server at " + streamingAddr);
  this.threadGroup = new ThreadGroup("dataXceiverServer");
  xserver = new DataXceiverServer(tcpPeerServer, conf, this);
  this.dataXceiverServer = new Daemon(threadGroup, xserver);
  this.threadGroup.setDaemon(true); // auto destroy when empty

  if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT) ||
      conf.getBoolean(DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC,
            DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT)) {
    DomainPeerServer domainPeerServer =
              getDomainPeerServer(conf, streamingAddr.getPort());
    if (domainPeerServer != null) {
      this.localDataXceiverServer = new Daemon(threadGroup,
          new DataXceiverServer(domainPeerServer, conf, this));
      LOG.info("Listening on UNIX domain socket: " +
          domainPeerServer.getBindPath());
    }
  }
  this.shortCircuitRegistry = new ShortCircuitRegistry(conf);
}
 
Example 15
/**
 * Used for an aggregate query in which the key order does not necessarily match the group by
 * key order. In this case, we must collect all distinct groups within a region into a map,
 * aggregating as we go.
 * @param limit TODO
 */
private RegionScanner scanUnordered(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan,
        final RegionScanner scanner, final List<Expression> expressions,
        final ServerAggregators aggregators, long limit) throws IOException {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(LogUtil.addCustomAnnotations(
                "Grouped aggregation over unordered rows with scan " + scan
                + ", group by " + expressions + ", aggregators " + aggregators,
                ScanUtil.getCustomAnnotations(scan)));
    }
    RegionCoprocessorEnvironment env = c.getEnvironment();
    Configuration conf = env.getConfiguration();
    int estDistVals = conf.getInt(GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB, DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES);
    byte[] estDistValsBytes = scan.getAttribute(BaseScannerRegionObserver.ESTIMATED_DISTINCT_VALUES);
    if (estDistValsBytes != null) {
        // Allocate 1.5x estimation
        estDistVals = Math.max(MIN_DISTINCT_VALUES,
                        (int) (Bytes.toInt(estDistValsBytes) * 1.5f));
    }
    
    Pair<Integer, Integer> minMaxQualifiers = EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan);
    boolean useQualifierAsIndex = EncodedColumnsUtil.useQualifierAsIndex(EncodedColumnsUtil.getMinMaxQualifiersFromScan(scan));
    final boolean spillableEnabled =
            conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE);
    final PTable.QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan);

    GroupByCache groupByCache =
            GroupByCacheFactory.INSTANCE.newCache(
                    env, ScanUtil.getTenantId(scan), ScanUtil.getCustomAnnotations(scan),
                    aggregators, estDistVals);
    boolean success = false;
    try {
        boolean hasMore;
        Tuple result = useQualifierAsIndex ? new PositionBasedMultiKeyValueTuple() : new MultiKeyValueTuple();
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(LogUtil.addCustomAnnotations(
                    "Spillable groupby enabled: " + spillableEnabled,
                    ScanUtil.getCustomAnnotations(scan)));
        }
        Region region = c.getEnvironment().getRegion();
        boolean acquiredLock = false;
        try {
            region.startRegionOperation();
            acquiredLock = true;
            synchronized (scanner) {
                do {
                    List<Cell> results = useQualifierAsIndex ? new EncodedColumnQualiferCellsList(minMaxQualifiers.getFirst(), minMaxQualifiers.getSecond(), encodingScheme) : new ArrayList<Cell>();
                    // Results are potentially returned even when the return
                    // value of s.next is false
                    // since this is an indication of whether or not there are
                    // more values after the
                    // ones returned
                    hasMore = scanner.nextRaw(results);
                    if (!results.isEmpty()) {
                        result.setKeyValues(results);
                        ImmutableBytesPtr key =
                            TupleUtil.getConcatenatedValue(result, expressions);
                        Aggregator[] rowAggregators = groupByCache.cache(key);
                        // Aggregate values here
                        aggregators.aggregate(rowAggregators, result);
                    }
                } while (hasMore && groupByCache.size() < limit);
            }
        }  finally {
            if (acquiredLock) region.closeRegionOperation();
        }

        RegionScanner regionScanner = groupByCache.getScanner(scanner);

        // Do not sort here, but sort back on the client instead
        // The reason is that if the scan ever extends beyond a region
        // (which can happen if we're basing our parallelization split
        // points on old metadata), we'll get incorrect query results.
        success = true;
        return regionScanner;
    } finally {
        if (!success) {
            Closeables.closeQuietly(groupByCache);
        }
    }
}
 
Example 16
public static boolean getRefDirIsSet(Configuration conf) {
    return conf.getBoolean(refDirIsSet, false);
}
 
Example 17
Source Project: hbase   File: StochasticLoadBalancer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void setConf(Configuration conf) {
  super.setConf(conf);
  maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
  stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
  maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
  runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);

  numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
  minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
  if (localityCandidateGenerator == null) {
    localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
  }
  localityCost = new ServerLocalityCostFunction(conf, services);
  rackLocalityCost = new RackLocalityCostFunction(conf, services);

  if (this.candidateGenerators == null) {
    candidateGenerators = Lists.newArrayList();
    candidateGenerators.add(new RandomCandidateGenerator());
    candidateGenerators.add(new LoadCandidateGenerator());
    candidateGenerators.add(localityCandidateGenerator);
    candidateGenerators.add(new RegionReplicaRackCandidateGenerator());
  }
  regionLoadFunctions = new CostFromRegionLoadFunction[] {
    new ReadRequestCostFunction(conf),
    new CPRequestCostFunction(conf),
    new WriteRequestCostFunction(conf),
    new MemStoreSizeCostFunction(conf),
    new StoreFileCostFunction(conf)
  };
  regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf);
  regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf);

  costFunctions = new ArrayList<>();
  costFunctions.add(new RegionCountSkewCostFunction(conf));
  costFunctions.add(new PrimaryRegionCountSkewCostFunction(conf));
  costFunctions.add(new MoveCostFunction(conf));
  costFunctions.add(localityCost);
  costFunctions.add(rackLocalityCost);
  costFunctions.add(new TableSkewCostFunction(conf));
  costFunctions.add(regionReplicaHostCostFunction);
  costFunctions.add(regionReplicaRackCostFunction);
  costFunctions.add(regionLoadFunctions[0]);
  costFunctions.add(regionLoadFunctions[1]);
  costFunctions.add(regionLoadFunctions[2]);
  costFunctions.add(regionLoadFunctions[3]);
  costFunctions.add(regionLoadFunctions[4]);
  loadCustomCostFunctions(conf);

  curFunctionCosts= new Double[costFunctions.size()];
  tempFunctionCosts= new Double[costFunctions.size()];
  LOG.info("Loaded config; maxSteps=" + maxSteps + ", stepsPerRegion=" + stepsPerRegion +
          ", maxRunningTime=" + maxRunningTime + ", isByTable=" + isByTable + ", CostFunctions=" +
          Arrays.toString(getCostFunctionNames()) + " etc.");
}
 
Example 18
Source Project: anthelion   File: CCParseFilter.java    License: Apache License 2.0 4 votes vote down vote up
/** Scan the document adding attributes to metadata.*/
public static void walk(Node doc, URL base, Metadata metadata, Configuration conf)
  throws ParseException {

  // walk the DOM tree, scanning for license data
  Walker walker = new Walker(base);
  walker.walk(doc);

  // interpret results of walk
  String licenseUrl = null;
  String licenseLocation = null;
  if (walker.rdfLicense != null) {            // 1st choice: subject in RDF
    licenseLocation = "rdf";
    licenseUrl = walker.rdfLicense;
  } else if (walker.relLicense != null) {     // 2nd: anchor w/ rel=license
    licenseLocation = "rel";
    licenseUrl = walker.relLicense.toString();
  } else if (walker.anchorLicense != null) {  // 3rd: anchor w/ CC license
    licenseLocation = "a";
    licenseUrl = walker.anchorLicense.toString();
  } else if (conf.getBoolean("creativecommons.exclude.unlicensed", false)) {
    throw new ParseException("No CC license.  Excluding.");
  }

  // add license to metadata
  if (licenseUrl != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+licenseUrl+" in "+licenseLocation+" of "+base);
    }
    metadata.add(CreativeCommons.LICENSE_URL, licenseUrl);
    metadata.add(CreativeCommons.LICENSE_LOCATION, licenseLocation);
  }

  if (walker.workType != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+walker.workType+" in "+base);
    }
    metadata.add(CreativeCommons.WORK_TYPE, walker.workType);
  }

}
 
Example 19
Source Project: zerowing   File: ConfigUtil.java    License: MIT License 4 votes vote down vote up
public static boolean getSkipDeletes(Configuration conf) {
  return conf.getBoolean(SKIP_DELETES, false);
}
 
Example 20
Source Project: datawave   File: BulkInputFormat.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Determines whether a configuration uses local iterators.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @return true if uses local iterators, false otherwise
 * @see #setLocalIterators(Configuration, boolean)
 */
protected static boolean usesLocalIterators(Configuration conf) {
    return conf.getBoolean(LOCAL_ITERATORS, false);
}