Java Code Examples for org.apache.commons.configuration2.Configuration#containsKey()

public DriverRemoteConnection(final Configuration conf) {
    final boolean hasClusterConf = IteratorUtils.anyMatch(conf.getKeys(), k -> k.startsWith("clusterConfiguration"));
    if (conf.containsKey(GREMLIN_REMOTE_DRIVER_CLUSTERFILE) && hasClusterConf)
        throw new IllegalStateException(String.format("A configuration should not contain both '%s' and 'clusterConfiguration'", GREMLIN_REMOTE_DRIVER_CLUSTERFILE));

    remoteTraversalSourceName = conf.getString(GREMLIN_REMOTE_DRIVER_SOURCENAME, DEFAULT_TRAVERSAL_SOURCE);

    try {
        final Cluster cluster;
        if (!conf.containsKey(GREMLIN_REMOTE_DRIVER_CLUSTERFILE) && !hasClusterConf)
            cluster = Cluster.open();
        else
            cluster = conf.containsKey(GREMLIN_REMOTE_DRIVER_CLUSTERFILE) ?
                    Cluster.open(conf.getString(GREMLIN_REMOTE_DRIVER_CLUSTERFILE)) : Cluster.open(conf.subset("clusterConfiguration"));

        client = cluster.connect(Client.Settings.build().create()).alias(remoteTraversalSourceName);
    } catch (Exception ex) {
        throw new IllegalStateException(ex);
    }

    attachElements = false;

    tryCloseCluster = true;
    tryCloseClient = true;
    this.conf = Optional.of(conf);
}
 

@Override
public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD) {
    if (!configuration.getBoolean(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false))
        LOGGER.warn("The SparkContext should be persisted in order for the RDD to persist across jobs. To do so, set " + Constants.GREMLIN_SPARK_PERSIST_CONTEXT + " to true");
    if (!configuration.containsKey(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION))
        throw new IllegalArgumentException("There is no provided " + Constants.GREMLIN_HADOOP_OUTPUT_LOCATION + " to write the persisted RDD to");
    SparkContextStorage.open(configuration).rm(configuration.getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION));  // this might be bad cause it unpersists the job RDD
    // determine which storage level to persist the RDD as with MEMORY_ONLY being the default cache()
    final StorageLevel storageLevel = StorageLevel.fromString(configuration.getString(Constants.GREMLIN_SPARK_PERSIST_STORAGE_LEVEL, "MEMORY_ONLY"));
    if (!configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true))
        graphRDD.mapValues(vertex -> {
            vertex.get().dropEdges(Direction.BOTH);
            return vertex;
        }).setName(Constants.getGraphLocation(configuration.getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION))).persist(storageLevel)
                // call action to eager store rdd
                .count();
    else
        graphRDD.setName(Constants.getGraphLocation(configuration.getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION))).persist(storageLevel)
                // call action to eager store rdd
                .count();
    Spark.refresh(); // necessary to do really fast so the Spark GC doesn't clear out the RDD
}
 

/**
 * Creates a new {@link RemoteGraph} instance using the specified configuration, which allows {@link RemoteGraph}
 * to be compliant with {@link GraphFactory}. Expects key for {@link RemoteConnection#GREMLIN_REMOTE_CONNECTION_CLASS}
 * as well as any configuration required by the underlying {@link RemoteConnection} which will be instantiated.
 * Note that the {@code Configuration} object is passed down without change to the creation of the
 * {@link RemoteConnection} instance.
 */
public static RemoteGraph open(final Configuration conf) {
    if (!conf.containsKey(RemoteConnection.GREMLIN_REMOTE_CONNECTION_CLASS))
        throw new IllegalArgumentException("Configuration must contain the '" + RemoteConnection.GREMLIN_REMOTE_CONNECTION_CLASS + "' key");

    final RemoteConnection remoteConnection;
    try {
        final Class<? extends RemoteConnection> clazz = Class.forName(conf.getString(RemoteConnection.GREMLIN_REMOTE_CONNECTION_CLASS)).asSubclass(RemoteConnection.class);
        final Constructor<? extends RemoteConnection> ctor = clazz.getConstructor(Configuration.class);
        remoteConnection = ctor.newInstance(conf);
    } catch (Exception ex) {
        throw new IllegalStateException(ex);
    }

    return new RemoteGraph(remoteConnection, conf);
}
 

@Override
public void loadState(final Graph graph, final Configuration configuration) {
    if (configuration.containsKey(INITIAL_RANK_TRAVERSAL))
        this.initialRankTraversal = PureTraversal.loadState(configuration, INITIAL_RANK_TRAVERSAL, graph);
    if (configuration.containsKey(EDGE_TRAVERSAL)) {
        this.edgeTraversal = PureTraversal.loadState(configuration, EDGE_TRAVERSAL, graph);
        this.incidentMessageScope = MessageScope.Local.of(() -> this.edgeTraversal.get().clone());
        this.countMessageScope = MessageScope.Local.of(new MessageScope.Local.ReverseTraversalSupplier(this.incidentMessageScope));
    }
    this.alpha = configuration.getDouble(ALPHA, this.alpha);
    this.epsilon = configuration.getDouble(EPSILON, this.epsilon);
    this.maxIterations = configuration.getInt(MAX_ITERATIONS, 20);
    this.property = configuration.getString(PROPERTY, PAGE_RANK);
    this.vertexComputeKeys = new HashSet<>(Arrays.asList(
            VertexComputeKey.of(this.property, false),
            VertexComputeKey.of(EDGE_COUNT, true)));
    this.memoryComputeKeys = new HashSet<>(Arrays.asList(
            MemoryComputeKey.of(TELEPORTATION_ENERGY, Operator.sum, true, true),
            MemoryComputeKey.of(VERTEX_COUNT, Operator.sum, true, true),
            MemoryComputeKey.of(CONVERGENCE_ERROR, Operator.sum, false, true)));
}
 

@Override
public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
    if (!configuration.containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
        throw new IllegalArgumentException("There is no provided " + Constants.GREMLIN_HADOOP_INPUT_LOCATION + " to read the persisted RDD from");
    Spark.create(sparkContext.sc());
    final Optional<String> graphLocation = Constants.getSearchGraphLocation(configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION), SparkContextStorage.open());
    return graphLocation.isPresent() ? JavaPairRDD.fromJavaRDD((JavaRDD) Spark.getRDD(graphLocation.get()).toJavaRDD()) : JavaPairRDD.fromJavaRDD(sparkContext.emptyRDD());
}
 

public static RichIterable<DbMergeInfo> parseFromProperties(Configuration config) {
    MutableSet<String> dbs = CollectionAdapter.wrapSet(config.getList(String.class, "instances", Lists.mutable.<String>empty()));

    MutableList<String> exceptions = Lists.mutable.empty();
    MutableList<DbMergeInfo> dbMergeInfos = Lists.mutable.empty();
    for (String db : dbs) {
        Configuration subset = config.subset(db);
        if (subset.containsKey("inputDir")) {
            File inputDir = new File(subset.getString("inputDir"));
            if (!inputDir.canRead()) {
                if (inputDir.getPath().contains("\r")) {
                    exceptions.add("Could not find " + db + "." + "inputDir file (use forward-slash instead of back-slash in path): " + inputDir.getPath().replaceAll("\r", ""));
                } else {
                    exceptions.add("Could not find " + db + "." + "inputDir file: " + inputDir);
                }
            }
            DbMergeInfo mergeInfo = new DbMergeInfo(db, inputDir);
            if (subset.containsKey("driverClassName")) {
                mergeInfo.setDriverClassName(subset.getString("driverClassName"));
                mergeInfo.setUrl(subset.getString("url"));
                mergeInfo.setUsername(subset.getString("username"));
                mergeInfo.setPassword(subset.getString("password"));
                mergeInfo.setPhysicalSchema(subset.getString("physicalSchema"));
            }

            dbMergeInfos.add(mergeInfo);
        }
    }

    if (exceptions.notEmpty()) {
        throw new IllegalArgumentException("Invalid properties found in configuration:\n" + exceptions.collect(new Function<String, String>() {
            @Override
            public String valueOf(String it) {
                return "* " + it;
            }
        }).makeString("\n"));
    }
    return dbMergeInfos;
}
 

@Override
public void clear(final Graph graph, final Configuration configuration) throws Exception {
    if (null != graph) {
        if (graph.tx().isOpen()) graph.tx().rollback();
        graph.close();
    }

    if (null != configuration && configuration.containsKey(Neo4jGraph.CONFIG_DIRECTORY)) {
        // this is a non-in-sideEffects configuration so blow away the directory
        final File graphDirectory = new File(configuration.getString(Neo4jGraph.CONFIG_DIRECTORY));
        deleteDirectory(graphDirectory);
    }
}
 

/**
 * Create a {@link RemoteConnection} from a {@code Configuration} object. The configuration must contain a
 * {@code gremlin.remote.remoteConnectionClass} key which is the fully qualified class name of a
 * {@link RemoteConnection} class.
 */
public static RemoteConnection from(final Configuration conf) {
    if (!conf.containsKey(RemoteConnection.GREMLIN_REMOTE_CONNECTION_CLASS))
        throw new IllegalArgumentException("Configuration must contain the '" + GREMLIN_REMOTE_CONNECTION_CLASS + "' key");

    try {
        final Class<? extends RemoteConnection> clazz = Class.forName(
                conf.getString(GREMLIN_REMOTE_CONNECTION_CLASS)).asSubclass(RemoteConnection.class);
        final Constructor<? extends RemoteConnection> ctor = clazz.getConstructor(Configuration.class);
        return ctor.newInstance(conf);
    } catch (Exception ex) {
        throw new IllegalStateException(ex);
    }
}
 

@Override
public void loadState(final Graph graph, final Configuration configuration) {
    if (configuration.containsKey(INITIAL_VOTE_STRENGTH_TRAVERSAL))
        this.initialVoteStrengthTraversal = PureTraversal.loadState(configuration, INITIAL_VOTE_STRENGTH_TRAVERSAL, graph);
    if (configuration.containsKey(EDGE_TRAVERSAL)) {
        this.edgeTraversal = PureTraversal.loadState(configuration, EDGE_TRAVERSAL, graph);
        this.voteScope = MessageScope.Local.of(() -> this.edgeTraversal.get().clone());
        this.countScope = MessageScope.Local.of(new MessageScope.Local.ReverseTraversalSupplier(this.voteScope));
    }
    this.property = configuration.getString(PROPERTY, CLUSTER);
    this.maxIterations = configuration.getInt(MAX_ITERATIONS, 30);
    this.distributeVote = configuration.getBoolean(DISTRIBUTE_VOTE, false);
}
 

public static SubgraphStrategy create(final Configuration configuration) {
    final Builder builder = SubgraphStrategy.build();
    if (configuration.containsKey(VERTICES))
        builder.vertices((Traversal) configuration.getProperty(VERTICES));
    if (configuration.containsKey(EDGES))
        builder.edges((Traversal) configuration.getProperty(EDGES));
    if (configuration.containsKey(VERTEX_PROPERTIES))
        builder.vertexProperties((Traversal) configuration.getProperty(VERTEX_PROPERTIES));
    if (configuration.containsKey(CHECK_ADJACENT_VERTICES))
        builder.checkAdjacentVertices(configuration.getBoolean(CHECK_ADJACENT_VERTICES));
    return builder.create();
}
 

public static ElementIdStrategy create(final Configuration configuration) {
    final ElementIdStrategy.Builder builder = ElementIdStrategy.build();
    if (configuration.containsKey(ID_MAKER))
        builder.idMaker((Supplier) configuration.getProperty(ID_MAKER));
    if (configuration.containsKey(ID_PROPERTY_KEY))
        builder.idPropertyKey(configuration.getString(ID_PROPERTY_KEY));
    return builder.create();
}
 

public static PartitionStrategy create(final Configuration configuration) {
    final PartitionStrategy.Builder builder = PartitionStrategy.build();
    if (configuration.containsKey(INCLUDE_META_PROPERTIES))
        builder.includeMetaProperties(configuration.getBoolean(INCLUDE_META_PROPERTIES));
    if (configuration.containsKey(WRITE_PARTITION))
        builder.writePartition(configuration.getString(WRITE_PARTITION));
    if (configuration.containsKey(PARTITION_KEY))
        builder.partitionKey(configuration.getString(PARTITION_KEY));
    if (configuration.containsKey(READ_PARTITIONS))
        builder.readPartitions(new ArrayList((Collection)configuration.getProperty(READ_PARTITIONS)));
    return builder.create();
}
 

/**
 * Returns the list of additional fields that navigation items should extract from the item descriptor.
 */
public static String[] getNavigationAdditionalFields() {
    Configuration config = ConfigUtils.getCurrentConfig();
    if(config != null && config.containsKey(NAVIGATION_ADDITIONAL_FIELDS_CONFIG_KEY)) {
        return config.getStringArray(NAVIGATION_ADDITIONAL_FIELDS_CONFIG_KEY);
    } else {
        return new String[] {};
    }
}
 

@Override
public boolean contains(final String name, final String key)
{
    final Configuration subset = getConfiguration().subset(name);
    if (subset == null)
    {
        throw new IllegalArgumentException("Mapped property '" + name + "' does not exist.");
    }

    return subset.containsKey(key);
}
 

/**
 * This constructor is largely just for unit testing purposes and should not typically be used externally.
 */
DriverRemoteConnection(final Cluster cluster, final Configuration conf) {
    remoteTraversalSourceName = conf.getString(GREMLIN_REMOTE_DRIVER_SOURCENAME, DEFAULT_TRAVERSAL_SOURCE);

    attachElements = conf.containsKey(GREMLIN_REMOTE + "attachment");

    client = cluster.connect(Client.Settings.build().create()).alias(remoteTraversalSourceName);
    tryCloseCluster = false;
    tryCloseClient = true;
    this.conf = Optional.of(conf);
}
 

public static SeedStrategy create(final Configuration configuration) {
    if (!configuration.containsKey(ID_SEED))
        throw new IllegalArgumentException("SeedStrategy configuration requires a 'seed' value");

    return new SeedStrategy(Long.parseLong(configuration.getProperty(ID_SEED).toString()));
}
 

@Override
public void loadState(final Graph graph, final Configuration configuration) {

    if (configuration.containsKey(SOURCE_VERTEX_FILTER))
        this.sourceVertexFilterTraversal = PureTraversal.loadState(configuration, SOURCE_VERTEX_FILTER, graph);

    if (configuration.containsKey(TARGET_VERTEX_FILTER))
        this.targetVertexFilterTraversal = PureTraversal.loadState(configuration, TARGET_VERTEX_FILTER, graph);

    if (configuration.containsKey(EDGE_TRAVERSAL))
        this.edgeTraversal = PureTraversal.loadState(configuration, EDGE_TRAVERSAL, graph);

    if (configuration.containsKey(DISTANCE_TRAVERSAL))
        this.distanceTraversal = PureTraversal.loadState(configuration, DISTANCE_TRAVERSAL, graph);

    if (configuration.containsKey(MAX_DISTANCE))
        this.maxDistance = (Number) configuration.getProperty(MAX_DISTANCE);

    this.distanceEqualsNumberOfHops = this.distanceTraversal.equals(DEFAULT_DISTANCE_TRAVERSAL);
    this.includeEdges = configuration.getBoolean(INCLUDE_EDGES, false);
    this.standalone = !configuration.containsKey(VertexProgramStep.ROOT_TRAVERSAL);

    if (!this.standalone) {
        this.traversal = PureTraversal.loadState(configuration, VertexProgramStep.ROOT_TRAVERSAL, graph);
        final String programStepId = configuration.getString(ProgramVertexProgramStep.STEP_ID);
        for (final Step step : this.traversal.get().getSteps()) {
            if (step.getId().equals(programStepId)) {
                //noinspection unchecked
                this.programStep = step;
                break;
            }
        }
    }

    // restore halted traversers from the configuration and build an index for direct access
    this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
    this.haltedTraversersIndex = new IndexedTraverserSet<>(v -> v);
    for (final Traverser.Admin<Vertex> traverser : this.haltedTraversers) {
        this.haltedTraversersIndex.add(traverser.split());
    }
    this.memoryComputeKeys.add(MemoryComputeKey.of(SHORTEST_PATHS, Operator.addAll, true, !standalone));
}
 

@Override
public <K, V> JavaPairRDD<K, V> readMemoryRDD(final Configuration configuration, final String memoryKey, final JavaSparkContext sparkContext) {
    if (!configuration.containsKey(Constants.GREMLIN_HADOOP_INPUT_LOCATION))
        throw new IllegalArgumentException("There is no provided " + Constants.GREMLIN_HADOOP_INPUT_LOCATION + " to read the persisted RDD from");
    return JavaPairRDD.fromJavaRDD((JavaRDD) Spark.getRDD(Constants.getMemoryLocation(configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION), memoryKey)).toJavaRDD());
}
 

@Override
public void loadState(final Graph graph, final Configuration configuration) {
    if (!configuration.containsKey(TRAVERSAL))
        throw new IllegalArgumentException("The configuration does not have a traversal: " + TRAVERSAL);
    this.traversal = PureTraversal.loadState(configuration, TRAVERSAL, graph);
    if (!this.traversal.get().isLocked())
        this.traversal.get().applyStrategies();
    /// traversal is compiled and ready to be introspected
    this.traversalMatrix = new TraversalMatrix<>(this.traversal.get());
    // get any master-traversal halted traversers
    this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
    // if results will be serialized out, don't save halted traversers across the cluster
    this.returnHaltedTraversers =
            (this.traversal.get().getParent().asStep().getNextStep() instanceof ComputerResultStep || // if its just going to stream it out, don't distribute
                    this.traversal.get().getParent().asStep().getNextStep() instanceof EmptyStep ||  // same as above, but if using TraversalVertexProgramStep directly
                    (this.traversal.get().getParent().asStep().getNextStep() instanceof ProfileStep && // same as above, but needed for profiling
                            this.traversal.get().getParent().asStep().getNextStep().getNextStep() instanceof ComputerResultStep));

    // determine how to store halted traversers
    final Iterator<?> itty = IteratorUtils.filter(this.traversal.get().getStrategies(), strategy -> strategy instanceof HaltedTraverserStrategy).iterator();
    this.haltedTraverserStrategy = itty.hasNext() ? (HaltedTraverserStrategy) itty.next() : HaltedTraverserStrategy.reference();

    // register traversal side-effects in memory
    this.memoryComputeKeys.addAll(MemoryTraversalSideEffects.getMemoryComputeKeys(this.traversal.get()));
    // register MapReducer memory compute keys
    for (final MapReducer<?, ?, ?, ?, ?> mapReducer : TraversalHelper.getStepsOfAssignableClassRecursively(MapReducer.class, this.traversal.get())) {
        this.mapReducers.add(mapReducer.getMapReduce());
        this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false));
    }
    // register memory computing steps that use memory compute keys
    for (final MemoryComputing<?> memoryComputing : TraversalHelper.getStepsOfAssignableClassRecursively(MemoryComputing.class, this.traversal.get())) {
        this.memoryComputeKeys.add(memoryComputing.getMemoryComputeKey());
    }
    // register profile steps (TODO: try to hide this)
    for (final ProfileStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get())) {
        this.traversal.get().getSideEffects().register(profileStep.getId(), new MutableMetricsSupplier(profileStep.getPreviousStep()), ProfileStep.ProfileBiOperator.instance());
    }
    // register TraversalVertexProgram specific memory compute keys
    this.memoryComputeKeys.add(MemoryComputeKey.of(VOTE_TO_HALT, Operator.and, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(HALTED_TRAVERSERS, Operator.addAll, false, false));
    this.memoryComputeKeys.add(MemoryComputeKey.of(ACTIVE_TRAVERSERS, Operator.addAll, true, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(MUTATED_MEMORY_KEYS, Operator.addAll, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(COMPLETED_BARRIERS, Operator.addAll, true, true));

    // does the traversal need profile information
    this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty();
}
 

public static Graph open(final Configuration conf) {
    if (conf.containsKey("throw")) throw new RuntimeException("you said to throw me");
    return new MockGraph(conf);
}