org.apache.tinkerpop.gremlin.process.computer.Memory Java Examples

@Override
public <R> R get(final String key) throws IllegalArgumentException {
    if (!this.memoryComputeKeys.containsKey(key)) {
        throw Memory.Exceptions.memoryDoesNotExist(key);
    }
    if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast()) {
        throw Memory.Exceptions.memoryDoesNotExist(key);
    }
    final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ?
            this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
    if (null == r || r.isEmpty()) {
        throw Memory.Exceptions.memoryDoesNotExist(key);
    } else {
        return r.get();
    }
}
 

@Override
public void setup(final Memory memory) {
    LOGGER.debug("MedianVertexProgram Started !!!!!!!!");
    memory.set(COUNT, 0L);
    memory.set(LABEL_SELECTED, memory.getIteration());
    memory.set(NEGATIVE_COUNT, 0L);
    memory.set(POSITIVE_COUNT, 0L);
    memory.set(FOUND, false);
    if (persistentProperties.get(ATTRIBUTE_VALUE_TYPE).equals(Schema.VertexProperty.VALUE_LONG.name())) {
        memory.set(MEDIAN, 0L);
        memory.set(PIVOT, 0L);
        memory.set(PIVOT_NEGATIVE, 0L);
        memory.set(PIVOT_POSITIVE, 0L);
    } else {
        memory.set(MEDIAN, 0D);
        memory.set(PIVOT, 0D);
        memory.set(PIVOT_NEGATIVE, 0D);
        memory.set(PIVOT_POSITIVE, 0D);
    }
}
 

private void updateHaltedTraversers(final Vertex vertex, final Memory memory) {
    if (isStartVertex(vertex)) {
        final List<Path> paths = memory.get(SHORTEST_PATHS);
        if (vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent()) {
            // replace the current set of halted traversers with new new traversers that hold the shortest paths
            // found for this vertex
            final TraverserSet<Vertex> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS);
            final TraverserSet<Path> newHaltedTraversers = new TraverserSet<>();
            for (final Traverser.Admin<Vertex> traverser : haltedTraversers) {
                final Vertex v = traverser.get();
                for (final Path path : paths) {
                    if (path.get(0).equals(v)) {
                        newHaltedTraversers.add(traverser.split(path, this.programStep));
                    }
                }
            }
            vertex.property(VertexProperty.Cardinality.single, TraversalVertexProgram.HALTED_TRAVERSERS, newHaltedTraversers);
        }
    }
}
 

@Override
public boolean terminate(final Memory memory) {
    if (memory.isInitialIteration() && this.haltedTraversersIndex != null) {
        this.haltedTraversersIndex.clear();
    }
    final boolean voteToHalt = memory.get(VOTE_TO_HALT);
    if (voteToHalt) {
        final int state = memory.get(STATE);
        if (state == COLLECT_PATHS) {
            // After paths were collected,
            // a) the VP is done in standalone mode (paths will be in memory) or
            // b) the halted traversers will be updated in order to have the paths available in the traversal
            if (this.standalone) return true;
            memory.set(STATE, UPDATE_HALTED_TRAVERSERS);
            return false;
        }
        if (state == UPDATE_HALTED_TRAVERSERS) return true;
        else memory.set(STATE, COLLECT_PATHS); // collect paths if no new paths were found
        return false;
    } else {
        memory.set(VOTE_TO_HALT, true);
        return false;
    }
}
 

@Override
public void execute(Vertex vertex, Messenger<Double> messenger, Memory memory) {
    if (memory.isInitialIteration()) {
        messenger.sendMessage(inE, 1D);
    } else if (1 == memory.getIteration()) {
        double initialPageRank = 1D / vertexCount;
        double edgeCount = IteratorUtils.stream(messenger.receiveMessages()).reduce(0D, (a, b) -> a + b);
        vertex.property(VertexProperty.Cardinality.single, PAGE_RANK, initialPageRank);
        vertex.property(VertexProperty.Cardinality.single, OUTGOING_EDGE_COUNT, edgeCount);
        messenger.sendMessage(outE, initialPageRank / edgeCount);
    } else {
        double newPageRank = IteratorUtils.stream(messenger.receiveMessages()).reduce(0D, (a, b) -> a + b);
        newPageRank =  (dampingFactor * newPageRank) + ((1D - dampingFactor) / vertexCount);
        vertex.property(VertexProperty.Cardinality.single, PAGE_RANK, newPageRank);
        messenger.sendMessage(outE, newPageRank / vertex.<Double>value(OUTGOING_EDGE_COUNT));
    }
}
 

@Override
public boolean terminate(final Memory memory) {
    if (memory.isInitialIteration() && this.haltedTraversersIndex != null) {
        this.haltedTraversersIndex.clear();
    }

    final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT);
    if (voteToHalt) {
        return true;
    } else {
        // it is basically always assumed that the program will want to halt, but if message passing occurs, the
        // program will want to continue, thus reset false values to true for future iterations
        memory.set(VOTE_TO_HALT, true);
        return false;
    }
}
 

protected static void processMemory(final TraversalMatrix<?, ?> traversalMatrix, final Memory memory, final TraverserSet<Object> toProcessTraversers, final Set<String> completedBarriers) {
    // handle traversers and data that were sent from the workers to the master traversal via memory
    if (memory.exists(TraversalVertexProgram.MUTATED_MEMORY_KEYS)) {
        for (final String key : memory.<Set<String>>get(TraversalVertexProgram.MUTATED_MEMORY_KEYS)) {
            final Step<Object, Object> step = traversalMatrix.getStepById(key);
            assert step instanceof Barrier;
            completedBarriers.add(step.getId());
            if (!(step instanceof LocalBarrier)) {  // local barriers don't do any processing on the master traversal (they just lock on the workers)
                final Barrier<Object> barrier = (Barrier<Object>) step;
                barrier.addBarrier(memory.get(key));
                step.forEachRemaining(toProcessTraversers::add);
                // if it was a reducing barrier step, reset the barrier to its seed value
                if (step instanceof ReducingBarrierStep)
                    memory.set(step.getId(), ((ReducingBarrierStep) step).getSeedSupplier().get());
            }
        }
    }
    memory.set(TraversalVertexProgram.MUTATED_MEMORY_KEYS, new HashSet<>());
}
 

@Override
public void workerIterationEnd(final Memory memory) {
    // store profile metrics in proper ProfileStep metrics
    if (this.profile) {
        final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get());
        // guess the profile step to store data
        int profileStepIndex = memory.getIteration();
        // if we guess wrongly write timing into last step
        profileStepIndex = profileStepIndex >= profileSteps.size() ? profileSteps.size() - 1 : profileStepIndex;
        this.iterationMetrics.finish(0);
        // reset counts
        this.iterationMetrics.setCount(TraversalMetrics.TRAVERSER_COUNT_ID,0);
        if (null != MemoryTraversalSideEffects.getMemorySideEffectsPhase(this.traversal.get())) {
            this.traversal.get().getSideEffects().add(profileSteps.get(profileStepIndex).getId(), this.iterationMetrics);
        }
        this.iterationMetrics = null;
    }
}
 

@Override
public boolean terminate(final Memory memory) {
    final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
    MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.TERMINATE);
    checkSideEffects();
    if (memory.isInitialIteration()) {
        assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
        return false;
    } else {
        ///
        assertTrue(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
        final TraverserSet<String> haltedTraversers = memory.get(TraversalVertexProgram.HALTED_TRAVERSERS);
        haltedTraversers.add(generator.generate("hello", this.programStep, 1l));
        haltedTraversers.add(generator.generate("gremlin", this.programStep, 1l));
        memory.set(TraversalVertexProgram.HALTED_TRAVERSERS, haltedTraversers);
        return true;
    }
}
 

@Override
public void set(final String key, final Object value) {
    checkKeyValue(key, value);
    if (this.inExecute)
        throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
    this.currentMap.put(key, value);
}
 

@Override
public PageRankVertexProgram generateProgram(final Graph graph, final Memory memory) {
    final Traversal.Admin<Vertex, Edge> detachedTraversal = this.edgeTraversal.getPure();
    detachedTraversal.setStrategies(TraversalStrategies.GlobalCache.getStrategies(graph.getClass()));
    final PageRankVertexProgram.Builder builder = PageRankVertexProgram.build()
            .property(this.pageRankProperty)
            .iterations(this.times + 1)
            .alpha(this.alpha)
            .edges(detachedTraversal);
    if (this.previousTraversalVertexProgram())
        builder.initialRank(new HaltedTraversersCountTraversal());
    return builder.create(graph);
}
 

private void processMemorySideEffects(final Memory memory) {
    // update the traversal side-effects with the state of the memory after the OLAP job execution
    final TraversalSideEffects sideEffects = this.getTraversal().getSideEffects();
    for (final String key : memory.keys()) {
        if (sideEffects.exists(key)) {
            // halted traversers should never be propagated through sideEffects
            assert !key.equals(TraversalVertexProgram.HALTED_TRAVERSERS);
            sideEffects.set(key, memory.get(key));
        }
    }
}
 

/**
 * Move any valid path into the VP's memory.
 * @param vertex The current vertex.
 * @param memory The VertexProgram's memory.
 */
private void collectShortestPaths(final Vertex vertex, final Memory memory) {

    final VertexProperty<Map<Vertex, Pair<Number, Set<Path>>>> pathProperty = vertex.property(PATHS);

    if (pathProperty.isPresent()) {

        final Map<Vertex, Pair<Number, Set<Path>>> paths = pathProperty.value();
        final List<Path> result = new ArrayList<>();

        for (final Pair<Number, Set<Path>> pair : paths.values()) {
            for (final Path path : pair.getValue1()) {
                if (isEndVertex(vertex)) {
                    if (this.distanceEqualsNumberOfHops ||
                            this.maxDistance == null ||
                            NumberHelper.compare(pair.getValue0(), this.maxDistance) <= 0) {
                        result.add(path);
                    }
                }
            }
        }

        pathProperty.remove();

        memory.add(SHORTEST_PATHS, result);
    }
}
 

@Override
public boolean terminate(final Memory memory) {
    final boolean voteToHalt = memory.<Boolean>get(VOTE_TO_HALT) || memory.getIteration() >= (this.distributeVote ? this.maxIterations + 1 : this.maxIterations);
    if (voteToHalt) {
        return true;
    } else {
        memory.set(VOTE_TO_HALT, true);
        return false;
    }
}
 

@Override
public void setup(final Memory memory) {
    MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.SETUP);
    final Map<Vertex, Long> map = (Map<Vertex, Long>) this.haltedTraversers.iterator().next().get();
    assertEquals(2, map.size());
    assertTrue(map.values().contains(3l));
    assertTrue(map.values().contains(1l));
    final IndexedTraverserSet<Object,Vertex> activeTraversers = new IndexedTraverserSet.VertexIndexedTraverserSet();
    map.keySet().forEach(vertex -> activeTraversers.add(this.haltedTraversers.peek().split(vertex, EmptyStep.instance())));
    this.haltedTraversers.clear();
    this.checkSideEffects();
    memory.set(TraversalVertexProgram.ACTIVE_TRAVERSERS, activeTraversers);
}
 

@Override
public void setup(final Memory memory) {
    // memory is local
    MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.SETUP);
    ((MemoryTraversalSideEffects) this.traversal.get().getSideEffects()).storeSideEffectsInMemory();
    memory.set(VOTE_TO_HALT, true);
    memory.set(MUTATED_MEMORY_KEYS, new HashSet<>());
    memory.set(COMPLETED_BARRIERS, new HashSet<>());
    // if halted traversers are being sent from a previous VertexProgram in an OLAP chain (non-distributed traversers), get them into the flow
    if (!this.haltedTraversers.isEmpty()) {
        final TraverserSet<Object> toProcessTraversers = new TraverserSet<>();
        IteratorUtils.removeOnNext(this.haltedTraversers.iterator()).forEachRemaining(traverser -> {
            traverser.setStepId(this.traversal.get().getStartStep().getId());
            toProcessTraversers.add(traverser);
        });
        assert this.haltedTraversers.isEmpty();
        final IndexedTraverserSet<Object,Vertex> remoteActiveTraversers = new IndexedTraverserSet.VertexIndexedTraverserSet();
        MasterExecutor.processTraversers(this.traversal, this.traversalMatrix, toProcessTraversers, remoteActiveTraversers, this.haltedTraversers, this.haltedTraverserStrategy);
        memory.set(HALTED_TRAVERSERS, this.haltedTraversers);
        memory.set(ACTIVE_TRAVERSERS, remoteActiveTraversers);
    } else {
        memory.set(HALTED_TRAVERSERS, new TraverserSet<>());
        memory.set(ACTIVE_TRAVERSERS, new IndexedTraverserSet.VertexIndexedTraverserSet());
    }
    // local variable will no longer be used so null it for GC
    this.haltedTraversers = null;
    // does the traversal need profile information
    this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty();
}
 

@Override
public void set(final String key, final Object value) {
    checkKeyValue(key, value);
    if (this.inExecute) {
        throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
    } else {
        this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
    }
}
 

@Override
public void set(final String key, final Object value) {
    checkKeyValue(key, value);
    if (this.inExecute)
        throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
    this.currentMap.put(key, value);
}
 

@Override
public void safeExecute(final Vertex vertex, Messenger<Long> messenger, final Memory memory) {
    switch (memory.getIteration()) {
        case 0:
            messenger.sendMessage(messageScopeOut, 1L);
            break;
        case 1:
            if (messenger.receiveMessages().hasNext()) {
                vertex.property(EDGE_COUNT, getMessageCount(messenger));
            }
            break;
        default:
            throw GraknAnalyticsException.unreachableStatement("Exceeded expected maximum number of iterations");
    }
}
 

@Override
public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) {
    if (mode == Mode.UNSET)
        throw new IllegalStateException("IO mode was not set to read() or write()");
    else if (mode == Mode.READING)
        configureForRead(graph);
    else if (mode == Mode.WRITING)
        configureForWrite(graph);
    else
        throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name());

    return CloneVertexProgram.build().create(graph);
}
 

@Override
public void set(final String key, final Object value) {
    checkKeyValue(key, value);
    if (this.inExecute)
        throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
    else
        this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
}
 

@Override
public <R> R get(final String key) throws IllegalArgumentException {
    if (!this.memoryComputeKeys.containsKey(key))
        throw Memory.Exceptions.memoryDoesNotExist(key);
    if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
        throw Memory.Exceptions.memoryDoesNotExist(key);
    final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value());
    if (null == r || r.isEmpty())
        throw Memory.Exceptions.memoryDoesNotExist(key);
    else
        return r.get();
}
 

@Override
public void safeExecute(final Vertex vertex, Messenger<Boolean> messenger, final Memory memory) {
    if (memory.isInitialIteration()) {
        if (vertex.id().toString().equals(persistentProperties.get(SOURCE))) {
            update(vertex, messenger, memory, persistentProperties.get(SOURCE).toString());
        }
    } else {
        if (messenger.receiveMessages().hasNext() && !vertex.property(CLUSTER_LABEL).isPresent()) {
            update(vertex, messenger, memory, persistentProperties.get(SOURCE).toString());
        }
    }
}
 

@Override
public <R> R get(final String key) throws IllegalArgumentException {
    final R r = (R) this.previousMap.get(key);
    if (null == r)
        throw Memory.Exceptions.memoryDoesNotExist(key);
    else if (this.inExecute && !this.memoryKeys.get(key).isBroadcast())
        throw Memory.Exceptions.memoryDoesNotExist(key);
    else
        return r;
}
 

@Override
public void safeExecute(Vertex vertex, Messenger<Long> messenger, Memory memory) {
    switch (memory.getIteration()) {
        case 0:
            degreeMessagePassing(messenger);
            break;
        case 1:
            degreeMessageCounting(messenger, vertex);
            break;
        default:
            throw GraknAnalyticsException.unreachableStatement("Exceeded expected maximum number of iterations");
    }
}
 

@Override
public void execute(final Vertex vertex, final Messenger messenger, final Memory memory) {
    assertFalse(memory.exists(TraversalVertexProgram.HALTED_TRAVERSERS));
    final TraverserGenerator generator = this.traversal.get().getTraverserGenerator();
    MemoryTraversalSideEffects.setMemorySideEffects(this.traversal.get(), memory, ProgramPhase.EXECUTE);
    this.checkSideEffects();
    final TraverserSet<Vertex> activeTraversers = memory.get(TraversalVertexProgram.ACTIVE_TRAVERSERS);
    if (vertex.label().equals("software")) {
        assertEquals(1, activeTraversers.stream().filter(v -> v.get().equals(vertex)).count());
        if (memory.isInitialIteration()) {
            assertFalse(vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent());
            vertex.property(
                    TraversalVertexProgram.HALTED_TRAVERSERS,
                    new TraverserSet<>(generator.generate(vertex.value("name"), this.programStep, 1l)));
        } else {
            assertTrue(vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent());
        }
    } else {
        assertFalse(vertex.property(TraversalVertexProgram.HALTED_TRAVERSERS).isPresent());
        assertEquals(0, activeTraversers.stream().filter(v -> v.get().equals(vertex)).count());
        if (!memory.isInitialIteration()) {
            if (vertex.value("name").equals("marko"))
                memory.add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(generator.generate("marko-is-my-name", this.programStep, 1l)));
            else if (vertex.value("name").equals("vadas"))
                this.traversal.get().getSideEffects().add(TraversalVertexProgram.HALTED_TRAVERSERS, new TraverserSet<>(generator.generate("the-v-o-double-g", this.programStep, 1l)));
        }
    }
}
 

@Override
public boolean terminate(final Memory memory) {
    LOGGER.debug("Finished Iteration {}", memory.getIteration());
    if (memory.isInitialIteration()) return false;

    if (memory.getIteration() == MAX_ITERATION) {
        LOGGER.debug("Reached Max Iteration: {}", MAX_ITERATION);
        throw GraqlQueryException.maxIterationsReached(this.getClass());
    }

    if (memory.<Boolean>get(CONNECTED_COMPONENT_STARTED)) {
        if (memory.<Boolean>get(VOTE_TO_HALT)) {
            LOGGER.debug("KCoreVertexProgram Finished");
            return true; // connected component is done
        } else {
            memory.set(VOTE_TO_HALT, true);
            return false;
        }
    } else {
        if (!atRelations(memory)) {
            if (!memory.<Boolean>get(K_CORE_EXIST)) {
                LOGGER.debug("KCoreVertexProgram Finished");
                LOGGER.debug("No Such Core Areas Found");
                throw new NoResultException();
            } else {
                if (memory.<Boolean>get(K_CORE_STABLE)) {
                    memory.set(CONNECTED_COMPONENT_STARTED, true);
                    LOGGER.debug("Found Core Areas");
                    LOGGER.debug("Starting Connected Components");
                } else {
                    memory.set(K_CORE_EXIST, false);
                    memory.set(K_CORE_STABLE, true);
                }
                return false;
            }
        } else {
            return false; // can not end after relayOrSaveMessages
        }
    }
}
 

@Override
public <R> R get(final String key) throws IllegalArgumentException {
    final R r = (R) this.memoryMap.get(key);
    if (null == r)
        throw Memory.Exceptions.memoryDoesNotExist(key);
    else
        return r;
}
 

private static void relayClusterLabel(Messenger<String> messenger, Memory memory) {
    String firstMessage = messenger.receiveMessages().next();
    String max = IteratorUtils.reduce(messenger.receiveMessages(), firstMessage,
            (a, b) -> a.compareTo(b) > 0 ? a : b);
    sendMessage(messenger, max);
    memory.add(VOTE_TO_HALT, false);
}
 

private static void updateClusterLabel(Vertex vertex, Messenger<String> messenger, Memory memory) {
    String currentMax = vertex.value(K_CORE_LABEL);
    String max = IteratorUtils.reduce(messenger.receiveMessages(), currentMax,
            (a, b) -> a.compareTo(b) > 0 ? a : b);
    if (!max.equals(currentMax)) {
        LOGGER.trace("Cluster label of {} changed from {} to {}", vertex, currentMax, max);
        vertex.property(K_CORE_LABEL, max);
        sendMessage(messenger, max);
        memory.add(VOTE_TO_HALT, false);
    } else {
        LOGGER.trace("Cluster label of {} is still {}", vertex, currentMax);
    }
}