org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep Java Examples

The following examples show how to use org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Grouping.java    From tinkerpop with Apache License 2.0 6 votes vote down vote up
/**
 * Determines the first (non-local) barrier step in the provided traversal. This method is used by {@link GroupStep}
 * and {@link GroupSideEffectStep} to ultimately determine the reducing bi-operator.
 *
 * @param traversal The traversal to inspect.
 * @return The first non-local barrier step or {@code null} if no such step was found.
 */
public default Barrier determineBarrierStep(final Traversal.Admin<S, V> traversal) {
    final List<Step> steps = traversal.getSteps();
    for (int ix = 0; ix < steps.size(); ix++) {
        final Step step = steps.get(ix);
        if (step instanceof Barrier && !(step instanceof LocalBarrier)) {
            final Barrier b = (Barrier) step;

            // when profile() is enabled the step needs to be wrapped up with the barrier so that the timer on
            // the ProfileStep is properly triggered
            if (ix < steps.size() - 1 && steps.get(ix + 1) instanceof ProfileStep)
                return new ProfilingAware.ProfiledBarrier(b, (ProfileStep) steps.get(ix + 1));
            else
                return b;
        }
    }
    return null;
}
 
Example #2
Source File: DefaultTraversalMetrics.java    From tinkerpop with Apache License 2.0 6 votes vote down vote up
private void addTopLevelMetrics(final Traversal.Admin traversal, final boolean onGraphComputer) {
    this.totalStepDuration = 0;

    final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfClass(ProfileStep.class, traversal);
    final List<Pair<Integer, MutableMetrics>> tempMetrics = new ArrayList<>(profileSteps.size());

    for (int ii = 0; ii < profileSteps.size(); ii++) {
        // The index is necessary to ensure that step order is preserved after a merge.
        final ProfileStep step = profileSteps.get(ii);
        final MutableMetrics stepMetrics = onGraphComputer ? traversal.getSideEffects().get(step.getId()) : step.getMetrics();

        this.totalStepDuration += stepMetrics.getDuration(MutableMetrics.SOURCE_UNIT);
        tempMetrics.add(Pair.with(ii, stepMetrics.clone()));
    }

    tempMetrics.forEach(m -> {
        final double dur = m.getValue1().getDuration(TimeUnit.NANOSECONDS) * 100.d / this.totalStepDuration;
        m.getValue1().setAnnotation(PERCENT_DURATION_KEY, dur);
    });

    tempMetrics.forEach(p -> {
        this.stepIndexedMetrics.put(p.getValue1().getId(), p.getValue1().getImmutableClone());
        this.positionIndexedMetrics.put(p.getValue0(), p.getValue1().getImmutableClone());
    });
}
 
Example #3
Source File: TraversalVertexProgram.java    From tinkerpop with Apache License 2.0 6 votes vote down vote up
@Override
public void workerIterationEnd(final Memory memory) {
    // store profile metrics in proper ProfileStep metrics
    if (this.profile) {
        final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get());
        // guess the profile step to store data
        int profileStepIndex = memory.getIteration();
        // if we guess wrongly write timing into last step
        profileStepIndex = profileStepIndex >= profileSteps.size() ? profileSteps.size() - 1 : profileStepIndex;
        this.iterationMetrics.finish(0);
        // reset counts
        this.iterationMetrics.setCount(TraversalMetrics.TRAVERSER_COUNT_ID,0);
        if (null != MemoryTraversalSideEffects.getMemorySideEffectsPhase(this.traversal.get())) {
            this.traversal.get().getSideEffects().add(profileSteps.get(profileStepIndex).getId(), this.iterationMetrics);
        }
        this.iterationMetrics = null;
    }
}
 
Example #4
Source File: ProfileTest.java    From tinkerpop with Apache License 2.0 6 votes vote down vote up
private void validate_g_V_out_out_profile_modern(final Traversal traversal, final TraversalMetrics traversalMetrics) {
    traversalMetrics.toString(); // ensure no exceptions are thrown

    assumeThat("The following assertions apply to TinkerGraph only as provider strategies can alter the steps to not comply with expectations",
            graph.getClass().getSimpleName(), equalTo("TinkerGraph"));

    Metrics metrics = traversalMetrics.getMetrics(0);
    assertEquals(6, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());
    assertEquals(6, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());

    metrics = traversalMetrics.getMetrics(1);
    assertEquals(6, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());
    assertNotEquals(0, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());

    metrics = traversalMetrics.getMetrics(2);
    assertEquals(2, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());
    assertNotEquals(0, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());

    if (!onGraphComputer(traversal.asAdmin())) {
        // Every other step should be a Profile step
        List<Step> steps = traversal.asAdmin().getSteps();
        for (int ii = 1; ii <= 6; ii += 2) {
            assertEquals("Every other Step should be a ProfileStep.", ProfileStep.class, steps.get(ii).getClass());
        }
    }
}
 
Example #5
Source File: JanusGraphVertexStep.java    From grakn with GNU Affero General Public License v3.0 5 votes vote down vote up
/**
 * Many parent traversals drip feed their start vertices in one at a time. To best exploit
 * the multiQuery we need to load all possible starts in one go so this method will attempt
 * to find a JanusGraphMultiQueryStep with the starts of the parent, and if found cache it.
 */
private void setParentMultiQueryStep() {
    Step firstStep = traversal.getStartStep();
    while (firstStep instanceof StartStep || firstStep instanceof SideEffectStep) {
        // Want the next step if this is a side effect
        firstStep = firstStep.getNextStep();
    }
    if (this.equals(firstStep)) {
        Step<?, ?> parentStep = traversal.getParent().asStep();
        if (JanusGraphTraversalUtil.isMultiQueryCompatibleStep(parentStep)) {
            Step<?, ?> parentPreviousStep = parentStep.getPreviousStep();
            if (parentStep instanceof RepeatStep) {
                RepeatStep repeatStep = (RepeatStep) parentStep;
                List<RepeatEndStep> repeatEndSteps = TraversalHelper.getStepsOfClass(RepeatEndStep.class, repeatStep.getRepeatTraversal());
                if (repeatEndSteps.size() == 1) {
                    parentPreviousStep = repeatEndSteps.get(0).getPreviousStep();
                }
            }
            if (parentPreviousStep instanceof ProfileStep) {
                parentPreviousStep = parentPreviousStep.getPreviousStep();
            }
            if (parentPreviousStep instanceof JanusGraphMultiQueryStep) {
                parentMultiQueryStep = (JanusGraphMultiQueryStep) parentPreviousStep;
            }
        }
    }
}
 
Example #6
Source File: WhereTraversalStep.java    From tinkerpop with Apache License 2.0 5 votes vote down vote up
@Override
protected Object map(final Traverser.Admin<S> traverser) {
    if (this.getTraversal().getEndStep() instanceof WhereEndStep)
        ((WhereEndStep) this.getTraversal().getEndStep()).processStartTraverser(traverser);
    else if (this.getTraversal().getEndStep() instanceof ProfileStep && this.getTraversal().getEndStep().getPreviousStep() instanceof WhereEndStep)     // TOTAL SUCKY HACK!
        ((WhereEndStep) this.getTraversal().getEndStep().getPreviousStep()).processStartTraverser(traverser);
    return null == this.selectKey ? traverser.get() : this.getSafeScopeValue(Pop.last, this.selectKey, traverser);
}
 
Example #7
Source File: ProfileStrategy.java    From tinkerpop with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
    if (!traversal.getEndStep().getLabels().contains(MARKER) &&
            (traversal.isRoot() || traversal.getParent() instanceof VertexProgramStep) &&
            TraversalHelper.hasStepOfAssignableClassRecursively(ProfileSideEffectStep.class, traversal))
        TraversalHelper.applyTraversalRecursively(t -> t.getEndStep().addLabel(MARKER), traversal);

    if (traversal.getEndStep().getLabels().contains(MARKER)) {
        traversal.getEndStep().removeLabel(MARKER);
        // Add .profile() step after every pre-existing step.
        final List<Step> steps = traversal.getSteps();
        final int numSteps = steps.size();
        for (int i = 0; i < numSteps; i++) {
            // Do not inject profiling after ProfileSideEffectStep as this will be the last step on the root traversal.
            if (steps.get(i * 2) instanceof ProfileSideEffectStep)
                break;
            // Create and inject ProfileStep
            final ProfileStep profileStepToAdd = new ProfileStep(traversal);
            traversal.addStep((i * 2) + 1, profileStepToAdd);

            final Step stepToBeProfiled = traversal.getSteps().get(i * 2);
            if (stepToBeProfiled instanceof ProfilingAware) {
                ((ProfilingAware) stepToBeProfiled).prepareForProfiling();
            }
        }
    }
}
 
Example #8
Source File: ComputerVerificationStrategy.java    From tinkerpop with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
    if (!TraversalHelper.onGraphComputer(traversal))
        return;

    if (traversal.getParent() instanceof TraversalVertexProgramStep) {
        if (TraversalHelper.getStepsOfAssignableClassRecursively(GraphStep.class, traversal).size() > 1)
            throw new VerificationException("Mid-traversal V()/E() is currently not supported on GraphComputer", traversal);
        if (TraversalHelper.hasStepOfAssignableClassRecursively(ProfileStep.class, traversal) && TraversalHelper.getStepsOfAssignableClass(VertexProgramStep.class, TraversalHelper.getRootTraversal(traversal)).size() > 1)
            throw new VerificationException("Profiling a multi-VertexProgramStep traversal is currently not supported on GraphComputer", traversal);
    }

    // this is a problem because sideEffect.merge() is transient on the OLAP reduction
    if (TraversalHelper.getRootTraversal(traversal).getTraverserRequirements().contains(TraverserRequirement.ONE_BULK))
        throw new VerificationException("One bulk is currently not supported on GraphComputer: " + traversal, traversal);

    // you can not traverse past the local star graph with localChildren (e.g. by()-modulators).
    if (!TraversalHelper.isGlobalChild(traversal) && !TraversalHelper.isLocalStarGraph(traversal))
        throw new VerificationException("Local traversals may not traverse past the local star-graph on GraphComputer: " + traversal, traversal);

    for (final Step<?, ?> step : traversal.getSteps()) {
        if (step instanceof PathProcessor && ((PathProcessor) step).getMaxRequirement() != PathProcessor.ElementRequirement.ID)
            throw new VerificationException("It is not possible to access more than a path element's id on GraphComputer: " + step + " requires " + ((PathProcessor) step).getMaxRequirement(), traversal);

        if (UNSUPPORTED_STEPS.stream().filter(c -> c.isAssignableFrom(step.getClass())).findFirst().isPresent())
            throw new VerificationException("The following step is currently not supported on GraphComputer: " + step, traversal);
    }

    Step<?, ?> nextParentStep = traversal.getParent().asStep();
    while (!(nextParentStep instanceof EmptyStep)) {
        if (nextParentStep instanceof PathProcessor && ((PathProcessor) nextParentStep).getMaxRequirement() != PathProcessor.ElementRequirement.ID)
            throw new VerificationException("The following path processor step requires more than the element id on GraphComputer: " + nextParentStep + " requires " + ((PathProcessor) nextParentStep).getMaxRequirement(), traversal);
        nextParentStep = nextParentStep.getNextStep();
    }
}
 
Example #9
Source File: SqlgWhereTraversalStepBarrier.java    From sqlg with MIT License 5 votes vote down vote up
@Override
protected Object map(final Traverser.Admin<S> traverser) {
    if (this.getTraversal().getEndStep() instanceof SqlgWhereTraversalStepBarrier.SqlgWhereEndStep)
        ((SqlgWhereEndStep) this.getTraversal().getEndStep()).processStartTraverser(traverser);
    else if (this.getTraversal().getEndStep() instanceof ProfileStep && this.getTraversal().getEndStep().getPreviousStep() instanceof SqlgWhereTraversalStepBarrier.SqlgWhereEndStep)     // TOTAL SUCKY HACK!
        ((SqlgWhereEndStep) this.getTraversal().getEndStep().getPreviousStep()).processStartTraverser(traverser);
    return null == this.selectKey ? traverser.get() : this.getScopeValue(Pop.last, this.selectKey, traverser);
}
 
Example #10
Source File: MatchStep.java    From tinkerpop with Apache License 2.0 4 votes vote down vote up
public static Optional<String> getEndLabel(final Traversal.Admin<Object, Object> traversal) {
    final Step<?, ?> endStep = traversal.getEndStep();
    return endStep instanceof ProfileStep ?           // TOTAL HACK
            ((MatchEndStep) endStep.getPreviousStep()).getMatchKey() :
            ((MatchEndStep) endStep).getMatchKey();
}
 
Example #11
Source File: ProfilingAware.java    From tinkerpop with Apache License 2.0 4 votes vote down vote up
public ProfiledBarrier(final Barrier barrier, final ProfileStep profileStep) {
    this.barrier = barrier;
    this.profileStep = profileStep;
}
 
Example #12
Source File: VertexProgramStep.java    From tinkerpop with Apache License 2.0 4 votes vote down vote up
protected boolean isEndStep() {
    return this.getNextStep() instanceof ComputerResultStep || (this.getNextStep() instanceof ProfileStep && this.getNextStep().getNextStep() instanceof ComputerResultStep);
}
 
Example #13
Source File: TraversalVertexProgram.java    From tinkerpop with Apache License 2.0 4 votes vote down vote up
@Override
public void loadState(final Graph graph, final Configuration configuration) {
    if (!configuration.containsKey(TRAVERSAL))
        throw new IllegalArgumentException("The configuration does not have a traversal: " + TRAVERSAL);
    this.traversal = PureTraversal.loadState(configuration, TRAVERSAL, graph);
    if (!this.traversal.get().isLocked())
        this.traversal.get().applyStrategies();
    /// traversal is compiled and ready to be introspected
    this.traversalMatrix = new TraversalMatrix<>(this.traversal.get());
    // get any master-traversal halted traversers
    this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
    // if results will be serialized out, don't save halted traversers across the cluster
    this.returnHaltedTraversers =
            (this.traversal.get().getParent().asStep().getNextStep() instanceof ComputerResultStep || // if its just going to stream it out, don't distribute
                    this.traversal.get().getParent().asStep().getNextStep() instanceof EmptyStep ||  // same as above, but if using TraversalVertexProgramStep directly
                    (this.traversal.get().getParent().asStep().getNextStep() instanceof ProfileStep && // same as above, but needed for profiling
                            this.traversal.get().getParent().asStep().getNextStep().getNextStep() instanceof ComputerResultStep));

    // determine how to store halted traversers
    final Iterator<?> itty = IteratorUtils.filter(this.traversal.get().getStrategies(), strategy -> strategy instanceof HaltedTraverserStrategy).iterator();
    this.haltedTraverserStrategy = itty.hasNext() ? (HaltedTraverserStrategy) itty.next() : HaltedTraverserStrategy.reference();

    // register traversal side-effects in memory
    this.memoryComputeKeys.addAll(MemoryTraversalSideEffects.getMemoryComputeKeys(this.traversal.get()));
    // register MapReducer memory compute keys
    for (final MapReducer<?, ?, ?, ?, ?> mapReducer : TraversalHelper.getStepsOfAssignableClassRecursively(MapReducer.class, this.traversal.get())) {
        this.mapReducers.add(mapReducer.getMapReduce());
        this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false));
    }
    // register memory computing steps that use memory compute keys
    for (final MemoryComputing<?> memoryComputing : TraversalHelper.getStepsOfAssignableClassRecursively(MemoryComputing.class, this.traversal.get())) {
        this.memoryComputeKeys.add(memoryComputing.getMemoryComputeKey());
    }
    // register profile steps (TODO: try to hide this)
    for (final ProfileStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get())) {
        this.traversal.get().getSideEffects().register(profileStep.getId(), new MutableMetricsSupplier(profileStep.getPreviousStep()), ProfileStep.ProfileBiOperator.instance());
    }
    // register TraversalVertexProgram specific memory compute keys
    this.memoryComputeKeys.add(MemoryComputeKey.of(VOTE_TO_HALT, Operator.and, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(HALTED_TRAVERSERS, Operator.addAll, false, false));
    this.memoryComputeKeys.add(MemoryComputeKey.of(ACTIVE_TRAVERSERS, Operator.addAll, true, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(MUTATED_MEMORY_KEYS, Operator.addAll, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(COMPLETED_BARRIERS, Operator.addAll, true, true));

    // does the traversal need profile information
    this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty();
}