Java Code Examples for org.nd4j.linalg.ops.transforms.Transforms#softPlus()
The following examples show how to use
org.nd4j.linalg.ops.transforms.Transforms#softPlus() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: SameDiffTests.java From nd4j with Apache License 2.0 | 4 votes |
|
@Test
public void testActivationBackprop() {
Activation[] afns = new Activation[]{
Activation.TANH,
Activation.SIGMOID,
Activation.ELU,
Activation.SOFTPLUS,
Activation.SOFTSIGN,
Activation.HARDTANH,
Activation.CUBE, //WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
Activation.RELU, //JVM crash
Activation.LEAKYRELU //JVM crash
};
for (Activation a : afns) {
SameDiff sd = SameDiff.create();
INDArray inArr = Nd4j.linspace(-3, 3, 7);
INDArray labelArr = Nd4j.linspace(-3, 3, 7).muli(0.5);
SDVariable in = sd.var("in", inArr.dup());
// System.out.println("inArr: " + inArr);
INDArray outExp;
SDVariable out;
switch (a) {
case ELU:
out = sd.elu("out", in);
outExp = Transforms.elu(inArr, true);
break;
case HARDTANH:
out = sd.hardTanh("out", in);
outExp = Transforms.hardTanh(inArr, true);
break;
case LEAKYRELU:
out = sd.leakyRelu("out", in, 0.01);
outExp = Transforms.leakyRelu(inArr, true);
break;
case RELU:
out = sd.relu("out", in, 0.0);
outExp = Transforms.relu(inArr, true);
break;
case SIGMOID:
out = sd.sigmoid("out", in);
outExp = Transforms.sigmoid(inArr, true);
break;
case SOFTPLUS:
out = sd.softplus("out", in);
outExp = Transforms.softPlus(inArr, true);
break;
case SOFTSIGN:
out = sd.softsign("out", in);
outExp = Transforms.softsign(inArr, true);
break;
case TANH:
out = sd.tanh("out", in);
outExp = Transforms.tanh(inArr, true);
break;
case CUBE:
out = sd.cube("out", in);
outExp = Transforms.pow(inArr, 3, true);
break;
default:
throw new RuntimeException(a.toString());
}
//Sum squared error loss:
SDVariable label = sd.var("label", labelArr.dup());
SDVariable diff = label.sub("diff", out);
SDVariable sqDiff = diff.mul("sqDiff", diff);
SDVariable totSum = sd.sum("totSum", sqDiff, Integer.MAX_VALUE); //Loss function...
sd.exec();
INDArray outAct = sd.getVariable("out").getArr();
assertEquals(a.toString(), outExp, outAct);
// L = sum_i (label - out)^2
//dL/dOut = 2(out - label)
INDArray dLdOutExp = outExp.sub(labelArr).mul(2);
INDArray dLdInExp = a.getActivationFunction().backprop(inArr.dup(), dLdOutExp.dup()).getFirst();
sd.execBackwards();
SameDiff gradFn = sd.getFunction("grad");
INDArray dLdOutAct = gradFn.getVariable("out-grad").getArr();
INDArray dLdInAct = gradFn.getVariable("in-grad").getArr();
assertEquals(a.toString(), dLdOutExp, dLdOutAct);
assertEquals(a.toString(), dLdInExp, dLdInAct);
}
}
Example 2
| Source File: SameDiffTests.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Test
public void testActivationBackprop() {
Activation[] afns = new Activation[]{
Activation.TANH,
Activation.SIGMOID,
Activation.ELU,
Activation.SOFTPLUS,
Activation.SOFTSIGN,
Activation.HARDTANH,
Activation.CUBE,
//WRONG output - see issue https://github.com/deeplearning4j/nd4j/issues/2426
Activation.RELU, //JVM crash
Activation.LEAKYRELU //JVM crash
};
for (Activation a : afns) {
SameDiff sd = SameDiff.create();
INDArray inArr = Nd4j.linspace(-3, 3, 7);
INDArray labelArr = Nd4j.linspace(-3, 3, 7).muli(0.5);
SDVariable in = sd.var("in", inArr.dup());
// System.out.println("inArr: " + inArr);
INDArray outExp;
SDVariable out;
switch (a) {
case ELU:
out = sd.nn().elu("out", in);
outExp = Transforms.elu(inArr, true);
break;
case HARDTANH:
out = sd.nn().hardTanh("out", in);
outExp = Transforms.hardTanh(inArr, true);
break;
case LEAKYRELU:
out = sd.nn().leakyRelu("out", in, 0.01);
outExp = Transforms.leakyRelu(inArr, true);
break;
case RELU:
out = sd.nn().relu("out", in, 0.0);
outExp = Transforms.relu(inArr, true);
break;
case SIGMOID:
out = sd.nn().sigmoid("out", in);
outExp = Transforms.sigmoid(inArr, true);
break;
case SOFTPLUS:
out = sd.nn().softplus("out", in);
outExp = Transforms.softPlus(inArr, true);
break;
case SOFTSIGN:
out = sd.nn().softsign("out", in);
outExp = Transforms.softsign(inArr, true);
break;
case TANH:
out = sd.math().tanh("out", in);
outExp = Transforms.tanh(inArr, true);
break;
case CUBE:
out = sd.math().cube("out", in);
outExp = Transforms.pow(inArr, 3, true);
break;
default:
throw new RuntimeException(a.toString());
}
//Sum squared error loss:
SDVariable label = sd.var("label", labelArr.dup());
SDVariable diff = label.sub("diff", out);
SDVariable sqDiff = diff.mul("sqDiff", diff);
SDVariable totSum = sd.sum("totSum", sqDiff, Integer.MAX_VALUE); //Loss function...
Map<String,INDArray> m = sd.output(Collections.emptyMap(), "out");
INDArray outAct = m.get("out");
assertEquals(a.toString(), outExp, outAct);
// L = sum_i (label - out)^2
//dL/dOut = 2(out - label)
INDArray dLdOutExp = outExp.sub(labelArr).mul(2);
INDArray dLdInExp = a.getActivationFunction().backprop(inArr.dup(), dLdOutExp.dup()).getFirst();
Map<String,INDArray> grads = sd.calculateGradients(null, "out", "in");
// sd.execBackwards(Collections.emptyMap());
// SameDiff gradFn = sd.getFunction("grad");
INDArray dLdOutAct = grads.get("out");
INDArray dLdInAct = grads.get("in");
assertEquals(a.toString(), dLdOutExp, dLdOutAct);
assertEquals(a.toString(), dLdInExp, dLdInAct);
}
}
