org.datavec.api.transform.MathOp Java Examples

@Test
public void testCustomTransform() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnDouble("secondCol").build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .transform(new CustomTransform("secondCol", 3.14159))
                    .doubleMathOp("secondCol", MathOp.Multiply, 2.0).filter(new CustomFilter(123))
                    .filter(new CustomCondition("someArg")).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
 

@Test
public void testTransformProcessAndSchema() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnNDArray("nd1a", new long[] {1, 10})
                    .addColumnNDArray("nd1b", new long[] {1, 10}).addColumnNDArray("nd2", new long[] {1, 100})
                    .addColumnNDArray("nd3", new long[] {-1, -1}).build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .ndArrayColumnsMathOpTransform("added", MathOp.Add, "nd1a", "nd1b")
                    .ndArrayMathFunctionTransform("nd2", MathFunction.SQRT)
                    .ndArrayScalarOpTransform("nd3", MathOp.Multiply, 2.0).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
 

public TimeMathOpTransform(@JsonProperty("columnName") String columnName, @JsonProperty("mathOp") MathOp mathOp,
                @JsonProperty("timeQuantity") long timeQuantity, @JsonProperty("timeUnit") TimeUnit timeUnit) {
    super(columnName);
    if (mathOp != MathOp.Add && mathOp != MathOp.Subtract && mathOp != MathOp.ScalarMin
                    && mathOp != MathOp.ScalarMax) {
        throw new IllegalArgumentException("Invalid MathOp: only Add/Subtract/ScalarMin/ScalarMax supported");
    }
    if ((mathOp == MathOp.ScalarMin || mathOp == MathOp.ScalarMax) && timeUnit != TimeUnit.MILLISECONDS) {
        throw new IllegalArgumentException(
                        "Only valid time unit for ScalarMin/Max is Milliseconds (i.e., timestamp format)");
    }

    this.mathOp = mathOp;
    this.timeQuantity = timeQuantity;
    this.timeUnit = timeUnit;
    this.asMilliseconds = TimeUnit.MILLISECONDS.convert(timeQuantity, timeUnit);
}
 

public TimeMathOpTransform(@JsonProperty("columnName") String columnName, @JsonProperty("mathOp") MathOp mathOp,
                @JsonProperty("timeQuantity") long timeQuantity, @JsonProperty("timeUnit") TimeUnit timeUnit) {
    super(columnName);
    if (mathOp != MathOp.Add && mathOp != MathOp.Subtract && mathOp != MathOp.ScalarMin
                    && mathOp != MathOp.ScalarMax) {
        throw new IllegalArgumentException("Invalid MathOp: only Add/Subtract/ScalarMin/ScalarMax supported");
    }
    if ((mathOp == MathOp.ScalarMin || mathOp == MathOp.ScalarMax) && timeUnit != TimeUnit.MILLISECONDS) {
        throw new IllegalArgumentException(
                        "Only valid time unit for ScalarMin/Max is Milliseconds (i.e., timestamp format)");
    }

    this.mathOp = mathOp;
    this.timeQuantity = timeQuantity;
    this.timeUnit = timeUnit;
    this.asMilliseconds = TimeUnit.MILLISECONDS.convert(timeQuantity, timeUnit);
}
 

@Test
public void testCustomTransform() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnDouble("secondCol").build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .transform(new CustomTransform("secondCol", 3.14159))
                    .doubleMathOp("secondCol", MathOp.Multiply, 2.0).filter(new CustomFilter(123))
                    .filter(new CustomCondition("someArg")).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
 

@Test
public void testTransformProcessAndSchema() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnNDArray("nd1a", new long[] {1, 10})
                    .addColumnNDArray("nd1b", new long[] {1, 10}).addColumnNDArray("nd2", new long[] {1, 100})
                    .addColumnNDArray("nd3", new long[] {-1, -1}).build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .ndArrayColumnsMathOpTransform("added", MathOp.Add, "nd1a", "nd1b")
                    .ndArrayMathFunctionTransform("nd2", MathFunction.SQRT)
                    .ndArrayScalarOpTransform("nd3", MathOp.Multiply, 2.0).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
 

public CoordinatesDistanceTransform(@JsonProperty("newColumnName") String newColumnName,
                @JsonProperty("firstColumn") String firstColumn, @JsonProperty("secondColumn") String secondColumn,
                @JsonProperty("stdevColumn") String stdevColumn, @JsonProperty("delimiter") String delimiter) {
    super(newColumnName, MathOp.Add /* dummy op */,
                    stdevColumn != null ? new String[] {firstColumn, secondColumn, stdevColumn}
                                    : new String[] {firstColumn, secondColumn});
    this.delimiter = delimiter;
}
 

@Test
public void testExecutionSimple() {
    Schema schema = new Schema.Builder().addColumnInteger("col0")
                    .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
                    .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<Writable>> inputData = new ArrayList<>();
    inputData.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));

    JavaRDD<List<Writable>> rdd = sc.parallelize(inputData);

    List<List<Writable>> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect());

    Collections.sort(out, new Comparator<List<Writable>>() {
        @Override
        public int compare(List<Writable> o1, List<Writable> o2) {
            return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt());
        }
    });

    List<List<Writable>> expected = new ArrayList<>();
    expected.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));

    assertEquals(expected, out);
}
 

@Test
public void testTransforms() {


    Transform[] transforms =
                    new Transform[] {new NDArrayColumnsMathOpTransform("newCol", MathOp.Divide, "in1", "in2"),
                                    new NDArrayMathFunctionTransform("inCol", MathFunction.SQRT),
                                    new NDArrayScalarOpTransform("inCol", MathOp.ScalarMax, 3.0)};

    for (Transform t : transforms) {
        String yaml = y.serialize(t);
        String json = j.serialize(t);

        //            System.out.println(yaml);
        //            System.out.println(json);
        //            System.out.println();

        Transform t2 = y.deserializeTransform(yaml);
        Transform t3 = j.deserializeTransform(json);
        assertEquals(t, t2);
        assertEquals(t, t3);
    }


    String tArrAsYaml = y.serialize(transforms);
    String tArrAsJson = j.serialize(transforms);
    String tListAsYaml = y.serializeTransformList(Arrays.asList(transforms));
    String tListAsJson = j.serializeTransformList(Arrays.asList(transforms));

    //        System.out.println("\n\n\n\n");
    //        System.out.println(tListAsYaml);

    List<Transform> lFromYaml = y.deserializeTransformList(tListAsYaml);
    List<Transform> lFromJson = j.deserializeTransformList(tListAsJson);

    assertEquals(Arrays.asList(transforms), y.deserializeTransformList(tArrAsYaml));
    assertEquals(Arrays.asList(transforms), j.deserializeTransformList(tArrAsJson));
    assertEquals(Arrays.asList(transforms), lFromYaml);
    assertEquals(Arrays.asList(transforms), lFromJson);
}
 

/**
 *
 * @param columnName Name of the column to perform the operation on
 * @param mathOp     Operation to perform
 * @param scalar     Scalar value for the operation
 */
public NDArrayScalarOpTransform(@JsonProperty("columnName") String columnName,
                @JsonProperty("mathOp") MathOp mathOp, @JsonProperty("scalar") double scalar) {
    super(columnName);
    this.mathOp = mathOp;
    this.scalar = scalar;
}
 

@Test
public void testExecutionSimple() {
    Schema schema = new Schema.Builder().addColumnInteger("col0")
            .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").
                    addColumnFloat("col3").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
            .doubleMathOp("col2", MathOp.Add, 10.0).floatMathOp("col3", MathOp.Add, 5f).build();

    List<List<Writable>> inputData = new ArrayList<>();
    inputData.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1), new FloatWritable(0.3f)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1), new FloatWritable(1.7f)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1), new FloatWritable(3.6f)));

    List<List<Writable>> rdd = (inputData);

    List<List<Writable>> out = new ArrayList<>(LocalTransformExecutor.execute(rdd, tp));

    Collections.sort(out, new Comparator<List<Writable>>() {
        @Override
        public int compare(List<Writable> o1, List<Writable> o2) {
            return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt());
        }
    });

    List<List<Writable>> expected = new ArrayList<>();
    expected.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1), new FloatWritable(5.3f)));
    expected.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1), new FloatWritable(6.7f)));
    expected.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1), new FloatWritable(8.6f)));

    assertEquals(expected, out);
}
 

public CoordinatesDistanceTransform(@JsonProperty("newColumnName") String newColumnName,
                @JsonProperty("firstColumn") String firstColumn, @JsonProperty("secondColumn") String secondColumn,
                @JsonProperty("stdevColumn") String stdevColumn, @JsonProperty("delimiter") String delimiter) {
    super(newColumnName, MathOp.Add /* dummy op */,
                    stdevColumn != null ? new String[] {firstColumn, secondColumn, stdevColumn}
                                    : new String[] {firstColumn, secondColumn});
    this.delimiter = delimiter;
}
 

@Test
public void testExecutionSimple() {
    Schema schema = new Schema.Builder().addColumnInteger("col0")
                    .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
                    .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<Writable>> inputData = new ArrayList<>();
    inputData.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));

    JavaRDD<List<Writable>> rdd = sc.parallelize(inputData);

    List<List<Writable>> out = new ArrayList<>(SparkTransformExecutor.execute(rdd, tp).collect());

    Collections.sort(out, new Comparator<List<Writable>>() {
        @Override
        public int compare(List<Writable> o1, List<Writable> o2) {
            return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt());
        }
    });

    List<List<Writable>> expected = new ArrayList<>();
    expected.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));

    assertEquals(expected, out);
}
 

@Test
public void testTransforms() {


    Transform[] transforms =
                    new Transform[] {new NDArrayColumnsMathOpTransform("newCol", MathOp.Divide, "in1", "in2"),
                                    new NDArrayMathFunctionTransform("inCol", MathFunction.SQRT),
                                    new NDArrayScalarOpTransform("inCol", MathOp.ScalarMax, 3.0)};

    for (Transform t : transforms) {
        String yaml = y.serialize(t);
        String json = j.serialize(t);

        //            System.out.println(yaml);
        //            System.out.println(json);
        //            System.out.println();

        Transform t2 = y.deserializeTransform(yaml);
        Transform t3 = j.deserializeTransform(json);
        assertEquals(t, t2);
        assertEquals(t, t3);
    }


    String tArrAsYaml = y.serialize(transforms);
    String tArrAsJson = j.serialize(transforms);
    String tListAsYaml = y.serializeTransformList(Arrays.asList(transforms));
    String tListAsJson = j.serializeTransformList(Arrays.asList(transforms));

    //        System.out.println("\n\n\n\n");
    //        System.out.println(tListAsYaml);

    List<Transform> lFromYaml = y.deserializeTransformList(tListAsYaml);
    List<Transform> lFromJson = j.deserializeTransformList(tListAsJson);

    assertEquals(Arrays.asList(transforms), y.deserializeTransformList(tArrAsYaml));
    assertEquals(Arrays.asList(transforms), j.deserializeTransformList(tArrAsJson));
    assertEquals(Arrays.asList(transforms), lFromYaml);
    assertEquals(Arrays.asList(transforms), lFromJson);
}
 

@Test
public void testNdArrayTransform() {
    Schema schema = new Schema.Builder()
            .addColumnNDArray("first", new long[]{1, 1})
            .build();

    TransformProcess transformProcess = new TransformProcess.Builder(schema)
            .ndArrayScalarOpTransform("first", MathOp.Add, 1.0)
            .build();

    TransformProcessStep config = TransformProcessStep.builder()
            .inputName("default")
            .inputColumnName("default", Collections.singletonList("first"))
            .transformProcess("default", transformProcess)
            .build();

    TransformProcessStepRunner step = new TransformProcessStepRunner(config);

    List<Writable> ret = new ArrayList<>();
    ret.add(new NDArrayWritable(Nd4j.scalar(1.0)));

    Record[] transform = step.transform(new Record[]{
            new org.datavec.api.records.impl.Record(ret, null)
    });

    assertEquals(1, transform.length);

    INDArray[] transformed = SchemaTypeUtils.toArrays(transform);
    assertEquals(Nd4j.scalar(2.0).reshape(1,1), transformed[0].reshape(1,1));
}
 

@Test
public void testExecutionSimple() {
    Schema schema = new Schema.Builder().addColumnInteger("col0")
            .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
            .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<Writable>> inputData = new ArrayList<>();
    inputData.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    inputData.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));

    List<List<Writable>> rdd = (inputData);

    List<List<Writable>> out = new ArrayList<>(LocalTransformExecutor.execute(rdd, tp));

    Collections.sort(out, new Comparator<List<Writable>>() {
        @Override
        public int compare(List<Writable> o1, List<Writable> o2) {
            return Integer.compare(o1.get(0).toInt(), o2.get(0).toInt());
        }
    });

    List<List<Writable>> expected = new ArrayList<>();
    expected.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    expected.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));

    assertEquals(expected, out);
}
 

/**
 *
 * @param columnName Name of the column to perform the operation on
 * @param mathOp     Operation to perform
 * @param scalar     Scalar value for the operation
 */
public NDArrayScalarOpTransform(@JsonProperty("columnName") String columnName,
                @JsonProperty("mathOp") MathOp mathOp, @JsonProperty("scalar") double scalar) {
    super(columnName);
    this.mathOp = mathOp;
    this.scalar = scalar;
}
 

public BaseColumnsMathOpTransform(String newColumnName, MathOp mathOp, String... columns) {
    if (columns == null || columns.length == 0)
        throw new IllegalArgumentException("Invalid input: cannot have null/0 columns");
    this.newColumnName = newColumnName;
    this.mathOp = mathOp;
    this.columns = columns;

    switch (mathOp) {
        case Add:
            if (columns.length < 2)
                throw new IllegalArgumentException(
                                "Need 2 or more columns for Add op. Got: " + Arrays.toString(columns));
            break;
        case Subtract:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Subtract op. Got: " + Arrays.toString(columns));
            break;
        case Multiply:
            if (columns.length < 2)
                throw new IllegalArgumentException(
                                "Need 2 or more columns for Multiply op. Got: " + Arrays.toString(columns));
            break;
        case Divide:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Divide op. Got: " + Arrays.toString(columns));
            break;
        case Modulus:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Modulus op. Got: " + Arrays.toString(columns));
            break;
        case ReverseSubtract:
        case ReverseDivide:
        case ScalarMin:
        case ScalarMax:
            throw new IllegalArgumentException(
                            "Invalid MathOp: cannot use " + mathOp + " with ...ColumnsMathOpTransform");
        default:
            throw new RuntimeException("Unknown MathOp: " + mathOp);
    }
}
 

@Test
public void testExecutionSequence() {

    Schema schema = new SequenceSchema.Builder().addColumnInteger("col0")
                    .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
                    .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<List<Writable>>> inputSequences = new ArrayList<>();
    List<List<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));
    List<List<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1)));
    seq2.add(Arrays.<Writable>asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1)));

    inputSequences.add(seq1);
    inputSequences.add(seq2);

    JavaRDD<List<List<Writable>>> rdd = sc.parallelize(inputSequences);

    List<List<List<Writable>>> out =
                    new ArrayList<>(SparkTransformExecutor.executeSequenceToSequence(rdd, tp).collect());

    Collections.sort(out, new Comparator<List<List<Writable>>>() {
        @Override
        public int compare(List<List<Writable>> o1, List<List<Writable>> o2) {
            return -Integer.compare(o1.size(), o2.size());
        }
    });

    List<List<List<Writable>>> expectedSequence = new ArrayList<>();
    List<List<Writable>> seq1e = new ArrayList<>();
    seq1e.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));
    List<List<Writable>> seq2e = new ArrayList<>();
    seq2e.add(Arrays.<Writable>asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1)));
    seq2e.add(Arrays.<Writable>asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1)));

    expectedSequence.add(seq1e);
    expectedSequence.add(seq2e);

    assertEquals(expectedSequence, out);
}
 

@Test
public void testNDArrayWritableBasic() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10}).addColumnString("col2")
                    .build();


    TransformProcess tp = new TransformProcess.Builder(s).ndArrayScalarOpTransform("col1", MathOp.Add, 100).build();

    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new Text("str0"));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0),
                    new NDArrayWritable(Nd4j.linspace(0, 9, 10).addi(100)), new Text("str0"));

    assertEquals(exp, out);

}
 

@Test
public void testExecutionSequence() {

    Schema schema = new SequenceSchema.Builder().addColumnInteger("col0")
            .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
            .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<List<Writable>>> inputSequences = new ArrayList<>();
    List<List<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));
    List<List<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1)));
    seq2.add(Arrays.<Writable>asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1)));

    inputSequences.add(seq1);
    inputSequences.add(seq2);

    List<List<List<Writable>>> rdd =  (inputSequences);

    List<List<List<Writable>>> out = LocalTransformExecutor.executeSequenceToSequence(rdd, tp);

    Collections.sort(out, new Comparator<List<List<Writable>>>() {
        @Override
        public int compare(List<List<Writable>> o1, List<List<Writable>> o2) {
            return -Integer.compare(o1.size(), o2.size());
        }
    });

    List<List<List<Writable>>> expectedSequence = new ArrayList<>();
    List<List<Writable>> seq1e = new ArrayList<>();
    seq1e.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));
    List<List<Writable>> seq2e = new ArrayList<>();
    seq2e.add(Arrays.<Writable>asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1)));
    seq2e.add(Arrays.<Writable>asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1)));

    expectedSequence.add(seq1e);
    expectedSequence.add(seq2e);

    assertEquals(expectedSequence, out);
}
 

@Test
public void regressionTestJson100a() throws Exception {
    //JSON saved in 1.0.0-alpha, before JSON format change

    File f = new ClassPathResource("datavec-api/regression_test/100a/transformprocess_regression_100a.json").getFile();
    String s = FileUtils.readFileToString(f);

    TransformProcess fromJson = TransformProcess.fromJson(s);



    Schema schema = new Schema.Builder().addColumnCategorical("Cat", "State1", "State2")
            .addColumnCategorical("Cat2", "State1", "State2").addColumnDouble("Dbl")
            .addColumnDouble("Dbl2", null, 100.0, true, false).addColumnInteger("Int")
            .addColumnInteger("Int2", 0, 10).addColumnLong("Long").addColumnLong("Long2", -100L, null)
            .addColumnString("Str").addColumnString("Str2", "someregexhere", 1, null)
            .addColumnString("Str3")
            .addColumnTime("TimeCol", DateTimeZone.UTC)
            .addColumnTime("TimeCol2", DateTimeZone.UTC, null, 1000L).build();

    Map<String, String> map = new HashMap<>();
    map.put("from", "to");
    map.put("anotherFrom", "anotherTo");

    TransformProcess expected =
            new TransformProcess.Builder(schema).categoricalToInteger("Cat").categoricalToOneHot("Cat2")
                    .appendStringColumnTransform("Str3", "ToAppend")
                    .integerToCategorical("Cat", Arrays.asList("State1", "State2"))
                    .stringToCategorical("Str", Arrays.asList("State1", "State2"))
                    .duplicateColumn("Str", "Str2a").removeColumns("Str2a")
                    .renameColumn("Str2", "Str2a").reorderColumns("Cat", "Dbl")
                    .conditionalCopyValueTransform("Dbl", "Dbl2",
                            new DoubleColumnCondition("Dbl", ConditionOp.Equal, 0.0))
                    .conditionalReplaceValueTransform("Dbl", new DoubleWritable(1.0),
                            new DoubleColumnCondition("Dbl", ConditionOp.Equal, 1.0))
                    .doubleColumnsMathOp("NewDouble", MathOp.Add, "Dbl", "Dbl2")
                    .doubleMathOp("Dbl", MathOp.Add, 1.0)
                    .integerColumnsMathOp("NewInt", MathOp.Subtract, "Int", "Int2")
                    .integerMathOp("Int", MathOp.Multiply, 2)
                    .transform(new ReplaceEmptyIntegerWithValueTransform("Int", 1))
                    .transform(new ReplaceInvalidWithIntegerTransform("Int", 1))
                    .longColumnsMathOp("Long", MathOp.Multiply, "Long", "Long2")
                    .longMathOp("Long", MathOp.ScalarMax, 0)
                    .transform(new MapAllStringsExceptListTransform("Str", "Other",
                            Arrays.asList("Ok", "SomeVal")))
                    .stringRemoveWhitespaceTransform("Str")
                    .transform(new ReplaceEmptyStringTransform("Str", "WasEmpty"))
                    .replaceStringTransform("Str", map)
                    .transform(new StringListToCategoricalSetTransform("Str",
                            Arrays.asList("StrA", "StrB"), Arrays.asList("StrA", "StrB"),
                            ","))
                    .stringMapTransform("Str2a", map)
                    .transform(new DeriveColumnsFromTimeTransform.Builder("TimeCol")
                            .addIntegerDerivedColumn("Hour", DateTimeFieldType.hourOfDay())
                            .addStringDerivedColumn("Date", "YYYY-MM-dd", DateTimeZone.UTC)
                            .build())
                    .stringToTimeTransform("Str2a", "YYYY-MM-dd hh:mm:ss", DateTimeZone.UTC)
                    .timeMathOp("TimeCol2", MathOp.Add, 1, TimeUnit.HOURS)

                    //Filters:
                    .filter(new FilterInvalidValues("Cat", "Str2a"))
                    .filter(new ConditionFilter(new NullWritableColumnCondition("Long")))

                    //Convert to/from sequence
                    .convertToSequence("Int", new NumericalColumnComparator("TimeCol2"))
                    .convertFromSequence()

                    //Sequence split
                    .convertToSequence("Int", new StringComparator("Str2a"))
                    .splitSequence(new SequenceSplitTimeSeparation("TimeCol2", 1, TimeUnit.HOURS))

                    //Reducers and reduce by window:
                    .reduce(new Reducer.Builder(ReduceOp.TakeFirst).keyColumns("TimeCol2")
                            .countColumns("Cat").sumColumns("Dbl").build())
                    .reduceSequenceByWindow(
                            new Reducer.Builder(ReduceOp.TakeFirst).countColumns("Cat2")
                                    .stdevColumns("Dbl2").build(),
                            new OverlappingTimeWindowFunction.Builder()
                                    .timeColumn("TimeCol2")
                                    .addWindowStartTimeColumn(true)
                                    .addWindowEndTimeColumn(true)
                                    .windowSize(1, TimeUnit.HOURS)
                                    .offset(5, TimeUnit.MINUTES)
                                    .windowSeparation(15, TimeUnit.MINUTES)
                                    .excludeEmptyWindows(true).build())

                    //Calculate sorted rank
                    .convertFromSequence()
                    .calculateSortedRank("rankColName", "TimeCol2", new LongWritableComparator())
                    .sequenceMovingWindowReduce("rankColName", 20, ReduceOp.Mean)
                    .addConstantColumn("someIntColumn", ColumnType.Integer, new IntWritable(0))
                    .integerToOneHot("someIntColumn", 0, 3)
                    .filter(new SequenceLengthCondition(ConditionOp.LessThan, 1))
                    .addConstantColumn("testColSeq", ColumnType.Integer, new DoubleWritable(0))
                    .offsetSequence(Collections.singletonList("testColSeq"), 1, SequenceOffsetTransform.OperationType.InPlace)
                    .addConstantColumn("someTextCol", ColumnType.String, new Text("some values"))
                    .build();


    assertEquals(expected, fromJson);
}
 

public IntegerMathOpTransform(@JsonProperty("columnName") String columnName, @JsonProperty("mathOp") MathOp mathOp,
                @JsonProperty("scalar") int scalar) {
    super(columnName);
    this.mathOp = mathOp;
    this.scalar = scalar;
}
 

public NDArrayColumnsMathOpTransform(@JsonProperty("newColumnName") String newColumnName,
                @JsonProperty("mathOp") MathOp mathOp, @JsonProperty("columns") String... columns) {
    super(newColumnName, mathOp, columns);
}
 

public LongColumnsMathOpTransform(@JsonProperty("newColumnName") String newColumnName,
                @JsonProperty("mathOp") MathOp mathOp, @JsonProperty("columns") String... columns) {
    super(newColumnName, mathOp, columns);
}
 

public LongMathOpTransform(@JsonProperty("columnName") String columnName, @JsonProperty("mathOp") MathOp mathOp,
                @JsonProperty("scalar") long scalar) {
    super(columnName);
    this.mathOp = mathOp;
    this.scalar = scalar;
}
 

public DoubleMathOpTransform(@JsonProperty("columnName") String columnName, @JsonProperty("mathOp") MathOp mathOp,
                @JsonProperty("scalar") double scalar) {
    super(columnName);
    this.mathOp = mathOp;
    this.scalar = scalar;
}
 

public DoubleColumnsMathOpTransform(String newColumnName, MathOp mathOp, String... columns) {
    super(newColumnName, mathOp, columns);
}
 

public DoubleColumnsMathOpTransform(@JsonProperty("newColumnName") String newColumnName,
                @JsonProperty("mathOp") MathOp mathOp, @JsonProperty("columns") List<String> columns) {
    this(newColumnName, mathOp, columns.toArray(new String[columns.size()]));
}
 

public BaseColumnsMathOpTransform(String newColumnName, MathOp mathOp, String... columns) {
    if (columns == null || columns.length == 0)
        throw new IllegalArgumentException("Invalid input: cannot have null/0 columns");
    this.newColumnName = newColumnName;
    this.mathOp = mathOp;
    this.columns = columns;

    switch (mathOp) {
        case Add:
            if (columns.length < 2)
                throw new IllegalArgumentException(
                                "Need 2 or more columns for Add op. Got: " + Arrays.toString(columns));
            break;
        case Subtract:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Subtract op. Got: " + Arrays.toString(columns));
            break;
        case Multiply:
            if (columns.length < 2)
                throw new IllegalArgumentException(
                                "Need 2 or more columns for Multiply op. Got: " + Arrays.toString(columns));
            break;
        case Divide:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Divide op. Got: " + Arrays.toString(columns));
            break;
        case Modulus:
            if (columns.length != 2)
                throw new IllegalArgumentException(
                                "Need exactly 2 columns for Modulus op. Got: " + Arrays.toString(columns));
            break;
        case ReverseSubtract:
        case ReverseDivide:
        case ScalarMin:
        case ScalarMax:
            throw new IllegalArgumentException(
                            "Invalid MathOp: cannot use " + mathOp + " with ...ColumnsMathOpTransform");
        default:
            throw new RuntimeException("Unknown MathOp: " + mathOp);
    }
}