Java Code Examples for org.datavec.api.transform.schema.Schema#newSchema()

The following examples show how to use org.datavec.api.transform.schema.Schema#newSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TimeWindowFunction.java From DataVec with Apache License 2.0

6 votes

@Override
public Schema transform(Schema inputSchema) {
    if (!addWindowStartTimeColumn && !addWindowEndTimeColumn)
        return inputSchema;
    List<ColumnMetaData> newMeta = new ArrayList<>();

    newMeta.addAll(inputSchema.getColumnMetaData());

    if (addWindowStartTimeColumn) {
        newMeta.add(new TimeMetaData("windowStartTime"));
    }

    if (addWindowEndTimeColumn) {
        newMeta.add(new TimeMetaData("windowEndTime"));
    }

    return inputSchema.newSchema(newMeta);
}

Example 2

Source File: TimeWindowFunction.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public Schema transform(Schema inputSchema) {
    if (!addWindowStartTimeColumn && !addWindowEndTimeColumn)
        return inputSchema;
    List<ColumnMetaData> newMeta = new ArrayList<>();

    newMeta.addAll(inputSchema.getColumnMetaData());

    if (addWindowStartTimeColumn) {
        newMeta.add(new TimeMetaData("windowStartTime"));
    }

    if (addWindowEndTimeColumn) {
        newMeta.add(new TimeMetaData("windowEndTime"));
    }

    return inputSchema.newSchema(newMeta);
}

Example 3

Source File: RemoveAllColumnsExceptForTransform.java From deeplearning4j with Apache License 2.0

6 votes

@Override
public Schema transform(Schema schema) {
    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    Set<String> keepSet = new HashSet<>();
    Collections.addAll(keepSet, columnsToKeep);


    List<ColumnMetaData> newMeta = new ArrayList<>(columnsToKeep.length);

    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> metaIter = origMeta.iterator();

    while (namesIter.hasNext()) {
        String n = namesIter.next();
        ColumnMetaData t = metaIter.next();
        if (keepSet.contains(n)) {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 4

Source File: CategoricalToIntegerTransform.java From DataVec with Apache License 2.0

6 votes

@Override
public Schema transform(Schema schema) {
    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    int i = 0;
    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> typesIter = origMeta.iterator();

    List<ColumnMetaData> newMeta = new ArrayList<>(schema.numColumns());

    while (namesIter.hasNext()) {
        String s = namesIter.next();
        ColumnMetaData t = typesIter.next();

        if (i++ == columnIdx) {
            //Convert this to integer
            int nClasses = stateNames.size();
            newMeta.add(new IntegerMetaData(t.getName(), 0, nClasses - 1));
        } else {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 5

Source File: BaseColumnsMathOpTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {
    for (String name : columns) {
        if (!inputSchema.hasColumn(name))
            throw new IllegalStateException("Input schema does not have column with name \"" + name + "\"");
    }

    List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.getColumnMetaData());

    newMeta.add(derivedColumnMetaData(newColumnName, inputSchema));

    return inputSchema.newSchema(newMeta);
}

Example 6

Source File: FirstDigitTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {
    List<String> origNames = inputSchema.getColumnNames();
    List<ColumnMetaData> origMeta = inputSchema.getColumnMetaData();

    Preconditions.checkState(origNames.contains(inputColumn), "Input column with name \"%s\" not found in schema", inputColumn);
    Preconditions.checkState(inputColumn.equals(outputColumn) || !origNames.contains(outputColumn),
            "Output column with name \"%s\" already exists in schema (only allowable if input column == output column)", outputColumn);

    List<ColumnMetaData> outMeta = new ArrayList<>(origNames.size()+1);
    for( int i=0; i<origNames.size(); i++ ){
        String s = origNames.get(i);
        if(s.equals(inputColumn)){
            if(!outputColumn.equals(inputColumn)){
                outMeta.add(origMeta.get(i));
            }

            List<String> l = Collections.unmodifiableList(
                    mode == Mode.INCLUDE_OTHER_CATEGORY ?
                            Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", OTHER_CATEGORY) :
                            Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"));

            CategoricalMetaData cm = new CategoricalMetaData(outputColumn, l);

            outMeta.add(cm);
        } else {
            outMeta.add(origMeta.get(i));
        }
    }

    return inputSchema.newSchema(outMeta);
}

Example 7

Source File: CategoricalToOneHotTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema schema) {
    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    int i = 0;
    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> typesIter = origMeta.iterator();

    List<ColumnMetaData> newMeta = new ArrayList<>(schema.numColumns());

    while (namesIter.hasNext()) {
        String s = namesIter.next();
        ColumnMetaData t = typesIter.next();

        if (i++ == columnIdx) {
            //Convert this to one-hot:
            for (String stateName : stateNames) {
                String newName = s + "[" + stateName + "]";
                newMeta.add(new IntegerMetaData(newName, 0, 1));
            }
        } else {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 8

Source File: CalculateSortedRank.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {
    if (inputSchema instanceof SequenceSchema)
        throw new IllegalStateException("Calculating sorted rank on sequences: not yet supported");

    List<ColumnMetaData> origMeta = inputSchema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>(origMeta);

    newMeta.add(new LongMetaData(newColumnName, 0L, null));

    return inputSchema.newSchema(newMeta);
}

Example 9

Source File: CategoricalToOneHotTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema schema) {
    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    int i = 0;
    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> typesIter = origMeta.iterator();

    List<ColumnMetaData> newMeta = new ArrayList<>(schema.numColumns());

    while (namesIter.hasNext()) {
        String s = namesIter.next();
        ColumnMetaData t = typesIter.next();

        if (i++ == columnIdx) {
            //Convert this to one-hot:
            for (String stateName : stateNames) {
                String newName = s + "[" + stateName + "]";
                newMeta.add(new IntegerMetaData(newName, 0, 1));
            }
        } else {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 10

Source File: RemoveColumnsTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema schema) {
    int nToRemove = columnsToRemove.length;
    int newNumColumns = schema.numColumns() - nToRemove;
    if (newNumColumns <= 0)
        throw new IllegalStateException("Number of columns after executing operation is " + newNumColumns
                        + " (is <= 0). " + "origColumns = " + schema.getColumnNames() + ", toRemove = "
                        + Arrays.toString(columnsToRemove));

    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    Set<String> set = new HashSet<>();
    Collections.addAll(set, columnsToRemove);


    List<ColumnMetaData> newMeta = new ArrayList<>(newNumColumns);

    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> metaIter = origMeta.iterator();

    while (namesIter.hasNext()) {
        String n = namesIter.next();
        ColumnMetaData t = metaIter.next();
        if (!set.contains(n)) {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 11

Source File: IntegerToOneHotTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema schema) {
    List<String> origNames = schema.getColumnNames();
    List<ColumnMetaData> origMeta = schema.getColumnMetaData();

    int i = 0;
    Iterator<String> namesIter = origNames.iterator();
    Iterator<ColumnMetaData> typesIter = origMeta.iterator();

    List<ColumnMetaData> newMeta = new ArrayList<>(schema.numColumns());

    while (namesIter.hasNext()) {
        String s = namesIter.next();
        ColumnMetaData t = typesIter.next();

        if (i++ == columnIdx) {
            //Convert this to one-hot:
            for (int x = minValue; x <= maxValue; x++) {
                String newName = s + "[" + x + "]";
                newMeta.add(new IntegerMetaData(newName, 0, 1));
            }
        } else {
            newMeta.add(t);
        }
    }

    return schema.newSchema(newMeta);
}

Example 12

Source File: StringReducer.java From deeplearning4j with Apache License 2.0

5 votes

/**
 * Get the output schema, given the input schema
 */
@Override
public Schema transform(Schema schema) {
    int nCols = schema.numColumns();
    List<ColumnMetaData> meta = schema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>(nCols);
    newMeta.addAll(meta);
    newMeta.add(new StringMetaData(outputColumnName));
    return schema.newSchema(newMeta);
}

Example 13

Source File: StringListToCategoricalSetTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {

    int colIdx = inputSchema.getIndexOfColumn(columnName);

    List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>(oldMeta.size() + newColumnNames.size() - 1);
    List<String> oldNames = inputSchema.getColumnNames();

    Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
    Iterator<String> namesIter = oldNames.iterator();

    int i = 0;
    while (typesIter.hasNext()) {
        ColumnMetaData t = typesIter.next();
        String name = namesIter.next();
        if (i++ == colIdx) {
            //Replace String column with a set of binary/categorical columns
            if (t.getColumnType() != ColumnType.String)
                throw new IllegalStateException("Cannot convert non-string type");

            for (int j = 0; j < newColumnNames.size(); j++) {
                ColumnMetaData meta = new CategoricalMetaData(newColumnNames.get(j), "true", "false");
                newMeta.add(meta);
            }
        } else {
            newMeta.add(t);
        }
    }

    return inputSchema.newSchema(newMeta);

}

Example 14

Source File: StringListToCountsNDArrayTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {

    int colIdx = inputSchema.getIndexOfColumn(columnName);

    List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>();
    List<String> oldNames = inputSchema.getColumnNames();

    Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
    Iterator<String> namesIter = oldNames.iterator();

    int i = 0;
    while (typesIter.hasNext()) {
        ColumnMetaData t = typesIter.next();
        String name = namesIter.next();
        if (i++ == colIdx) {
            //Replace String column with a set of binary/integer columns
            if (t.getColumnType() != ColumnType.String)
                throw new IllegalStateException("Cannot convert non-string type");

            ColumnMetaData meta = new NDArrayMetaData(newColumnName, new long[] {vocabulary.size()});
            newMeta.add(meta);
        } else {
            newMeta.add(t);
        }
    }

    return inputSchema.newSchema(newMeta);

}

Example 15

Source File: AddConstantColumnTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {
    List<ColumnMetaData> outMeta = new ArrayList<>();
    outMeta.addAll(inputSchema.getColumnMetaData());

    ColumnMetaData newColMeta = newColumnType.newColumnMetaData(newColumnName);
    outMeta.add(newColMeta);
    return inputSchema.newSchema(outMeta);
}

Example 16

Source File: StringListToCategoricalSetTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {

    int colIdx = inputSchema.getIndexOfColumn(columnName);

    List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>(oldMeta.size() + newColumnNames.size() - 1);
    List<String> oldNames = inputSchema.getColumnNames();

    Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
    Iterator<String> namesIter = oldNames.iterator();

    int i = 0;
    while (typesIter.hasNext()) {
        ColumnMetaData t = typesIter.next();
        String name = namesIter.next();
        if (i++ == colIdx) {
            //Replace String column with a set of binary/categorical columns
            if (t.getColumnType() != ColumnType.String)
                throw new IllegalStateException("Cannot convert non-string type");

            for (int j = 0; j < newColumnNames.size(); j++) {
                ColumnMetaData meta = new CategoricalMetaData(newColumnNames.get(j), "true", "false");
                newMeta.add(meta);
            }
        } else {
            newMeta.add(t);
        }
    }

    return inputSchema.newSchema(newMeta);

}

Example 17

Source File: StringListToCountsNDArrayTransform.java From DataVec with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {

    int colIdx = inputSchema.getIndexOfColumn(columnName);

    List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
    List<ColumnMetaData> newMeta = new ArrayList<>();
    List<String> oldNames = inputSchema.getColumnNames();

    Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
    Iterator<String> namesIter = oldNames.iterator();

    int i = 0;
    while (typesIter.hasNext()) {
        ColumnMetaData t = typesIter.next();
        String name = namesIter.next();
        if (i++ == colIdx) {
            //Replace String column with a set of binary/integer columns
            if (t.getColumnType() != ColumnType.String)
                throw new IllegalStateException("Cannot convert non-string type");

            ColumnMetaData meta = new NDArrayMetaData(newColumnName, new long[] {vocabulary.size()});
            newMeta.add(meta);
        } else {
            newMeta.add(t);
        }
    }

    return inputSchema.newSchema(newMeta);

}

Example 18

Source File: BaseColumnsMathOpTransform.java From deeplearning4j with Apache License 2.0

5 votes

@Override
public Schema transform(Schema inputSchema) {
    for (String name : columns) {
        if (!inputSchema.hasColumn(name))
            throw new IllegalStateException("Input schema does not have column with name \"" + name + "\"");
    }

    List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.getColumnMetaData());

    newMeta.add(derivedColumnMetaData(newColumnName, inputSchema));

    return inputSchema.newSchema(newMeta);
}

Example 19

Source File: NDArrayDistanceTransform.java From deeplearning4j with Apache License 2.0

4 votes

@Override
public Schema transform(Schema inputSchema) {
    List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.getColumnMetaData());
    newMeta.add(new DoubleMetaData(newColumnName));
    return inputSchema.newSchema(newMeta);
}

Example 20

Source File: NDArrayDistanceTransform.java From DataVec with Apache License 2.0

4 votes

@Override
public Schema transform(Schema inputSchema) {
    List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.getColumnMetaData());
    newMeta.add(new DoubleMetaData(newColumnName));
    return inputSchema.newSchema(newMeta);
}