Java Code Examples for org.neuroph.core.data.DataSet#shuffle()

public void run() throws InterruptedException, ExecutionException {
    System.out.println("Creating training set...");
    // get path to training set
    String dataSetFile = "data_sets/iris_data_normalised.txt";
    int inputsCount = 4;
    int outputsCount = 3;

    // create training set from file
    DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, ",");
 //   dataSet.setColumnNames(new String[]{"sepal.length", "sepal.width", "petal.length", "petal.width",  "setosa", "versicolor", "virginica"});
    dataSet.setColumnNames(new String[]{"setosa", "versicolor", "virginica"});
    dataSet.shuffle();

    System.out.println("Creating neural network...");
    MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(TransferFunctionType.TANH, inputsCount, 5, outputsCount);

    String[] classLabels = new String[]{"setosa", "versicolor", "virginica"};
    neuralNet.setOutputLabels(classLabels);

    KFoldCrossValidation crossVal = new KFoldCrossValidation(neuralNet, dataSet, 5);
    EvaluationResult totalResult= crossVal.run();
    List<FoldResult> cflist= crossVal.getResultsByFolds();

}
 

/**
 * Train the specified MLP network using the specified training data, store metrics in the
 * specified metrics object.
 * 
 * @param trainingData
 *          The data used to train the network.
 * @param network
 *          The MLP network to be trained.
 * @param metrics
 *          The {@link NetworkMetrics} object where metrics info is stored.
 */
private void trainNetwork(DataSet trainingData, MultiLayerPerceptron network) {
  //
  // Shuffle the training data. Adds an element of randomness to the data.
  trainingData.shuffle();
  //
  // Now learn, you!
  network.learn(trainingData);
  //
  // Learning complete. Set metrics.
  NetworkMetrics metrics = networkMetricsCache.get(network);
  metrics.setIterationLearnTime(System.currentTimeMillis() - metrics.getLearnStartTime());
  metrics.setTotalLearnTime(metrics.getTotalLearnTime() + metrics.getIterationLearnTime());
  metrics.setNumberOfAsymmetricWinsThisIteration(0);
  metrics.setNumberOfSymmetricWinsThisIteration(0);
  metrics.setNumberOfGamesThisIteration(0);
}
 

@Override
public DataSet[] sample(DataSet dataSet) {
    // if object was initializes by specifying numParts calculate subSetSizes so all subsets are equally sized
    if (subSetSizes == null) {
        final double singleSubSetSize = 1.0d / numSubSets;
        subSetSizes = new double[numSubSets];
        for (int i = 0; i < numSubSets; i++) {
            subSetSizes[i] = singleSubSetSize;
        }
    }

    // create list of data sets to return
    List<DataSet> subSets = new ArrayList<>();

    // shuffle dataset in order to randomize rows that will be used to fill subsets
    dataSet.shuffle();

    int idxCounter = 0; // index of main data set
    for (int subSetIdx = 0; subSetIdx < numSubSets; subSetIdx++) {
        // create new subset
        DataSet newSubSet = new DataSet(dataSet.getInputSize(), dataSet.getOutputSize());
        // cop column names if there are any
        newSubSet.setColumnNames(dataSet.getColumnNames());

        // fill subset with rows
        long subSetSize = Math.round(subSetSizes[subSetIdx] * dataSet.size()); // calculate size of the current subset
        for (int i = 0; i < subSetSize; i++) {
            if (idxCounter >= dataSet.size()) {
                break;
            }
            newSubSet.add(dataSet.getRowAt(idxCounter));
            idxCounter++;
        }

        // add current subset to list that will be returned
        subSets.add(newSubSet);
    }

    return subSets.toArray(new DataSet[numSubSets]);
}
 

public void createTrainingAndTestSet() {
    //Creating data set from file
    DataSet dataSet = createDataSet();
    dataSet.shuffle();

    //Splitting main data set to training set (75%) and test set (25%)
    DataSet[] trainingAndTestSet = dataSet.createTrainingAndTestSubsets(75, 25);

    //Saving training set to file
    DataSet trainingSet = trainingAndTestSet[0];
    System.out.println("Saving training set to file...");
    trainingSet.save(config.getTrainingFileName());

    System.out.println("Training set successfully saved!");

    //Normalizing test set
    DataSet testSet = trainingAndTestSet[1];
    System.out.println("Normalizing test set...");

    Normalizer nor = new MaxNormalizer(trainingSet);
    nor.normalize(testSet);

    System.out.println("Saving normalized test set to file...");
    testSet.shuffle();
    testSet.save(config.getTestFileName());
    System.out.println("Normalized test set successfully saved!");
    System.out.println("Training set size: " + trainingSet.getRows().size() + " rows. ");
    System.out.println("Test set size: " + testSet.getRows().size() + " rows. ");
    System.out.println("-----------------------------------------------");

    double percentTraining = (double) trainingSet.getRows().size() * 100.0 / (double) dataSet.getRows().size();
    double percentTest = (double) testSet.getRows().size() * 100.0 / (double) dataSet.getRows().size();
    System.out.println("Training set takes " + formatDecimalNumber(percentTraining) + "% of main data set. ");
    System.out.println("Test set takes " + formatDecimalNumber(percentTest) + "% of main data set. ");

}
 

public void normalizeBalancedTrainingSet() {
    //Normalizing balanced training set with MaxNormalizer
    DataSet dataSet = DataSet.load(config.getBalancedFileName());
    Normalizer normalizer = new MaxNormalizer(dataSet);
    normalizer.normalize(dataSet);

    System.out.println("Saving normalized training data set to file... ");
    dataSet.shuffle();
    dataSet.save(config.getNormalizedBalancedFileName());
    System.out.println("Normalized training data set successfully saved!");
}
 

public void run() throws InterruptedException, ExecutionException {
    System.out.println("Creating training set...");
    // get path to training set
    String dataSetFile = "data_sets/wine.txt";
    int inputsCount = 11;
    int outputsCount = 10;

    // create training set from file
    DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, "\t", true);
    Normalizer norm = new MaxNormalizer(dataSet);
    norm.normalize(dataSet);
    dataSet.shuffle();

    System.out.println("Creating neural network...");
    MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 20, 15, outputsCount);

    neuralNet.setLearningRule(new MomentumBackpropagation());
    MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();

    // set learning rate and max error
    learningRule.setLearningRate(0.1);
    learningRule.setMaxIterations(10);

    String classLabels[] = new String[]{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"};
    neuralNet.setOutputLabels(classLabels);
    KFoldCrossValidation crossVal = new KFoldCrossValidation(neuralNet, dataSet, 10);
    EvaluationResult totalResult= crossVal.run();
    List<FoldResult> cflist= crossVal.getResultsByFolds();
}
 

public void run() throws InterruptedException, ExecutionException {
    System.out.println("Creating training set...");
    // get path to training set
    String dataSetFile = "data_sets/seeds.txt";
    int inputsCount = 7;
    int outputsCount = 3;

    // create training set from file
    DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, "\t");
    dataSet.shuffle();

    System.out.println("Creating neural network...");
    MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 15, 2, outputsCount);

    neuralNet.setLearningRule(new MomentumBackpropagation());
    MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();

    // set learning rate and max error
    learningRule.setLearningRate(0.1);
    learningRule.setMaxError(0.01);
    learningRule.setMaxIterations(1000);

    String[] classLabels = new String[]{"Cama", "Rosa", "Canadian"};
    neuralNet.setOutputLabels(classLabels);
    KFoldCrossValidation crossVal = new KFoldCrossValidation(neuralNet, dataSet, 10);
    EvaluationResult totalResult= crossVal.run();
     List<FoldResult> cflist= crossVal.getResultsByFolds();
}
 

/**
 * Loads training data for the specified years. The data is assumed to be at a location
 * specified by the {@link NetworkProperties} object (or <code>network.properties</code> file
 * if the default location is to be overridden).
 * 
 * @param yearsForTrainingData
 *          The years for which training data is to be loaded. Each element
 *          should contain a separate year.
 * 
 * @return Neuroph {@link DataSet} object containing all of the training data to be used.
 */
protected DataSet loadTrainingData(Integer[] yearsForTrainingData) {
  DataSet ret = new DataSet(NetworkProperties.getNumberOfInputs(), NetworkProperties.getNumberOfOutputs());
  List<DataSet> dataSets = new ArrayList<>();
  //
  // Build out the expected file name based on the constants
  /// and the years in the parameter
  for (Integer year : yearsForTrainingData) {
    String filename = NetworkUtils.computeTrainingDataFileName(year);
    // NetworkProperties.getBaseDirectory() + File.separator + trainingDataDirectory
    // + File.separator +
    // NetworkProperties.getTrainingDataFileBase() + "-" + year + NetworkProperties.getTrainingDataFileExtension();
    log.info("Loading training data from file: '" + filename + "'...");
    DataSet loadedDataSet = DataSet.load(filename);
    log.info("Training data loaded: " + loadedDataSet.getRows().size() + " rows.");
    dataSets.add(loadedDataSet);
  }
  //
  // Now combine all of the data sets into one
  for (DataSet dataSet : dataSets) {
    for (DataSetRow row : dataSet.getRows()) {
      ret.addRow(row);
    }
  }
  log.info("Combined " + dataSets.size() + " data sets, consisting of a total of " + ret.size() + " rows.");
  //
  log.info("Shuffling training data...");
  ret.shuffle();
  return ret;
}
 

private static DataSet loadDataSet() {
    DataSet irisDataSet = DataSet.createFromFile(inputFileName, 4, 3, ",", false);
    irisDataSet.shuffle();
    return irisDataSet;
}
 

public void run() {

        System.out.println("Creating training and test set from file...");
        String dataSetFile = "data_sets/german credit data.txt";
        int inputsCount = 24;
        int outputsCount = 2;

        //Create data set from file
        DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, " ");
        dataSet.shuffle();

        //Normalizing data set
        Normalizer normalizer = new MaxNormalizer(dataSet);
        normalizer.normalize(dataSet);

        //Creatinig training set (70%) and test set (30%)
        DataSet[] trainingAndTestSet = dataSet.createTrainingAndTestSubsets(70, 30);
        DataSet trainingSet = trainingAndTestSet[0];
        DataSet testSet = trainingAndTestSet[1];

        System.out.println("Creating neural network...");
        //Create MultiLayerPerceptron neural network
        MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 12, 6, outputsCount);

        //attach listener to learning rule
        MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();
        learningRule.addListener(this);

        learningRule.setLearningRate(0.01);
        learningRule.setMaxError(0.001);
        learningRule.setMaxIterations(10000);

        System.out.println("Training network...");
        //train the network with training set
        neuralNet.learn(trainingSet);

        System.out.println("Testing network...\n\n");
        testNeuralNetwork(neuralNet, testSet);

        System.out.println("Done.");

        System.out.println("**************************************************");

    }
 

public void run() {

        System.out.println("Creating training and test set from file...");
        String dataSetFile = "data_sets/ionosphere.csv";
        int inputsCount = 34;
        int outputsCount = 1;

        //Create data set from file
        DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, ",");
        dataSet.shuffle();

        //Normalizing data set
        Normalizer normalizer = new MaxNormalizer(dataSet);
        normalizer.normalize(dataSet);

        //Creatinig training set (70%) and test set (30%)
        DataSet[] trainingAndTestSet = dataSet.createTrainingAndTestSubsets(70, 30);
        DataSet trainingSet = trainingAndTestSet[0];
        DataSet testSet = trainingAndTestSet[1];

        // ovde ubaci u petlju sa hidden neuronima i learning rates

        System.out.println("Creating neural network...");
        //Create MultiLayerPerceptron neural network
        MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 10, 8, outputsCount);

        //attach listener to learning rule
        MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();
        learningRule.addListener(this);

        learningRule.setLearningRate(0.4);
        learningRule.setMaxError(0.01);
        learningRule.setMaxIterations(10000);

        System.out.println("Training network...");
        //train the network with training set
        neuralNet.learn(trainingSet);

//        System.out.println("Testing network...\n\n");
//        testNeuralNetwork(neuralNet, testSet);

        System.out.println("Done.");
        System.out.println("**************************************************");
//        }
    }
 

public void run() {

        System.out.println("Creating training and test set from file...");
        String dataSetFile = "data_sets/ionosphere.csv";
        int inputsCount = 34;
        int outputsCount = 1;

        //Create data set from file
        DataSet dataSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, ",");
        dataSet.shuffle();

        //Normalizing data set
        Normalizer normalizer = new MaxNormalizer(dataSet);
        normalizer.normalize(dataSet);

        //Creatinig training set (70%) and test set (30%)
        DataSet[] trainingAndTestSet = dataSet.createTrainingAndTestSubsets(70, 30);
        DataSet trainingSet = trainingAndTestSet[0];
        DataSet testSet = trainingAndTestSet[1];
//        for (int i = 0; i < 21; i++) {
        System.out.println("Creating neural network...");
        //Create MultiLayerPerceptron neural network
        MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 16, 8, outputsCount);
//            System.out.println("HIDDEN COUNT: " + i);
        //attach listener to learning rule
        MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();
        learningRule.addListener(this);

        learningRule.setLearningRate(0.2);
        learningRule.setMaxError(0.01);
        learningRule.setMaxIterations(10000);

        System.out.println("Training network...");
        //train the network with training set
        neuralNet.learn(trainingSet);

        System.out.println("Testing network...\n\n");
        testNeuralNetwork(neuralNet, testSet);

        System.out.println("Done.");
        System.out.println("**************************************************");
//        }
    }
 

public void run() {

        System.out.println("Creating training and test set from file...");
        String dataSetFile = "data_sets/segment challenge.txt";
        String testSetFileName = "data_sets/segment test.txt";
        int inputsCount = 19;
        int outputsCount = 7;

        //Create training data set from file
        DataSet trainingSet = DataSet.createFromFile(dataSetFile, inputsCount, outputsCount, ",");
        System.out.println("Training set size: " + trainingSet.getRows().size());
        trainingSet.shuffle();

        //Normalizing training data set
        Normalizer normalizer = new MaxNormalizer(trainingSet);
        normalizer.normalize(trainingSet);

        //Create test data set from file
        DataSet testSet = DataSet.createFromFile(testSetFileName, inputsCount, outputsCount, ",");
        System.out.println("Test set size: " + testSet.getRows().size());
        System.out.println("--------------------------------------------------");
        testSet.shuffle();

        //Normalizing training data set
        normalizer.normalize(testSet);

        System.out.println("Creating neural network...");
        //Create MultiLayerPerceptron neural network
        MultiLayerPerceptron neuralNet = new MultiLayerPerceptron(inputsCount, 17, 10, outputsCount);
        //attach listener to learning rule
        MomentumBackpropagation learningRule = (MomentumBackpropagation) neuralNet.getLearningRule();
        learningRule.addListener((event) -> {
            BackPropagation bp = (BackPropagation) event.getSource();
            if (event.getEventType().equals(LearningEvent.Type.LEARNING_STOPPED)) {
                double error = bp.getTotalNetworkError();
                System.out.println("Training completed in " + bp.getCurrentIteration() + " iterations, ");
                System.out.println("With total error: " + formatDecimalNumber(error));
            } else {
                System.out.println("Iteration: " + bp.getCurrentIteration() + " | Network error: " + bp.getTotalNetworkError());
            }
        });

        learningRule.setLearningRate(0.01);
        learningRule.setMaxError(0.001);
        learningRule.setMaxIterations(12000);

        System.out.println("Training network...");
        //train the network with training set
        neuralNet.learn(trainingSet);

        System.out.println("Testing network...\n\n");
        testNeuralNetwork(neuralNet, testSet);

        System.out.println("Done.");
        System.out.println("**************************************************");
//        }
    }