Java Code Examples for java.io.StreamTokenizer#eolIsSignificant()

/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range
 */
public ArffLoader(Reader reader, Range range) {
  this.range = range;
  BufferedReader br = new BufferedReader(reader);

  //Init streamTokenizer
  streamTokenizer = new StreamTokenizer(br);
  streamTokenizer.resetSyntax();
  streamTokenizer.whitespaceChars(0, ' ');
  streamTokenizer.wordChars(' ' + 1, '\u00FF');
  streamTokenizer.whitespaceChars(',', ',');
  streamTokenizer.commentChar('%');
  streamTokenizer.quoteChar('"');
  streamTokenizer.quoteChar('\'');
  streamTokenizer.ordinaryChar('{');
  streamTokenizer.ordinaryChar('}');
  streamTokenizer.eolIsSignificant(true);

  this.instanceInformation = this.getHeader();

  if (range != null) { //is MultiLabel
    this.instanceInformation.setRangeOutputIndices(range);
  }

}
 

private void initStreamTokenizer(Reader reader) {
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);

    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();
    if (classAttribute < 0) {
        this.instanceInformation.setClassIndex(this.instanceInformation.numAttributes() - 1);
        //System.out.print(this.instanceInformation.classIndex());
    } else if (classAttribute > 0) {
        this.instanceInformation.setClassIndex(classAttribute - 1);
    }
}
 

/**
 * Initializes the stream tokenizer.
 * 
 * @param tokenizer the tokenizer to initialize
 */
private void initTokenizer(StreamTokenizer tokenizer) {
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars(0, (' ' - 1));
  tokenizer.wordChars(' ', '\u00FF');
  tokenizer.whitespaceChars(m_FieldSeparator.charAt(0),
      m_FieldSeparator.charAt(0));
  // tokenizer.commentChar('%');

  String[] parts = m_Enclosures.split(",");
  for (String e : parts) {
    if (e.length() > 1 || e.length() == 0) {
      throw new IllegalArgumentException(
          "Enclosures can only be single characters");
    }
    tokenizer.quoteChar(e.charAt(0));
  }

  tokenizer.eolIsSignificant(true);
}
 

/** Function to initialize the stream tokenizer.
 *
 * @param tokenizer The tokenizer.
 */
protected void initTokenizer( StreamTokenizer tokenizer )
{
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars( 0, ' ' );
  tokenizer.wordChars( ' '+1,'\u00FF' );
  tokenizer.whitespaceChars( ',',',' );
  tokenizer.quoteChar( '"' );
  tokenizer.quoteChar( '\''  );
  tokenizer.ordinaryChar( '=' );
  tokenizer.ordinaryChar( '{' );
  tokenizer.ordinaryChar( '}' );
  tokenizer.ordinaryChar( '[' );
  tokenizer.ordinaryChar( ']' );
  tokenizer.eolIsSignificant( true );
}
 

/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range the range
 */
public ArffLoader(Reader reader, Range range) {
    this.range = range;
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);
    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();

    if (range != null) { //is MultiLabel
        this.instanceInformation.setRangeOutputIndices(range);
    }

}
 

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
 

/**
 * Method to override to configure the tokenizer behaviour. It is called each time a tokenizer is created (for the
 * parsed file and all included files).
 */
protected void configureTokenizer(final StreamTokenizer tok) throws IOException {
	if (COMMENT_CHAR > 0) {
		tok.commentChar(COMMENT_CHAR);
	}
	tok.quoteChar(QUOTE_CHAR);
	tok.eolIsSignificant(eol_is_significant);
	tok.wordChars('_', '_');
	tok.parseNumbers();
}
 

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
 

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
 

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
 

public Object parse(String text) throws IOException {
  StringReader rdr = new StringReader(text);
  StreamTokenizer tokens = new StreamTokenizer(rdr);
  tokens.eolIsSignificant(false);
  tokens.quoteChar(QUOTE);
  tokens.wordChars('a', 'z');
  tokens.wordChars('A', 'Z');
  tokens.wordChars('0', '9');
  tokens.wordChars('_', '_');
  tokens.wordChars('-', '-');
  tokens.wordChars('$', '$');
  Object result = parseR(tokens);
  return result;
}
 

/**
 * Inspect specified board file and return its dimensions.
 *
 * @param filepath
 *            The path to the board file.
 * @return A {@link BoardDimensions} object containing the dimension.
 */
public static BoardDimensions getSize(final File filepath) {
    int boardx = 0;
    int boardy = 0;
    try {
        // make inpustream for board
        Reader r = new BufferedReader(new FileReader(filepath));
        // read board, looking for "size"
        StreamTokenizer st = new StreamTokenizer(r);
        st.eolIsSignificant(true);
        st.commentChar('#');
        st.quoteChar('"');
        st.wordChars('_', '_');
        while (st.nextToken() != StreamTokenizer.TT_EOF) {
            if ((st.ttype == StreamTokenizer.TT_WORD) && st.sval.equalsIgnoreCase("size")) {
                st.nextToken();
                boardx = (int) st.nval;
                st.nextToken();
                boardy = (int) st.nval;
                break;
            }
        }
        r.close();
    } catch (IOException ex) {
        return null;
    }
    return new BoardDimensions(boardx, boardy);
}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

public void test_basicStringTokenizerMethods() throws IOException {
    String str = "Testing 12345 \n alpha \r\n omega";
    String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run";
    StringReader aa = new StringReader(str);
    StringReader ba = new StringReader(strb);
    StreamTokenizer a = new StreamTokenizer(aa);
    StreamTokenizer b = new StreamTokenizer(ba);

    Assert.assertTrue(a.lineno() == 1);
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[Testing], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(a.toString().equals("Token[n=12345.0], line 1"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[alpha], line 2"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(a.toString().equals("Token[omega], line 3"));
    Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(a.toString().equals("Token[EOF], line 3"));

    b.commentChar('u');
    b.eolIsSignificant(true);
    b.lowerCaseMode(true);
    b.ordinaryChar('y');
    b.slashStarComments(true);

    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_NUMBER);
    Assert.assertTrue(b.nval == -3.8);
    Assert.assertTrue(b.toString().equals("Token[n=-3.8], line 1"));
    Assert.assertTrue(b.nextToken() == 39); // '
    Assert.assertTrue(b.toString().equals("Token[BLIND mice], line 1"));
    Assert.assertTrue(b.nextToken() == 10); // \n
    Assert.assertTrue(b.toString().equals("Token[EOL], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[see], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[the], line 2"));
    Assert.assertTrue(b.nextToken() == 121); // y
    Assert.assertTrue(b.toString().equals("Token['y'], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD);
    Assert.assertTrue(b.toString().equals("Token[r], line 2"));
    Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_EOF);
    Assert.assertTrue(b.toString().equals("Token[EOF], line 2"));
}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

/** Create a Chemical model by parsing an input stream */
XYZChemModel(InputStream is) throws Exception {
    this();
    StreamTokenizer st = new StreamTokenizer(
            new BufferedReader(new InputStreamReader(is, "UTF-8")));
    st.eolIsSignificant(true);
    st.commentChar('#');

    try {
        scan:
        while (true) {
            switch (st.nextToken()) {
                case StreamTokenizer.TT_EOF:
                    break scan;
                default:
                    break;
                case StreamTokenizer.TT_WORD:
                    String name = st.sval;
                    double x = 0,
                     y = 0,
                     z = 0;
                    if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                        x = st.nval;
                        if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                            y = st.nval;
                            if (st.nextToken() == StreamTokenizer.TT_NUMBER) {
                                z = st.nval;
                            }
                        }
                    }
                    addVert(name, (float) x, (float) y, (float) z);
                    while (st.ttype != StreamTokenizer.TT_EOL
                            && st.ttype != StreamTokenizer.TT_EOF) {
                        st.nextToken();
                    }

            }   // end Switch

        }  // end while

        is.close();

    } // end Try
    catch (IOException e) {
    }

    if (st.ttype != StreamTokenizer.TT_EOF) {
        throw new Exception(st.toString());
    }

}
 

/**
 * Builds a DataPoint from the given CSV line. This method should only be
 * used to parse a line that is expected to be made up of numeric data
 * only. Use {@link #readHeaderRow} to read a header row if one is expected.
 * @param line the input line of comma separated values to parse and use
 * to construct a new DataPoint.
 * @return a DataPoint object with values as specified by the given input
 * String.
 */
private DataPoint build( String line )
    throws IOException
{
    Observation dataPoint = new Observation( 0.0 );
    
    StreamTokenizer tokenizer
        = new StreamTokenizer( new StringReader( line ) );
    
    tokenizer.commentChar( '#' );
    tokenizer.eolIsSignificant( true );
    tokenizer.parseNumbers();
    
    int i = 0;
    int n = getNumberOfVariables();
    int lastToken = SEPARATOR;
    do
        {
            // Read next token
            tokenizer.nextToken();
            switch ( tokenizer.ttype )
                {
                case '\t':
                case ' ':
                    // Skip whitespace
                    continue;
                    
                case SEPARATOR:
                    // Check for two adjacent commas
                    if ( lastToken != SEPARATOR )
                        break;
                    
                    // Two adjacent commas. Assume 0.0 between them
                    tokenizer.nval = 0.0;
                    
                    // Fall through, and handle as a number
                    
                case StreamTokenizer.TT_NUMBER:
                    // Handle numbers appropriately as data
                    
                    // If this is the last value on the line, treat it
                    //  as the dependent variable value
                    if ( i == n )
                        dataPoint.setDependentValue(tokenizer.nval);
                    else
                        dataPoint.setIndependentValue(getVariableName(i),
                                                      tokenizer.nval);
                    
                    i++;
                    break;
                    
                case StreamTokenizer.TT_WORD:
                    throw new IOException( "Invalid input in CSV file. Number expected, found '"+tokenizer.sval+"'");
                    
                case StreamTokenizer.TT_EOL:
                case StreamTokenizer.TT_EOF:
                    break;
                    
                default:
                }
            
            lastToken = tokenizer.ttype;
        }
    while ( tokenizer.ttype != StreamTokenizer.TT_EOF );
    
    return dataPoint;
}
 

/**
 * Create new benchmark harness with given configuration and reporter.
 * Throws ConfigFormatException if there was an error parsing the config
 * file.
 * <p>
 * <b>Config file syntax:</b>
 * <p>
 * '#' marks the beginning of a comment.  Blank lines are ignored.  All
 * other lines should adhere to the following format:
 * <pre>
 *     &lt;weight&gt; &lt;name&gt; &lt;class&gt; [&lt;args&gt;]
 * </pre>
 * &lt;weight&gt; is a floating point value which is multiplied times the
 * benchmark's execution time to determine its weighted score.  The
 * total score of the benchmark suite is the sum of all weighted scores
 * of its benchmarks.
 * <p>
 * &lt;name&gt; is a name used to identify the benchmark on the benchmark
 * report.  If the name contains whitespace, the quote character '"' should
 * be used as a delimiter.
 * <p>
 * &lt;class&gt; is the full name (including the package) of the class
 * containing the benchmark implementation.  This class must implement
 * bench.Benchmark.
 * <p>
 * [&lt;args&gt;] is a variable-length list of runtime arguments to pass to
 * the benchmark.  Arguments containing whitespace should use the quote
 * character '"' as a delimiter.
 * <p>
 * <b>Example:</b>
 * <pre>
 *      3.5 "My benchmark" bench.serial.Test first second "third arg"
 * </pre>
 */
public Harness(InputStream in) throws IOException, ConfigFormatException {
    Vector bvec = new Vector();
    StreamTokenizer tokens = new StreamTokenizer(new InputStreamReader(in));

    tokens.resetSyntax();
    tokens.wordChars(0, 255);
    tokens.whitespaceChars(0, ' ');
    tokens.commentChar('#');
    tokens.quoteChar('"');
    tokens.eolIsSignificant(true);

    tokens.nextToken();
    while (tokens.ttype != StreamTokenizer.TT_EOF) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_WORD:
            case '"':                       // parse line
                bvec.add(parseBenchInfo(tokens));
                break;

            default:                        // ignore
                tokens.nextToken();
                break;
        }
    }
    binfo = (BenchInfo[]) bvec.toArray(new BenchInfo[bvec.size()]);
}