java.io.StreamTokenizer Java Examples

protected Object parseR(StreamTokenizer tokens) throws IOException {
  int token = tokens.nextToken();
  switch (token) {
    case TT_EOF:
      return null;
    case TT_WORD:
      return parseAtomic(tokens);
    case LBRACE:
      return parseMap(tokens);
    case LBRACKET:
      return parseArray(tokens);
    case QUOTE:
      return parseAtomic(tokens);
    default:
      throw new IOException("Unexpected token:" + (char) token);
  }
}
 

float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

private String getUnquotedString(String str) {
  if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;

  new StreamTokenizer(new StringReader(str));
  StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
  String result;

  try {
    parser.nextToken();
    if (parser.ttype == '"') {
      result = parser.sval;
    } else {
      result = "ERROR!";
    }
  } catch (IOException e) {
    result = null;

    LoggerHelper.info("Encountered an IOException while parsing the input string", e);
  }

  return result;
}
 

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 

String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 

/**
 * Read an int or codepoint - codepoint is given as a string
 */
private int getIntOrCP() throws IOException
{
    int val = tkn.nextToken();
    if (val == StreamTokenizer.TT_NUMBER)
    {
        return new Double(tkn.nval).intValue();
    }
    else if (val == StreamTokenizer.TT_WORD)
    {
        return decodeCP(tkn.sval);
    }
    else
    {
        fail("Expecting number, got " + tkn.sval + " on line "
                           + tkn.lineno());
        System.exit(1);
    }
    return 0;
}
 

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

/**
 * Read a word, a symbol or EOF, or generate a parse error. If this is EOF, the string "EOF" is returned.
 */
protected String getWordOrSymbolOrEof() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_NUMBER || tok == QUOTE_CHAR) {
		parseError("expecting a word or symbol, " + gotWhat(tok));
	}

	if (tok == StreamTokenizer.TT_WORD) {
		return st.sval;
	} else if (tok == StreamTokenizer.TT_EOF) {
		return "EOF";
	} else {
		return Character.toString((char) tok);
	}
}
 

String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 

private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

/**
 * Reads instance. It detects if it is dense or sparse.
 *
 * @return the instance
 */
public Instance readInstance() {
    while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
        try {
            streamTokenizer.nextToken();
        } catch (IOException ex) {
            Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (streamTokenizer.ttype == '{') {
        return readInstanceSparse();
        // return readDenseInstanceSparse();
    } else {
        return readInstanceDense();
    }

}
 

/**
 * Print failure message and skip to the next test
	 *
 * @exception 	IOException 	error reading file
 */
private void fail(String msg) throws IOException
{
	System.out.println("FAILED - " + msg + " in line " + tkn.lineno());
	// skip remainder of the test look for endtest or end of file
	int val = tkn.nextToken();
	while (val != StreamTokenizer.TT_EOF)
	{
		if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest"))
			break;

		val = tkn.nextToken();
	}
	failed = true;
	// get ready for next test
	reset();
	// print out stack trace so we know where the failure occurred
	Exception e = new Exception();
	e.printStackTrace();
}
 

/**
 * Push a tokenizer created from a file name on the file stack and make it current.
 *
 * @param file
 *            Name of the file used as source for the tokenizer.
 */
protected void pushTokenizer(final String file) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	try {
		reader = createReaderFrom(file);
		tok = createTokenizer(reader);

		cur = new CurrentFile(file, tok, reader);
	} catch (final FileNotFoundException e) {
		throw new IOException("cannot read file '" + file + "', not found: " + e.getMessage());
	}

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = file;
}
 

/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range the range
 */
public ArffLoader(Reader reader, Range range) {
    this.range = range;
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);
    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();

    if (range != null) { //is MultiLabel
        this.instanceInformation.setRangeOutputIndices(range);
    }

}
 

/**
 * Processes the test commands in the stream.
 *
 * @param cmdStream command stream (see {@link #commandSequence})
 * @throws IOException if accessing a file or the socket fails, or if
 *      Derby detects an unexpected protocol error
 */
private void processCommands(Reader cmdStream)
        throws  IOException {
    StreamTokenizer tkn = new StreamTokenizer(cmdStream);
    boolean endSignalled = false;
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF) {
        assertFalse("End signalled, data to process left: " + tkn.sval,
                endSignalled);
        switch(val) {
            case StreamTokenizer.TT_NUMBER:
                break;
            case StreamTokenizer.TT_WORD:
                endSignalled = processCommand(tkn);
                break;
            case StreamTokenizer.TT_EOL:
                break;
        }
    }
}
 

Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

/** Function to initialize the stream tokenizer.
 *
 * @param tokenizer The tokenizer.
 */
protected void initTokenizer( StreamTokenizer tokenizer )
{
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars( 0, ' ' );
  tokenizer.wordChars( ' '+1,'\u00FF' );
  tokenizer.whitespaceChars( ',',',' );
  tokenizer.quoteChar( '"' );
  tokenizer.quoteChar( '\''  );
  tokenizer.ordinaryChar( '=' );
  tokenizer.ordinaryChar( '{' );
  tokenizer.ordinaryChar( '}' );
  tokenizer.ordinaryChar( '[' );
  tokenizer.ordinaryChar( ']' );
  tokenizer.eolIsSignificant( true );
}
 

/**
 * Push a tokenizer created from a stream on the file stack and make it current.
 *
 * @param stream
 *            The stream used as source for the tokenizer.
 * @param name
 *            The name of the input stream.
 */
protected void pushTokenizer(final InputStream stream, final String name) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	reader = createReaderFrom(stream);
	tok = createTokenizer(reader);
	cur = new CurrentFile(name, tok, reader);

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = name;
}
 

public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
  try (StringReader reader = new StringReader(wkt)) {
    // setup the tokenizer; configured to read words w/o numbers
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars('a', 'z');
    tokenizer.wordChars('A', 'Z');
    tokenizer.wordChars(128 + 32, 255);
    tokenizer.wordChars('0', '9');
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('+', '+');
    tokenizer.wordChars('.', '.');
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    Object geometry = parseGeometry(tokenizer, shapeType);
    checkEOF(tokenizer);
    return geometry;
  }
}
 

protected Object parseArray(StreamTokenizer tokens) throws IOException {
  assert (tokens.ttype == LBRACKET);
  List<Object> array = new ArrayList<>();
  loop: for (;;) {
    int token = tokens.nextToken();
    switch (token) {
      case TT_EOL:
        break; // ignore
      case TT_EOF:
        throw new IOException("Unexpected eof");
      case RBRACKET:
        break loop;
      default:
        tokens.pushBack();
        Object o = parseR(tokens);
        tokens.nextToken();
        if (tokens.ttype == TT_EOF)
          break;
        else if (tokens.ttype == RBRACKET)
          tokens.pushBack();
        else if (tokens.ttype != COMMA)
          throw new IOException("Missing comma in list");
        array.add(o);
    }
  }
  return array;
}
 

public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    if (dataClass == null)
    {
        throw new ParseException("no class name found before line "+st.lineno(), st.lineno());
    }
    MithraDataObject currentData;
    currentData = this.getParser().getCurrentParsedData().createAndAddDataObject(st.lineno());

    // parse the data
    int currentAttribute = 0;
    int token = st.ttype;

    boolean wantData = true;
    while(token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF)
    {
        if (wantData)
        {
            this.getParser().getCurrentParsedData().parseData(st, currentAttribute, currentData);
            currentAttribute++;
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantData = !wantData;
        token = st.nextToken();
    }

    return this.getParser().getBeginningOfLineState();
}