Java Code Examples for java.io.StreamTokenizer

The following examples show how to use java.io.StreamTokenizer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: netcdf-java   Source File: Json.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
protected Object parseR(StreamTokenizer tokens) throws IOException {
  int token = tokens.nextToken();
  switch (token) {
    case TT_EOF:
      return null;
    case TT_WORD:
      return parseAtomic(tokens);
    case LBRACE:
      return parseMap(tokens);
    case LBRACKET:
      return parseArray(tokens);
    case QUOTE:
      return parseAtomic(tokens);
    default:
      throw new IOException("Unexpected token:" + (char) token);
  }
}
 
Example 2
Source Project: jdk8u-dev-jdk   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 
Example 3
Source Project: jdk8u-jdk   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 
Example 4
private String getUnquotedString(String str) {
  if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;

  new StreamTokenizer(new StringReader(str));
  StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
  String result;

  try {
    parser.nextToken();
    if (parser.ttype == '"') {
      result = parser.sval;
    } else {
      result = "ERROR!";
    }
  } catch (IOException e) {
    result = null;

    LoggerHelper.info("Encountered an IOException while parsing the input string", e);
  }

  return result;
}
 
Example 5
Source Project: openjdk-jdk8u   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example 6
Source Project: dragonwell8_jdk   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 
Example 7
Source Project: dragonwell8_jdk   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 
Example 8
Source Project: gemfirexd-oss   Source File: TestProto.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read an int or codepoint - codepoint is given as a string
 */
private int getIntOrCP() throws IOException
{
    int val = tkn.nextToken();
    if (val == StreamTokenizer.TT_NUMBER)
    {
        return new Double(tkn.nval).intValue();
    }
    else if (val == StreamTokenizer.TT_WORD)
    {
        return decodeCP(tkn.sval);
    }
    else
    {
        fail("Expecting number, got " + tkn.sval + " on line "
                           + tkn.lineno());
        System.exit(1);
    }
    return 0;
}
 
Example 9
Source Project: openjdk-jdk9   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 
Example 10
Source Project: TencentKona-8   Source File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example 11
Source Project: openjdk-8-source   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example 12
private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 
Example 13
Source Project: gama   Source File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Read a word, a symbol or EOF, or generate a parse error. If this is EOF, the string "EOF" is returned.
 */
protected String getWordOrSymbolOrEof() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_NUMBER || tok == QUOTE_CHAR) {
		parseError("expecting a word or symbol, " + gotWhat(tok));
	}

	if (tok == StreamTokenizer.TT_WORD) {
		return st.sval;
	} else if (tok == StreamTokenizer.TT_EOF) {
		return "EOF";
	} else {
		return Character.toString((char) tok);
	}
}
 
Example 14
Source Project: TencentKona-8   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 
Example 15
Source Project: openjdk-8-source   Source File: CommandLine.java    License: GNU General Public License v2.0 6 votes vote down vote up
private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 
Example 16
Source Project: openjdk-jdk8u-backup   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example 17
Source Project: jdk8u_jdk   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example 18
Source Project: moa   Source File: ArffLoader.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Reads instance. It detects if it is dense or sparse.
 *
 * @return the instance
 */
public Instance readInstance() {
    while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
        try {
            streamTokenizer.nextToken();
        } catch (IOException ex) {
            Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (streamTokenizer.ttype == '{') {
        return readInstanceSparse();
        // return readDenseInstanceSparse();
    } else {
        return readInstanceDense();
    }

}
 
Example 19
Source Project: gemfirexd-oss   Source File: TestProto.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Print failure message and skip to the next test
	 *
 * @exception 	IOException 	error reading file
 */
private void fail(String msg) throws IOException
{
	System.out.println("FAILED - " + msg + " in line " + tkn.lineno());
	// skip remainder of the test look for endtest or end of file
	int val = tkn.nextToken();
	while (val != StreamTokenizer.TT_EOF)
	{
		if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest"))
			break;

		val = tkn.nextToken();
	}
	failed = true;
	// get ready for next test
	reset();
	// print out stack trace so we know where the failure occurred
	Exception e = new Exception();
	e.printStackTrace();
}
 
Example 20
Source Project: gama   Source File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Push a tokenizer created from a file name on the file stack and make it current.
 *
 * @param file
 *            Name of the file used as source for the tokenizer.
 */
protected void pushTokenizer(final String file) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	try {
		reader = createReaderFrom(file);
		tok = createTokenizer(reader);

		cur = new CurrentFile(file, tok, reader);
	} catch (final FileNotFoundException e) {
		throw new IOException("cannot read file '" + file + "', not found: " + e.getMessage());
	}

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = file;
}
 
Example 21
Source Project: moa   Source File: ArffLoader.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range the range
 */
public ArffLoader(Reader reader, Range range) {
    this.range = range;
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);
    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();

    if (range != null) { //is MultiLabel
        this.instanceInformation.setRangeOutputIndices(range);
    }

}
 
Example 22
/**
 * Processes the test commands in the stream.
 *
 * @param cmdStream command stream (see {@link #commandSequence})
 * @throws IOException if accessing a file or the socket fails, or if
 *      Derby detects an unexpected protocol error
 */
private void processCommands(Reader cmdStream)
        throws  IOException {
    StreamTokenizer tkn = new StreamTokenizer(cmdStream);
    boolean endSignalled = false;
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF) {
        assertFalse("End signalled, data to process left: " + tkn.sval,
                endSignalled);
        switch(val) {
            case StreamTokenizer.TT_NUMBER:
                break;
            case StreamTokenizer.TT_WORD:
                endSignalled = processCommand(tkn);
                break;
            case StreamTokenizer.TT_EOL:
                break;
        }
    }
}
 
Example 23
Source Project: hottub   Source File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example 24
Source Project: openjdk-jdk8u-backup   Source File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example 25
Source Project: jdk8u-dev-jdk   Source File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example 26
Source Project: KEEL   Source File: Algorithm.java    License: GNU General Public License v3.0 6 votes vote down vote up
/** Function to initialize the stream tokenizer.
 *
 * @param tokenizer The tokenizer.
 */
protected void initTokenizer( StreamTokenizer tokenizer )
{
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars( 0, ' ' );
  tokenizer.wordChars( ' '+1,'\u00FF' );
  tokenizer.whitespaceChars( ',',',' );
  tokenizer.quoteChar( '"' );
  tokenizer.quoteChar( '\''  );
  tokenizer.ordinaryChar( '=' );
  tokenizer.ordinaryChar( '{' );
  tokenizer.ordinaryChar( '}' );
  tokenizer.ordinaryChar( '[' );
  tokenizer.ordinaryChar( ']' );
  tokenizer.eolIsSignificant( true );
}
 
Example 27
Source Project: gama   Source File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Push a tokenizer created from a stream on the file stack and make it current.
 *
 * @param stream
 *            The stream used as source for the tokenizer.
 * @param name
 *            The name of the input stream.
 */
protected void pushTokenizer(final InputStream stream, final String name) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	reader = createReaderFrom(stream);
	tok = createTokenizer(reader);
	cur = new CurrentFile(name, tok, reader);

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = name;
}
 
Example 28
Source Project: lucene-solr   Source File: SimpleWKTShapeParser.java    License: Apache License 2.0 6 votes vote down vote up
public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
  try (StringReader reader = new StringReader(wkt)) {
    // setup the tokenizer; configured to read words w/o numbers
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars('a', 'z');
    tokenizer.wordChars('A', 'Z');
    tokenizer.wordChars(128 + 32, 255);
    tokenizer.wordChars('0', '9');
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('+', '+');
    tokenizer.wordChars('.', '.');
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    Object geometry = parseGeometry(tokenizer, shapeType);
    checkEOF(tokenizer);
    return geometry;
  }
}
 
Example 29
Source Project: netcdf-java   Source File: Json.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
protected Object parseArray(StreamTokenizer tokens) throws IOException {
  assert (tokens.ttype == LBRACKET);
  List<Object> array = new ArrayList<>();
  loop: for (;;) {
    int token = tokens.nextToken();
    switch (token) {
      case TT_EOL:
        break; // ignore
      case TT_EOF:
        throw new IOException("Unexpected eof");
      case RBRACKET:
        break loop;
      default:
        tokens.pushBack();
        Object o = parseR(tokens);
        tokens.nextToken();
        if (tokens.ttype == TT_EOF)
          break;
        else if (tokens.ttype == RBRACKET)
          tokens.pushBack();
        else if (tokens.ttype != COMMA)
          throw new IOException("Missing comma in list");
        array.add(o);
    }
  }
  return array;
}
 
Example 30
Source Project: reladomo   Source File: DataReaderState.java    License: Apache License 2.0 5 votes vote down vote up
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    if (dataClass == null)
    {
        throw new ParseException("no class name found before line "+st.lineno(), st.lineno());
    }
    MithraDataObject currentData;
    currentData = this.getParser().getCurrentParsedData().createAndAddDataObject(st.lineno());

    // parse the data
    int currentAttribute = 0;
    int token = st.ttype;

    boolean wantData = true;
    while(token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF)
    {
        if (wantData)
        {
            this.getParser().getCurrentParsedData().parseData(st, currentAttribute, currentData);
            currentAttribute++;
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantData = !wantData;
        token = st.nextToken();
    }

    return this.getParser().getBeginningOfLineState();
}