java.io.StreamTokenizer Java Examples

The following examples show how to use java.io.StreamTokenizer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: netcdf-java   Author: Unidata   File: Json.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
protected Object parseR(StreamTokenizer tokens) throws IOException {
  int token = tokens.nextToken();
  switch (token) {
    case TT_EOF:
      return null;
    case TT_WORD:
      return parseAtomic(tokens);
    case LBRACE:
      return parseMap(tokens);
    case LBRACKET:
      return parseArray(tokens);
    case QUOTE:
      return parseAtomic(tokens);
    default:
      throw new IOException("Unexpected token:" + (char) token);
  }
}
 
Example #2
Source Project: jdk8u-dev-jdk   Author: frohoff   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 
Example #3
Source Project: jdk8u-jdk   Author: lambdalab-mirror   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 
Example #4
Source Project: BungeeChat2   Author: AuraDevelopmentTeam   File: BungeeChatCommand.java    License: GNU General Public License v3.0 6 votes vote down vote up
private String getUnquotedString(String str) {
  if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str;

  new StreamTokenizer(new StringReader(str));
  StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
  String result;

  try {
    parser.nextToken();
    if (parser.ttype == '"') {
      result = parser.sval;
    } else {
      result = "ERROR!";
    }
  } catch (IOException e) {
    result = null;

    LoggerHelper.info("Encountered an IOException while parsing the input string", e);
  }

  return result;
}
 
Example #5
Source Project: openjdk-jdk8u   Author: AdoptOpenJDK   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example #6
Source Project: dragonwell8_jdk   Author: alibaba   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 
Example #7
Source Project: dragonwell8_jdk   Author: alibaba   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 
Example #8
Source Project: gemfirexd-oss   Author: gemxd   File: TestProto.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read an int or codepoint - codepoint is given as a string
 */
private int getIntOrCP() throws IOException
{
    int val = tkn.nextToken();
    if (val == StreamTokenizer.TT_NUMBER)
    {
        return new Double(tkn.nval).intValue();
    }
    else if (val == StreamTokenizer.TT_WORD)
    {
        return decodeCP(tkn.sval);
    }
    else
    {
        fail("Expecting number, got " + tkn.sval + " on line "
                           + tkn.lineno());
        System.exit(1);
    }
    return 0;
}
 
Example #9
Source Project: openjdk-jdk9   Author: AdoptOpenJDK   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 
Example #10
Source Project: TencentKona-8   Author: Tencent   File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example #11
Source Project: openjdk-8-source   Author: keerath   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example #12
Source Project: openjdk-jdk8u-backup   Author: AdoptOpenJDK   File: CommandLine.java    License: GNU General Public License v2.0 6 votes vote down vote up
private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 
Example #13
Source Project: gama   Author: gama-platform   File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Read a word, a symbol or EOF, or generate a parse error. If this is EOF, the string "EOF" is returned.
 */
protected String getWordOrSymbolOrEof() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_NUMBER || tok == QUOTE_CHAR) {
		parseError("expecting a word or symbol, " + gotWhat(tok));
	}

	if (tok == StreamTokenizer.TT_WORD) {
		return st.sval;
	} else if (tok == StreamTokenizer.TT_EOF) {
		return "EOF";
	} else {
		return Character.toString((char) tok);
	}
}
 
Example #14
Source Project: TencentKona-8   Author: Tencent   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 
Example #15
Source Project: openjdk-8-source   Author: keerath   File: CommandLine.java    License: GNU General Public License v2.0 6 votes vote down vote up
private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 
Example #16
Source Project: openjdk-jdk8u-backup   Author: AdoptOpenJDK   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example #17
Source Project: jdk8u_jdk   Author: JetBrains   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example #18
Source Project: moa   Author: Waikato   File: ArffLoader.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Reads instance. It detects if it is dense or sparse.
 *
 * @return the instance
 */
public Instance readInstance() {
    while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
        try {
            streamTokenizer.nextToken();
        } catch (IOException ex) {
            Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (streamTokenizer.ttype == '{') {
        return readInstanceSparse();
        // return readDenseInstanceSparse();
    } else {
        return readInstanceDense();
    }

}
 
Example #19
Source Project: gemfirexd-oss   Author: gemxd   File: TestProto.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Print failure message and skip to the next test
	 *
 * @exception 	IOException 	error reading file
 */
private void fail(String msg) throws IOException
{
	System.out.println("FAILED - " + msg + " in line " + tkn.lineno());
	// skip remainder of the test look for endtest or end of file
	int val = tkn.nextToken();
	while (val != StreamTokenizer.TT_EOF)
	{
		if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest"))
			break;

		val = tkn.nextToken();
	}
	failed = true;
	// get ready for next test
	reset();
	// print out stack trace so we know where the failure occurred
	Exception e = new Exception();
	e.printStackTrace();
}
 
Example #20
Source Project: gama   Author: gama-platform   File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Push a tokenizer created from a file name on the file stack and make it current.
 *
 * @param file
 *            Name of the file used as source for the tokenizer.
 */
protected void pushTokenizer(final String file) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	try {
		reader = createReaderFrom(file);
		tok = createTokenizer(reader);

		cur = new CurrentFile(file, tok, reader);
	} catch (final FileNotFoundException e) {
		throw new IOException("cannot read file '" + file + "', not found: " + e.getMessage());
	}

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = file;
}
 
Example #21
Source Project: moa   Author: Waikato   File: ArffLoader.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Instantiates a new arff loader.
 *
 * @param reader the reader
 * @param range the range
 */
public ArffLoader(Reader reader, Range range) {
    this.range = range;
    BufferedReader br = new BufferedReader(reader);

    //Init streamTokenizer
    streamTokenizer = new StreamTokenizer(br);
    streamTokenizer.resetSyntax();
    streamTokenizer.whitespaceChars(0, ' ');
    streamTokenizer.wordChars(' ' + 1, '\u00FF');
    streamTokenizer.whitespaceChars(',', ',');
    streamTokenizer.commentChar('%');
    streamTokenizer.quoteChar('"');
    streamTokenizer.quoteChar('\'');
    streamTokenizer.ordinaryChar('{');
    streamTokenizer.ordinaryChar('}');
    streamTokenizer.eolIsSignificant(true);

    this.instanceInformation = this.getHeader();

    if (range != null) { //is MultiLabel
        this.instanceInformation.setRangeOutputIndices(range);
    }

}
 
Example #22
Source Project: spliceengine   Author: splicemachine   File: ProtocolTest.java    License: GNU Affero General Public License v3.0 6 votes vote down vote up
/**
 * Processes the test commands in the stream.
 *
 * @param cmdStream command stream (see {@link #commandSequence})
 * @throws IOException if accessing a file or the socket fails, or if
 *      Derby detects an unexpected protocol error
 */
private void processCommands(Reader cmdStream)
        throws  IOException {
    StreamTokenizer tkn = new StreamTokenizer(cmdStream);
    boolean endSignalled = false;
    int val;
    while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF) {
        assertFalse("End signalled, data to process left: " + tkn.sval,
                endSignalled);
        switch(val) {
            case StreamTokenizer.TT_NUMBER:
                break;
            case StreamTokenizer.TT_WORD:
                endSignalled = processCommand(tkn);
                break;
            case StreamTokenizer.TT_EOL:
                break;
        }
    }
}
 
Example #23
Source Project: hottub   Author: dsrg-uoft   File: Harness.java    License: GNU General Public License v2.0 6 votes vote down vote up
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 
Example #24
Source Project: openjdk-jdk8u-backup   Author: AdoptOpenJDK   File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example #25
Source Project: jdk8u-dev-jdk   Author: frohoff   File: Token.java    License: GNU General Public License v2.0 6 votes vote down vote up
public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 
Example #26
Source Project: KEEL   Author: SCI2SUGR   File: Algorithm.java    License: GNU General Public License v3.0 6 votes vote down vote up
/** Function to initialize the stream tokenizer.
 *
 * @param tokenizer The tokenizer.
 */
protected void initTokenizer( StreamTokenizer tokenizer )
{
  tokenizer.resetSyntax();
  tokenizer.whitespaceChars( 0, ' ' );
  tokenizer.wordChars( ' '+1,'\u00FF' );
  tokenizer.whitespaceChars( ',',',' );
  tokenizer.quoteChar( '"' );
  tokenizer.quoteChar( '\''  );
  tokenizer.ordinaryChar( '=' );
  tokenizer.ordinaryChar( '{' );
  tokenizer.ordinaryChar( '}' );
  tokenizer.ordinaryChar( '[' );
  tokenizer.ordinaryChar( ']' );
  tokenizer.eolIsSignificant( true );
}
 
Example #27
Source Project: gama   Author: gama-platform   File: FileSourceBase.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Push a tokenizer created from a stream on the file stack and make it current.
 *
 * @param stream
 *            The stream used as source for the tokenizer.
 * @param name
 *            The name of the input stream.
 */
protected void pushTokenizer(final InputStream stream, final String name) throws IOException {
	StreamTokenizer tok;
	CurrentFile cur;
	Reader reader;

	reader = createReaderFrom(stream);
	tok = createTokenizer(reader);
	cur = new CurrentFile(name, tok, reader);

	configureTokenizer(tok);
	tok_stack.add(cur);

	st = tok;
	filename = name;
}
 
Example #28
Source Project: lucene-solr   Author: apache   File: SimpleWKTShapeParser.java    License: Apache License 2.0 6 votes vote down vote up
public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
  try (StringReader reader = new StringReader(wkt)) {
    // setup the tokenizer; configured to read words w/o numbers
    StreamTokenizer tokenizer = new StreamTokenizer(reader);
    tokenizer.resetSyntax();
    tokenizer.wordChars('a', 'z');
    tokenizer.wordChars('A', 'Z');
    tokenizer.wordChars(128 + 32, 255);
    tokenizer.wordChars('0', '9');
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('+', '+');
    tokenizer.wordChars('.', '.');
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    Object geometry = parseGeometry(tokenizer, shapeType);
    checkEOF(tokenizer);
    return geometry;
  }
}
 
Example #29
Source Project: netcdf-java   Author: Unidata   File: Json.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
protected Object parseArray(StreamTokenizer tokens) throws IOException {
  assert (tokens.ttype == LBRACKET);
  List<Object> array = new ArrayList<>();
  loop: for (;;) {
    int token = tokens.nextToken();
    switch (token) {
      case TT_EOL:
        break; // ignore
      case TT_EOF:
        throw new IOException("Unexpected eof");
      case RBRACKET:
        break loop;
      default:
        tokens.pushBack();
        Object o = parseR(tokens);
        tokens.nextToken();
        if (tokens.ttype == TT_EOF)
          break;
        else if (tokens.ttype == RBRACKET)
          tokens.pushBack();
        else if (tokens.ttype != COMMA)
          throw new IOException("Missing comma in list");
        array.add(o);
    }
  }
  return array;
}
 
Example #30
Source Project: reladomo   Author: goldmansachs   File: DataReaderState.java    License: Apache License 2.0 5 votes vote down vote up
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    if (dataClass == null)
    {
        throw new ParseException("no class name found before line "+st.lineno(), st.lineno());
    }
    MithraDataObject currentData;
    currentData = this.getParser().getCurrentParsedData().createAndAddDataObject(st.lineno());

    // parse the data
    int currentAttribute = 0;
    int token = st.ttype;

    boolean wantData = true;
    while(token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF)
    {
        if (wantData)
        {
            this.getParser().getCurrentParsedData().parseData(st, currentAttribute, currentData);
            currentAttribute++;
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantData = !wantData;
        token = st.nextToken();
    }

    return this.getParser().getBeginningOfLineState();
}