Java Code Examples for java.io.StreamTokenizer#TT_EOL

The following examples show how to use java.io.StreamTokenizer#TT_EOL . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FileSourceBase.java From gama with GNU General Public License v3.0

6 votes

/**
 * Read a string or number or pushback and return null. If it is a number it is converted to a string before being
 * returned.
 */
protected String getStringOrWordOrNumberOrPushback() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_EOL || tok == StreamTokenizer.TT_EOF) {
		pushBack();
		return null;
	}

	if (tok == StreamTokenizer.TT_NUMBER) {
		if (st.nval - (int) st.nval == 0) {
			return Integer.toString((int) st.nval);
		} else {
			return Double.toString(st.nval);
		}
	} else if (tok == StreamTokenizer.TT_WORD || tok == QUOTE_CHAR) {
		return st.sval;
	} else {
		pushBack();
		return null;
	}
}

Example 2

Source File: Harness.java From openjdk-jdk8u with GNU General Public License v2.0

6 votes

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}

Example 3

Source File: FileSourceBase.java From gama with GNU General Public License v3.0

6 votes

/**
 * Read a string or number or generate a parse error. If it is a number it is converted to a string before being
 * returned.
 */
protected String getStringOrWordOrNumber() throws IOException {
	final int tok = st.nextToken();

	if (tok == StreamTokenizer.TT_EOL || tok == StreamTokenizer.TT_EOF) {
		parseError("expecting word, string or number, " + gotWhat(tok));
	}

	if (tok == StreamTokenizer.TT_NUMBER) {
		if (st.nval - (int) st.nval == 0) {
			return Integer.toString((int) st.nval);
		} else {
			return Double.toString(st.nval);
		}
	} else {
		return st.sval;
	}
}

Example 4

Source File: ArffLoader.java From samoa with Apache License 2.0

6 votes

public Instance readInstance(Reader reader) {
    if (streamTokenizer == null) {
        initStreamTokenizer(reader);
    }
    while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) {
        try {
            streamTokenizer.nextToken();
        } catch (IOException ex) {
            Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    if (streamTokenizer.ttype == '{') {
        return readInstanceSparse();
        // return readDenseInstanceSparse();
    } else {
        return readInstanceDense();
    }

}

Example 5

Source File: Token.java From hottub with GNU General Public License v2.0

6 votes

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}

Example 6

Source File: Parser.java From big-c with Apache License 2.0

6 votes

Token next() throws IOException {
  int type = tok.nextToken();
  switch (type) {
    case StreamTokenizer.TT_EOF:
    case StreamTokenizer.TT_EOL:
      return null;
    case StreamTokenizer.TT_NUMBER:
      return new NumToken(tok.nval);
    case StreamTokenizer.TT_WORD:
      return new StrToken(TType.IDENT, tok.sval);
    case '"':
      return new StrToken(TType.QUOT, tok.sval);
    default:
      switch (type) {
        case ',':
          return new Token(TType.COMMA);
        case '(':
          return new Token(TType.LPAREN);
        case ')':
          return new Token(TType.RPAREN);
        default:
          throw new IOException("Unexpected: " + type);
      }
  }
}

Example 7

Source File: Token.java From openjdk-jdk8u-backup with GNU General Public License v2.0

6 votes

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}

Example 8

Source File: Token.java From jdk8u60 with GNU General Public License v2.0

6 votes

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}

Example 9

Source File: ArffLoader.java From tsml with GNU General Public License v3.0

5 votes

/**
 * Gets next token, checking for a premature and of line.
 *
 * @throws IOException 	if it finds a premature end of line
 */
protected void getNextToken() throws IOException {
  if (m_Tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    errorMessage("premature end of line");
  }
  if (m_Tokenizer.ttype == StreamTokenizer.TT_EOF) {
    errorMessage("premature end of file");
  } else if ((m_Tokenizer.ttype == '\'') ||
      (m_Tokenizer.ttype == '"')) {
    m_Tokenizer.ttype = StreamTokenizer.TT_WORD;
  } else if ((m_Tokenizer.ttype == StreamTokenizer.TT_WORD) &&
      (m_Tokenizer.sval.equals("?"))){
    m_Tokenizer.ttype = '?';
  }
}

Example 10

Source File: ClassReaderState.java From reladomo with Apache License 2.0

5 votes

public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.ttype;
    if (token != StreamTokenizer.TT_WORD || !st.sval.equals(AbstractMithraDataFileParser.CLASS_IDENTIFIER))
    {
        throw new ParseException("expected line " + st.lineno() + " to begin with class", st.lineno());
    }
    token = st.nextToken();
    if (token != StreamTokenizer.TT_WORD)
    {
        throw new ParseException("expected a class name on line "+st.lineno(), st.lineno());
    }
    this.getParser().addNewMithraParsedData();
    String className = st.sval;
    try
    {
        this.getParser().getCurrentParsedData().setParsedClassName(className);
    }
    catch (Exception e)
    {
        ParseException parseException = new ParseException("no such class (or finder): "+className+" on line "+st.lineno(), st.lineno());
        parseException.initCause(e);
        throw parseException;
    }
    this.getParser().getDataReaderState().setClass(className, st.lineno());

    token = st.nextToken();
    if (token != StreamTokenizer.TT_EOL)
    {
        throw new ParseException("invalid data after the class name on line "+st.lineno(), st.lineno());
    }

    return this.getParser().getAttributeReaderState();
}

Example 11

Source File: StreamTokenizerUtils.java From tsml with GNU General Public License v3.0

5 votes

/**
 * Gets token, skipping empty lines.
 * 
 * @param tokenizer the stream tokenizer
 * @throws IOException if reading the next token fails
 */
public static void getFirstToken(StreamTokenizer tokenizer)
    throws IOException {

  while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
  }
  ;
  if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
    tokenizer.ttype = StreamTokenizer.TT_WORD;
  } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD)
      && (tokenizer.sval.equals("?"))) {
    tokenizer.ttype = '?';
  }
}

Example 12

Source File: FileSourceBase.java From gama with GNU General Public License v3.0

5 votes

/**
 * Get a symbol.
 */
protected char getSymbol() throws IOException {
	final int tok = st.nextToken();

	if (tok > 0 && tok != StreamTokenizer.TT_WORD && tok != StreamTokenizer.TT_NUMBER
			&& tok != StreamTokenizer.TT_EOL && tok != StreamTokenizer.TT_EOF && tok != QUOTE_CHAR
			&& tok != COMMENT_CHAR) {
		return (char) tok;
	}

	parseError("expecting a symbol, " + gotWhat(tok));
	return (char) 0; // Never reached.
}

Example 13

Source File: FileSourceBase.java From gama with GNU General Public License v3.0

5 votes

/**
 * Eat all availables EOLs.
 */
protected void eatAllEols() throws IOException {
	if (!eol_is_significant) { return; }

	int tok = st.nextToken();

	while (tok == StreamTokenizer.TT_EOL) {
		tok = st.nextToken();
	}

	pushBack();
}

Example 14

Source File: ReadAhead.java From openjdk-jdk8u with GNU General Public License v2.0

5 votes

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}

Example 15

Source File: ReadAhead.java From jdk8u_jdk with GNU General Public License v2.0

5 votes

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}

Example 16

Source File: AttributeReaderState.java From reladomo with Apache License 2.0

5 votes

private void parseAttributes(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.nextToken();
    boolean wantAttribute = true;
    while(token != StreamTokenizer.TT_EOL)
    {
        if (wantAttribute)
        {
            if (token != StreamTokenizer.TT_WORD)
            {
                throw new ParseException("expected an attribute name on line "+st.lineno(), st.lineno());
            }
            this.getParser().getCurrentParsedData().addAttribute(st.sval, st.lineno());
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno());
            }
        }
        wantAttribute = !wantAttribute;
        token = st.nextToken();
    }
    if (wantAttribute)
    {
        throw new ParseException("extra comma at the end of line "+st.lineno(), st.lineno());
    }
}

Example 17

Source File: Harness.java From openjdk-jdk9 with GNU General Public License v2.0

5 votes

BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}

Example 18

Source File: Harness.java From openjdk-8-source with GNU General Public License v2.0

5 votes

BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}

Example 19

Source File: Token.java From jdk8u-jdk with GNU General Public License v2.0

5 votes

public String toString() {
    StringBuilder sb = new StringBuilder();
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        sb.append("ttype=TT_EOL");
        break;
    case StreamTokenizer.TT_EOF:
        sb.append("ttype=TT_EOF");
        break;
    case StreamTokenizer.TT_NUMBER:
        sb.append("ttype=TT_NUM,").append("nval="+nval);
        break;
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            sb.append("ttype=TT_WORD:IDENTIFIER");
        } else {
            sb.append("ttype=TT_WORD:").append("sval="+sval);
        }
        break;
    default:
        if (ttype == (int)'"') {
            sb.append("ttype=TT_STRING:").append("sval="+sval);
        } else {
            sb.append("ttype=TT_CHAR:").append((char)ttype);
        }
        break;
    }
    return sb.toString();
}

Example 20

Source File: ExpressionEndState.java From reladomo with Apache License 2.0

4 votes

@Override
public ComputedAttributeParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    ComputedAttributeParserState nextState = null;
    while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            ArrayList<Expression> stack = this.getParser().getStateStack();
            switch(nextToken)
            {
                case '.':
                    Expression expression = stack.remove(stack.size() - 1);
                    stack.add(new FunctionExpression(expression));
                    nextState = new FunctionBeginExpressionState(this.getParser());
                    break;
                case ',':
                    addParameterToFunction(st, stack);
                    nextState = new ExpressionBeginState(this.getParser());
                    break;
                case ';':
                    endCaseValue(st, stack);
                    nextState = new CaseSelectorBeginParserState(this.getParser());
                    break;
                case ')':
                    endFunctionOrCase(st, stack);
                    // nextState = new ExpressionEndState(this.getParser()); we can just loop here instead
                    break;
                case StreamTokenizer.TT_NUMBER:
                    throw new ParseException("unexpected number "+st.nval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
                case StreamTokenizer.TT_WORD:
                    throw new ParseException("unexpected word "+st.sval+" in expression "+this.getParser().getFormula()+" in "+this.getParser().getDiagnosticMessage());
                default:
                    char ch = (char)st.ttype;
                    throw createUnexpectedCharacterException(ch, ",;).");
            }
        }
    }
    return nextState;
}