java.io.StreamTokenizer Java Examples
The following examples show how to use
java.io.StreamTokenizer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleWKTShapeParser.java From lucene-solr with Apache License 2.0 | 6 votes |
public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException { try (StringReader reader = new StringReader(wkt)) { // setup the tokenizer; configured to read words w/o numbers StreamTokenizer tokenizer = new StreamTokenizer(reader); tokenizer.resetSyntax(); tokenizer.wordChars('a', 'z'); tokenizer.wordChars('A', 'Z'); tokenizer.wordChars(128 + 32, 255); tokenizer.wordChars('0', '9'); tokenizer.wordChars('-', '-'); tokenizer.wordChars('+', '+'); tokenizer.wordChars('.', '.'); tokenizer.whitespaceChars(0, ' '); tokenizer.commentChar('#'); Object geometry = parseGeometry(tokenizer, shapeType); checkEOF(tokenizer); return geometry; } }
Example #2
Source File: Harness.java From jdk8u-jdk with GNU General Public License v2.0 | 6 votes |
String[] parseBenchArgs(StreamTokenizer tokens) throws IOException, ConfigFormatException { Vector vec = new Vector(); for (;;) { switch (tokens.ttype) { case StreamTokenizer.TT_EOF: case StreamTokenizer.TT_EOL: return (String[]) vec.toArray(new String[vec.size()]); case StreamTokenizer.TT_WORD: case '"': vec.add(tokens.sval); tokens.nextToken(); break; default: throw new ConfigFormatException("unrecognized arg token " + "on line " + tokens.lineno()); } } }
Example #3
Source File: Harness.java From openjdk-jdk8u with GNU General Public License v2.0 | 6 votes |
Benchmark parseBenchClass(StreamTokenizer tokens) throws IOException, ConfigFormatException { Benchmark bench; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { Class cls = Class.forName(tokens.sval); bench = (Benchmark) cls.newInstance(); } catch (Exception e) { throw new ConfigFormatException("unable to instantiate " + "benchmark \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return bench; default: throw new ConfigFormatException("missing benchmark class " + "name on line " + tokens.lineno()); } }
Example #4
Source File: Harness.java From dragonwell8_jdk with GNU General Public License v2.0 | 6 votes |
String parseBenchName(StreamTokenizer tokens) throws IOException, ConfigFormatException { String name; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': name = tokens.sval; tokens.nextToken(); return name; default: throw new ConfigFormatException("missing benchmark name on " + "line " + tokens.lineno()); } }
Example #5
Source File: Harness.java From jdk8u-dev-jdk with GNU General Public License v2.0 | 6 votes |
float parseBenchWeight(StreamTokenizer tokens) throws IOException, ConfigFormatException { float weight; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { weight = Float.parseFloat(tokens.sval); } catch (NumberFormatException e) { throw new ConfigFormatException("illegal weight value \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return weight; default: throw new ConfigFormatException("missing weight value on line " + tokens.lineno()); } }
Example #6
Source File: Harness.java From dragonwell8_jdk with GNU General Public License v2.0 | 6 votes |
float parseBenchWeight(StreamTokenizer tokens) throws IOException, ConfigFormatException { float weight; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { weight = Float.parseFloat(tokens.sval); } catch (NumberFormatException e) { throw new ConfigFormatException("illegal weight value \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return weight; default: throw new ConfigFormatException("missing weight value on line " + tokens.lineno()); } }
Example #7
Source File: BungeeChatCommand.java From BungeeChat2 with GNU General Public License v3.0 | 6 votes |
private String getUnquotedString(String str) { if ((str == null) || !(str.startsWith("\"") && str.endsWith("\""))) return str; new StreamTokenizer(new StringReader(str)); StreamTokenizer parser = new StreamTokenizer(new StringReader(str)); String result; try { parser.nextToken(); if (parser.ttype == '"') { result = parser.sval; } else { result = "ERROR!"; } } catch (IOException e) { result = null; LoggerHelper.info("Encountered an IOException while parsing the input string", e); } return result; }
Example #8
Source File: TestProto.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
/** * Read an int or codepoint - codepoint is given as a string */ private int getIntOrCP() throws IOException { int val = tkn.nextToken(); if (val == StreamTokenizer.TT_NUMBER) { return new Double(tkn.nval).intValue(); } else if (val == StreamTokenizer.TT_WORD) { return decodeCP(tkn.sval); } else { fail("Expecting number, got " + tkn.sval + " on line " + tkn.lineno()); System.exit(1); } return 0; }
Example #9
Source File: Harness.java From openjdk-jdk9 with GNU General Public License v2.0 | 6 votes |
String[] parseBenchArgs(StreamTokenizer tokens) throws IOException, ConfigFormatException { Vector vec = new Vector(); for (;;) { switch (tokens.ttype) { case StreamTokenizer.TT_EOF: case StreamTokenizer.TT_EOL: return (String[]) vec.toArray(new String[vec.size()]); case StreamTokenizer.TT_WORD: case '"': vec.add(tokens.sval); tokens.nextToken(); break; default: throw new ConfigFormatException("unrecognized arg token " + "on line " + tokens.lineno()); } } }
Example #10
Source File: Token.java From TencentKona-8 with GNU General Public License v2.0 | 6 votes |
public String toMessage() { switch(ttype) { case StreamTokenizer.TT_EOL: return "\"EOL\""; case StreamTokenizer.TT_EOF: return "\"EOF\""; case StreamTokenizer.TT_NUMBER: return "NUMBER"; case StreamTokenizer.TT_WORD: if (sval == null) { return "IDENTIFIER"; } else { return "IDENTIFIER " + sval; } default: if (ttype == (int)'"') { String msg = "QUOTED STRING"; if (sval != null) msg = msg + " \"" + sval + "\""; return msg; } else { return "CHARACTER \'" + (char)ttype + "\'"; } } }
Example #11
Source File: Harness.java From openjdk-8-source with GNU General Public License v2.0 | 6 votes |
Benchmark parseBenchClass(StreamTokenizer tokens) throws IOException, ConfigFormatException { Benchmark bench; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { Class cls = Class.forName(tokens.sval); bench = (Benchmark) cls.newInstance(); } catch (Exception e) { throw new ConfigFormatException("unable to instantiate " + "benchmark \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return bench; default: throw new ConfigFormatException("missing benchmark class " + "name on line " + tokens.lineno()); } }
Example #12
Source File: CommandLine.java From openjdk-jdk8u-backup with GNU General Public License v2.0 | 6 votes |
private static void loadCmdFile(String name, List<String> args) throws IOException { Reader r = new BufferedReader(new FileReader(name)); StreamTokenizer st = new StreamTokenizer(r); st.resetSyntax(); st.wordChars(' ', 255); st.whitespaceChars(0, ' '); st.commentChar('#'); st.quoteChar('"'); st.quoteChar('\''); while (st.nextToken() != StreamTokenizer.TT_EOF) { args.add(st.sval); } r.close(); }
Example #13
Source File: FileSourceBase.java From gama with GNU General Public License v3.0 | 6 votes |
/** * Read a word, a symbol or EOF, or generate a parse error. If this is EOF, the string "EOF" is returned. */ protected String getWordOrSymbolOrEof() throws IOException { final int tok = st.nextToken(); if (tok == StreamTokenizer.TT_NUMBER || tok == QUOTE_CHAR) { parseError("expecting a word or symbol, " + gotWhat(tok)); } if (tok == StreamTokenizer.TT_WORD) { return st.sval; } else if (tok == StreamTokenizer.TT_EOF) { return "EOF"; } else { return Character.toString((char) tok); } }
Example #14
Source File: Harness.java From TencentKona-8 with GNU General Public License v2.0 | 6 votes |
String parseBenchName(StreamTokenizer tokens) throws IOException, ConfigFormatException { String name; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': name = tokens.sval; tokens.nextToken(); return name; default: throw new ConfigFormatException("missing benchmark name on " + "line " + tokens.lineno()); } }
Example #15
Source File: CommandLine.java From openjdk-8-source with GNU General Public License v2.0 | 6 votes |
private static void loadCmdFile(String name, List args) throws IOException { Reader r = new BufferedReader(new FileReader(name)); StreamTokenizer st = new StreamTokenizer(r); st.resetSyntax(); st.wordChars(' ', 255); st.whitespaceChars(0, ' '); st.commentChar('#'); st.quoteChar('"'); st.quoteChar('\''); while (st.nextToken() != st.TT_EOF) { args.add(st.sval); } r.close(); }
Example #16
Source File: Harness.java From openjdk-jdk8u-backup with GNU General Public License v2.0 | 6 votes |
Benchmark parseBenchClass(StreamTokenizer tokens) throws IOException, ConfigFormatException { Benchmark bench; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { Class cls = Class.forName(tokens.sval); bench = (Benchmark) cls.newInstance(); } catch (Exception e) { throw new ConfigFormatException("unable to instantiate " + "benchmark \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return bench; default: throw new ConfigFormatException("missing benchmark class " + "name on line " + tokens.lineno()); } }
Example #17
Source File: Harness.java From jdk8u_jdk with GNU General Public License v2.0 | 6 votes |
Benchmark parseBenchClass(StreamTokenizer tokens) throws IOException, ConfigFormatException { Benchmark bench; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { Class cls = Class.forName(tokens.sval); bench = (Benchmark) cls.newInstance(); } catch (Exception e) { throw new ConfigFormatException("unable to instantiate " + "benchmark \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return bench; default: throw new ConfigFormatException("missing benchmark class " + "name on line " + tokens.lineno()); } }
Example #18
Source File: ArffLoader.java From moa with GNU General Public License v3.0 | 6 votes |
/** * Reads instance. It detects if it is dense or sparse. * * @return the instance */ public Instance readInstance() { while (streamTokenizer.ttype == StreamTokenizer.TT_EOL) { try { streamTokenizer.nextToken(); } catch (IOException ex) { Logger.getLogger(ArffLoader.class.getName()).log(Level.SEVERE, null, ex); } } if (streamTokenizer.ttype == '{') { return readInstanceSparse(); // return readDenseInstanceSparse(); } else { return readInstanceDense(); } }
Example #19
Source File: TestProto.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
/** * Print failure message and skip to the next test * * @exception IOException error reading file */ private void fail(String msg) throws IOException { System.out.println("FAILED - " + msg + " in line " + tkn.lineno()); // skip remainder of the test look for endtest or end of file int val = tkn.nextToken(); while (val != StreamTokenizer.TT_EOF) { if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest")) break; val = tkn.nextToken(); } failed = true; // get ready for next test reset(); // print out stack trace so we know where the failure occurred Exception e = new Exception(); e.printStackTrace(); }
Example #20
Source File: FileSourceBase.java From gama with GNU General Public License v3.0 | 6 votes |
/** * Push a tokenizer created from a file name on the file stack and make it current. * * @param file * Name of the file used as source for the tokenizer. */ protected void pushTokenizer(final String file) throws IOException { StreamTokenizer tok; CurrentFile cur; Reader reader; try { reader = createReaderFrom(file); tok = createTokenizer(reader); cur = new CurrentFile(file, tok, reader); } catch (final FileNotFoundException e) { throw new IOException("cannot read file '" + file + "', not found: " + e.getMessage()); } configureTokenizer(tok); tok_stack.add(cur); st = tok; filename = file; }
Example #21
Source File: ArffLoader.java From moa with GNU General Public License v3.0 | 6 votes |
/** * Instantiates a new arff loader. * * @param reader the reader * @param range the range */ public ArffLoader(Reader reader, Range range) { this.range = range; BufferedReader br = new BufferedReader(reader); //Init streamTokenizer streamTokenizer = new StreamTokenizer(br); streamTokenizer.resetSyntax(); streamTokenizer.whitespaceChars(0, ' '); streamTokenizer.wordChars(' ' + 1, '\u00FF'); streamTokenizer.whitespaceChars(',', ','); streamTokenizer.commentChar('%'); streamTokenizer.quoteChar('"'); streamTokenizer.quoteChar('\''); streamTokenizer.ordinaryChar('{'); streamTokenizer.ordinaryChar('}'); streamTokenizer.eolIsSignificant(true); this.instanceInformation = this.getHeader(); if (range != null) { //is MultiLabel this.instanceInformation.setRangeOutputIndices(range); } }
Example #22
Source File: ProtocolTest.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
/** * Processes the test commands in the stream. * * @param cmdStream command stream (see {@link #commandSequence}) * @throws IOException if accessing a file or the socket fails, or if * Derby detects an unexpected protocol error */ private void processCommands(Reader cmdStream) throws IOException { StreamTokenizer tkn = new StreamTokenizer(cmdStream); boolean endSignalled = false; int val; while ( (val = tkn.nextToken()) != StreamTokenizer.TT_EOF) { assertFalse("End signalled, data to process left: " + tkn.sval, endSignalled); switch(val) { case StreamTokenizer.TT_NUMBER: break; case StreamTokenizer.TT_WORD: endSignalled = processCommand(tkn); break; case StreamTokenizer.TT_EOL: break; } } }
Example #23
Source File: Harness.java From hottub with GNU General Public License v2.0 | 6 votes |
Benchmark parseBenchClass(StreamTokenizer tokens) throws IOException, ConfigFormatException { Benchmark bench; switch (tokens.ttype) { case StreamTokenizer.TT_WORD: case '"': try { Class cls = Class.forName(tokens.sval); bench = (Benchmark) cls.newInstance(); } catch (Exception e) { throw new ConfigFormatException("unable to instantiate " + "benchmark \"" + tokens.sval + "\" on line " + tokens.lineno()); } tokens.nextToken(); return bench; default: throw new ConfigFormatException("missing benchmark class " + "name on line " + tokens.lineno()); } }
Example #24
Source File: Token.java From openjdk-jdk8u-backup with GNU General Public License v2.0 | 6 votes |
public String toMessage() { switch(ttype) { case StreamTokenizer.TT_EOL: return "\"EOL\""; case StreamTokenizer.TT_EOF: return "\"EOF\""; case StreamTokenizer.TT_NUMBER: return "NUMBER"; case StreamTokenizer.TT_WORD: if (sval == null) { return "IDENTIFIER"; } else { return "IDENTIFIER " + sval; } default: if (ttype == (int)'"') { String msg = "QUOTED STRING"; if (sval != null) msg = msg + " \"" + sval + "\""; return msg; } else { return "CHARACTER \'" + (char)ttype + "\'"; } } }
Example #25
Source File: Token.java From jdk8u-dev-jdk with GNU General Public License v2.0 | 6 votes |
public String toMessage() { switch(ttype) { case StreamTokenizer.TT_EOL: return "\"EOL\""; case StreamTokenizer.TT_EOF: return "\"EOF\""; case StreamTokenizer.TT_NUMBER: return "NUMBER"; case StreamTokenizer.TT_WORD: if (sval == null) { return "IDENTIFIER"; } else { return "IDENTIFIER " + sval; } default: if (ttype == (int)'"') { String msg = "QUOTED STRING"; if (sval != null) msg = msg + " \"" + sval + "\""; return msg; } else { return "CHARACTER \'" + (char)ttype + "\'"; } } }
Example #26
Source File: Algorithm.java From KEEL with GNU General Public License v3.0 | 6 votes |
/** Function to initialize the stream tokenizer. * * @param tokenizer The tokenizer. */ protected void initTokenizer( StreamTokenizer tokenizer ) { tokenizer.resetSyntax(); tokenizer.whitespaceChars( 0, ' ' ); tokenizer.wordChars( ' '+1,'\u00FF' ); tokenizer.whitespaceChars( ',',',' ); tokenizer.quoteChar( '"' ); tokenizer.quoteChar( '\'' ); tokenizer.ordinaryChar( '=' ); tokenizer.ordinaryChar( '{' ); tokenizer.ordinaryChar( '}' ); tokenizer.ordinaryChar( '[' ); tokenizer.ordinaryChar( ']' ); tokenizer.eolIsSignificant( true ); }
Example #27
Source File: FileSourceBase.java From gama with GNU General Public License v3.0 | 6 votes |
/** * Push a tokenizer created from a stream on the file stack and make it current. * * @param stream * The stream used as source for the tokenizer. * @param name * The name of the input stream. */ protected void pushTokenizer(final InputStream stream, final String name) throws IOException { StreamTokenizer tok; CurrentFile cur; Reader reader; reader = createReaderFrom(stream); tok = createTokenizer(reader); cur = new CurrentFile(name, tok, reader); configureTokenizer(tok); tok_stack.add(cur); st = tok; filename = name; }
Example #28
Source File: Json.java From netcdf-java with BSD 3-Clause "New" or "Revised" License | 6 votes |
protected Object parseR(StreamTokenizer tokens) throws IOException { int token = tokens.nextToken(); switch (token) { case TT_EOF: return null; case TT_WORD: return parseAtomic(tokens); case LBRACE: return parseMap(tokens); case LBRACKET: return parseArray(tokens); case QUOTE: return parseAtomic(tokens); default: throw new IOException("Unexpected token:" + (char) token); } }
Example #29
Source File: DataReaderState.java From reladomo with Apache License 2.0 | 5 votes |
public ParserState parse(StreamTokenizer st) throws IOException, ParseException { if (dataClass == null) { throw new ParseException("no class name found before line "+st.lineno(), st.lineno()); } MithraDataObject currentData; currentData = this.getParser().getCurrentParsedData().createAndAddDataObject(st.lineno()); // parse the data int currentAttribute = 0; int token = st.ttype; boolean wantData = true; while(token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF) { if (wantData) { this.getParser().getCurrentParsedData().parseData(st, currentAttribute, currentData); currentAttribute++; } else { if (token != ',') { throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno()); } } wantData = !wantData; token = st.nextToken(); } return this.getParser().getBeginningOfLineState(); }
Example #30
Source File: AttributeReaderState.java From reladomo with Apache License 2.0 | 5 votes |
private void parseAttributes(StreamTokenizer st) throws IOException, ParseException { int token = st.nextToken(); boolean wantAttribute = true; while(token != StreamTokenizer.TT_EOL) { if (wantAttribute) { if (token != StreamTokenizer.TT_WORD) { throw new ParseException("expected an attribute name on line "+st.lineno(), st.lineno()); } this.getParser().getCurrentParsedData().addAttribute(st.sval, st.lineno()); } else { if (token != ',') { throw new ParseException("Expected a comma on line "+st.lineno(), st.lineno()); } } wantAttribute = !wantAttribute; token = st.nextToken(); } if (wantAttribute) { throw new ParseException("extra comma at the end of line "+st.lineno(), st.lineno()); } }