org.apache.jena.riot.tokens.TokenType Java Examples
The following examples show how to use
org.apache.jena.riot.tokens.TokenType.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TokenizerText.java From RDFstarTools with Apache License 2.0 | 6 votes |
|
private void readPrefixedNameOrKeyword(Token token) {
long posn = reader.getPosition();
String prefixPart = readPrefixPart(); // Prefix part or keyword
token.setImage(prefixPart);
token.setType(TokenType.KEYWORD);
int ch = reader.peekChar();
if ( ch == CH_COLON ) {
reader.readChar();
token.setType(TokenType.PREFIXED_NAME);
String ln = readLocalPart(); // Local part
token.setImage2(ln);
if ( Checking )
checkPrefixedName(token.getImage(), token.getImage2());
}
// If we made no progress, nothing found, not even a keyword -- it's an
// error.
if ( posn == reader.getPosition() )
error("Failed to find a prefix name or keyword: %c(%d;0x%04X)", ch, ch, ch);
if ( Checking )
checkKeyword(token.getImage());
}
Example #2
| Source File: RdflintParserTurtle.java From rdflint with MIT License | 6 votes |
|
@Override
public Node create(Node currentGraph, Token token) {
int line = (int) token.getLine();
int col = (int) token.getColumn();
String str = token.getImage();
validationModels.forEach(m -> {
Node node = null;
int length = 0;
if (token.getType() == TokenType.PREFIXED_NAME) {
String prefix = str;
String suffix = token.getImage2();
String expansion = expandPrefixedName(prefix, suffix);
node = createURI(expansion, line, col);
length = str.length() + suffix.length();
} else if (token.getType() == TokenType.IRI) {
node = createURI(str, line, col);
length = str.length();
}
if (node != null) {
List<LintProblem> diagnostic = m
.validateNode(node, line, col, line, col + length);
diagnosticList.addAll(diagnostic);
}
});
return super.create(currentGraph, token);
}
Example #3
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void embeddedTripleOK() {
final Token token = getToken( "<<:s :p :o>>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
Example #4
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void nesting1() {
final Token token = getToken( "<< <<:s :p :o>> :p2 :o2>>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
confirmEmbeddedTripleToken(tokenS);
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
Example #5
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void nesting2() {
final Token token = getToken( "<<<<:s :p :o>> :p2 :o2>>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
confirmEmbeddedTripleToken(tokenS);
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenO.getType());
}
Example #6
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void nesting3() {
final Token token = getToken( "<< :s2 :p2 <<:s :p :o>> >>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
confirmEmbeddedTripleToken(tokenO);
}
Example #7
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void nesting4() {
final Token token = getToken( "<< :s2 :p2 <<:s :p :o>>>>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
assertEquals(TokenType.PREFIXED_NAME, tokenS.getType());
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
confirmEmbeddedTripleToken(tokenO);
}
Example #8
| Source File: TokenizerTextTest.java From RDFstarTools with Apache License 2.0 | 5 votes |
|
@Test
public void nesting5() {
final Token token = getToken( "<<<<:s :p :o>> :p2 <<:s :p :o>>>>" );
confirmEmbeddedTripleToken(token);
final Token tokenS = EmbeddedTripleTokenUtils.getSubjectSubToken(token);
final Token tokenP = EmbeddedTripleTokenUtils.getPredicateSubToken(token);
final Token tokenO = EmbeddedTripleTokenUtils.getObjectSubToken(token);
confirmEmbeddedTripleToken(tokenS);
assertEquals(TokenType.PREFIXED_NAME, tokenP.getType());
confirmEmbeddedTripleToken(tokenO);
}
Example #9
| Source File: TokenizerText.java From RDFstarTools with Apache License 2.0 | 4 votes |
|
private void readNumber() {
// One entry, definitely a number.
// Beware of '.' as a (non) decimal.
/*
maybeSign()
digits()
if dot ==> decimal, digits
if e ==> double, maybeSign, digits
else
check not "." for decimal.
*/
boolean isDouble = false;
boolean isDecimal = false;
stringBuilder.setLength(0);
/*
readPossibleSign(stringBuilder);
readDigits may be hex
readDot
readDigits
readExponent.
*/
int x = 0; // Digits before a dot.
int ch = reader.peekChar();
if ( ch == '0' ) {
x++;
reader.readChar();
stringBuilder.append((char)ch);
ch = reader.peekChar();
if ( ch == 'x' || ch == 'X' ) {
reader.readChar();
stringBuilder.append((char)ch);
readHex(reader, stringBuilder);
token.setImage(stringBuilder.toString());
token.setType(TokenType.HEX);
return;
}
} else if ( ch == '-' || ch == '+' ) {
readPossibleSign(stringBuilder);
}
x += readDigits(stringBuilder);
// if ( x == 0 ) {}
ch = reader.peekChar();
if ( ch == CH_DOT ) {
reader.readChar();
stringBuilder.append(CH_DOT);
isDecimal = true; // Includes things that will be doubles.
readDigits(stringBuilder);
}
if ( x == 0 && !isDecimal )
// Possible a tokenizer error - should not have entered readNumber
// in the first place.
error("Unrecognized as number");
if ( exponent(stringBuilder) ) {
isDouble = true;
isDecimal = false;
}
// Final part - "decimal" 123. is an integer 123 and a DOT.
if ( isDecimal ) {
int len = stringBuilder.length();
if ( stringBuilder.charAt(len - 1) == CH_DOT ) {
stringBuilder.setLength(len - 1);
reader.pushbackChar(CH_DOT);
isDecimal = false;
}
}
token.setImage(stringBuilder.toString());
if ( isDouble )
token.setType(TokenType.DOUBLE);
else if ( isDecimal )
token.setType(TokenType.DECIMAL);
else
token.setType(TokenType.INTEGER);
}
Example #10
| Source File: RDFPatchReaderText.java From rdf-delta with Apache License 2.0 | 4 votes |
|
private static void skip(Tokenizer tokenizer, TokenType tokenType ) {
Token tok = tokenizer.next();
if ( ! tok.hasType(tokenType) )
throw exception(tok, "Expected token type: "+tokenType+": got "+tok);
}
