Java Code Examples for org.eclipse.xtext.parsetree.reconstr.ITokenStream

The following examples show how to use org.eclipse.xtext.parsetree.reconstr.ITokenStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public TreeConstructionReport serializeSubtree(EObject object, ITokenStream out) throws IOException {
	TreeConstructionReportImpl report = createReport(object);
	AbstractToken root = serialize(object, report);
	Set<ICompositeNode> roots = Sets.newHashSet();
	Map<EObject, AbstractToken> obj2token = Maps.newHashMap();
	collectRootsAndEObjects(root, obj2token, roots);
	//		dump("", root);
	Map<ILeafNode, EObject> comments = commentAssociater.associateCommentsWithSemanticEObjects(object, roots);
	for (ICompositeNode r : roots)
		assignNodesByMatching(obj2token, r, comments);
	WsMergerStream wsout = new WsMergerStream(out);
	//		dump("", root);
	//		System.out.println(EmfFormatter.objToStr(roots.iterator().next(),
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_LENGTH,
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_OFFSET,
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_LINE, ParsetreePackage.Literals.ABSTRACT_NODE__PARENT));
	ITextRegion previousLocation = ITextRegion.EMPTY_REGION;
	initStream(root, wsout);
	previousLocation = write(root, wsout, previousLocation);
	wsout.flush();
	report.setPreviousLocation(previousLocation);
	return report;
}
 
Example 2
@Override
public IFormattedRegion format(ICompositeNode root, int offset, int length) {
	String indent = getIndentation(root, offset);
	TokenStringBuffer buf = new TokenStringBuffer();
	ITokenStream out = offset == 0 ? buf : new FilterFirstWhitespaceStream(buf);
	ITokenStream fmt;
	if (formatter instanceof IFormatterExtension) {
		EObject semanticElement = NodeModelUtils.findActualSemanticObjectFor(root);
		if (semanticElement != null)
			fmt = ((IFormatterExtension) formatter).createFormatterStream(semanticElement, indent, out, false);
		else {
			// see https://bugs.eclipse.org/bugs/show_bug.cgi?id=380406
			ITextRegion rootRegion = root.getTextRegion();
			return new FormattedRegion(rootRegion.getOffset(), rootRegion.getLength(), root.getText());
		}
	} else
		fmt = formatter.createFormatterStream(indent, out, false);
	try {
		ITextRegion range = nodeModelStreamer.feedTokenStream(fmt, root, offset, length);
		return new FormattedRegion(range.getOffset(), range.getLength(), buf.toString());
	} catch (IOException e) {
		// this should never happen since TokenStringBuffer doesn't throw IOEs.
		throw new RuntimeException(e);
	}
}
 
Example 3
/**
 * @param endIndex
 *            the index of the last entry to flush, exclusive.
 */
protected void flush(ITokenStream intoStream, int endIndex) throws IOException {
	for (int i = 0; i < endIndex; i++) {
		LineEntry e = this.entries.get(i);
		Pair<AbstractRule, String> spaces = getSpaces(e, i == 0);
		//				if (i == 0 && startWithNL && !e.isBreakable())
		//					throw new IllegalStateException("break for non-breakable item");
		//				else if (i != 0 && e.isBreak())
		//					throw new IllegalStateException("break within line");
		//				if (i == 0 && startWithNL && spaces.getSecond().indexOf('\n') < 0)
		//					throw new IllegalStateException("missing newline before " + e.value);
		// System.out.println("Spaces: '" + sp + "' before '" + e.val
		// + "'");
		if (spaces != null)
			intoStream.writeHidden(spaces.getFirst(), spaces.getSecond());
		if (e.isHidden)
			intoStream.writeHidden(e.grammarElement, e.value);
		else
			intoStream.writeSemantic(e.grammarElement, e.value);
	}
}
 
Example 4
@Override
public IFormattedRegion format(final ICompositeNode root, final int offset, final int length) {
  final TokenStringBuffer buf = new TokenStringBuffer();
  final ITokenStream out = offset == 0 ? buf : new FilterStream(buf);
  final ITokenStream formatStream = formatter.createFormatterStream(null, out, false);
  if (!(formatStream instanceof IDelegatingTokenStream)) {
    return super.format(root, offset, length);
  }
  try {
    ITextRegion range;
    if (offset > 0) {
      int noffset = root.getText().lastIndexOf('\n', offset) + 1; // Keep the new line
      int nlength = length + (offset - noffset); // Always start in the beginning of the line
      range = nodeModelStreamer.feedTokenStream(formatStream, root, noffset, nlength);
    } else {
      range = nodeModelStreamer.feedTokenStream(formatStream, root, offset, length);
    }
    return new FormattedRegion(range.getOffset(), range.getLength(), buf.toString());
  } catch (IOException e) {
    // this should never happen since TokenStringBuffer doesn't throw IOEs.
    throw new RuntimeException(e); // NOPMD
  }
}
 
Example 5
Source Project: gef   Source File: DotFormatter.java    License: Eclipse Public License 2.0 5 votes vote down vote up
public ITokenStream createFormatterStream(EObject context, String indent,
		ITokenStream out, boolean preserveWhitespaces) {
	if (context != null && context.eResource() != null
			&& context.eResource().getURI() != null) {
		contextResourceURI = EcoreUtil2
				.getPlatformResourceOrNormalizedURI(context).trimFragment();
	}
	return new DotFormattingConfigBasedStream(out, indent, getConfig(),
			createMatcher(), hiddenTokenHelper, preserveWhitespaces);
}
 
Example 6
Source Project: gef   Source File: DotFormatter.java    License: Eclipse Public License 2.0 5 votes vote down vote up
/**
 * Instantiates a formatting configuration based stream
 */
DotFormattingConfigBasedStream(ITokenStream out,
		String initialIndentation, FormattingConfig cfg,
		IElementMatcher<ElementPattern> matcher,
		IHiddenTokenHelper hiddenTokenHelper, boolean preserveSpaces) {
	super(out, initialIndentation, cfg, matcher, hiddenTokenHelper,
			preserveSpaces);
}
 
Example 7
Source Project: gef   Source File: DotNodeModelStreamer.java    License: Eclipse Public License 2.0 5 votes vote down vote up
@Override
protected void writeSemantic(ITokenStream out, ICompositeNode node)
		throws IOException {
	AbstractRule rule = tokenUtil.getTokenRule(node);
	Object val = valueConverter.toValue(tokenUtil.serializeNode(node),
			rule.getName(), node);

	if (val instanceof ID && ((ID) val).getType() == ID.Type.HTML_STRING) {
		writeHTMLStringSemantic(rule,
				(DotFormatter.DotFormattingConfigBasedStream) out, node);
	} else {
		super.writeSemantic(out, node);
	}
}
 
Example 8
Source Project: gef   Source File: DotNodeModelStreamer.java    License: Eclipse Public License 2.0 5 votes vote down vote up
private void writeHTMLStringSemantic(AbstractRule rule,
		DotFormattingConfigBasedStream out, ICompositeNode node)
		throws IOException {
	Injector htmlLabelInjector = new DotHtmlLabelStandaloneSetup()
			.createInjectorAndDoEMFRegistration();
	IFormatter dotHtmlLabelFormatter = htmlLabelInjector
			.getInstance(IFormatter.class);
	ITokenStream htmlLabelOut = new TokenStringBuffer();
	// TODO: calculate initial indentation properly
	ITokenStream fmt = dotHtmlLabelFormatter.createFormatterStream("\t\t",
			htmlLabelOut, false);
	INodeModelStreamer dothtmlLabelNodeModelStreamer = htmlLabelInjector
			.getInstance(INodeModelStreamer.class);

	IParser dotHtmlLabelParser = htmlLabelInjector
			.getInstance(IParser.class);
	// cut off the leading and the trailing white spaces
	String trimmedNodeText = node.getText().trim();
	String htmlLabelText = trimmedNodeText.substring(1,
			trimmedNodeText.length() - 1);
	IParseResult parseResult = dotHtmlLabelParser
			.parse(new StringReader(htmlLabelText));
	ICompositeNode htmlLabelRootNode = parseResult.getRootNode();

	dothtmlLabelNodeModelStreamer.feedTokenStream(fmt, htmlLabelRootNode, 0,
			htmlLabelText.length());
	out.writeSemantic(null, "<");
	out.addNewLine();
	out.addLineEntry(node.getGrammarElement(), htmlLabelOut.toString(),
			false);
	out.addNewLine();
	out.writeSemantic(null, ">");
}
 
Example 9
Source Project: xtext-core   Source File: Serializer.java    License: Eclipse Public License 2.0 5 votes vote down vote up
protected void serialize(EObject obj, ITokenStream tokenStream, SaveOptions options) throws IOException {
	ISerializationDiagnostic.Acceptor errors = ISerializationDiagnostic.EXCEPTION_THROWING_ACCEPTOR;
	ITokenStream formatterTokenStream;
	if (formatter instanceof IFormatterExtension)
		formatterTokenStream = ((IFormatterExtension) formatter).createFormatterStream(obj, null, tokenStream,
				!options.isFormatting());
	else
		formatterTokenStream = formatter.createFormatterStream(null, tokenStream, !options.isFormatting());
	ISerializationContext context = getIContext(obj);
	ISequenceAcceptor acceptor = new TokenStreamSequenceAdapter(formatterTokenStream, grammar.getGrammar(), errors);
	serialize(context, obj, acceptor, errors);
	formatterTokenStream.flush();
}
 
Example 10
Source Project: xtext-core   Source File: NodeModelStreamer.java    License: Eclipse Public License 2.0 5 votes vote down vote up
@Override
public ITextRegion feedTokenStream(ITokenStream out, ICompositeNode in, int offset, int length) throws IOException {
	List<INode> nodes = getLeafs(in, offset, offset + length);
	if (nodes.isEmpty())
		return new TextRegion(in.getOffset(), 0);
	if (out instanceof ITokenStreamExtension)
		((ITokenStreamExtension) out).init(findRootRuleForRegion(nodes.get(0)));
	boolean lastIsTokenOrComment = false;
	for (INode node : nodes) {
		boolean currentIsTokenOrComment = tokenUtil.isCommentNode(node) || tokenUtil.isToken(node);
		if (lastIsTokenOrComment && currentIsTokenOrComment)
			writeHiddenEmpty(out);
		lastIsTokenOrComment = currentIsTokenOrComment;
		if (node instanceof ILeafNode) {
			ILeafNode leaf = (ILeafNode) node;
			if (leaf.isHidden())
				writeHidden(out, leaf);
			else
				writeSemantic(out, leaf);
		} else if (node instanceof ICompositeNode)
			writeSemantic(out, (ICompositeNode) node);
	}
	out.flush();
	int rStart = nodes.get(0).getOffset();
	int rLength = nodes.get(nodes.size() - 1).getEndOffset() - rStart;
	return new TextRegion(rStart, rLength);
}
 
Example 11
Source Project: xtext-core   Source File: NodeModelStreamer.java    License: Eclipse Public License 2.0 5 votes vote down vote up
protected void writeSemantic(ITokenStream out, ICompositeNode node) throws IOException {
	AbstractRule rule = tokenUtil.getTokenRule(node);
	String text = tokenUtil.serializeNode(node);
	try {
		// there and back again - the value converter is used to obtain a canonical representation of the value
		text = getFormattedDatatypeValue(node, rule, text);
	} catch(ValueConverterException e) {
		// which may fail - fall back to plain text value
	}
	out.writeSemantic(node.getGrammarElement(), text);
}
 
Example 12
public FormattingConfigBasedStream(ITokenStream out, String initialIndentation, FormattingConfig cfg,
		IElementMatcher<ElementPattern> matcher, IHiddenTokenHelper hiddenTokenHelper, boolean preserveSpaces) {
	super(out);
	this.cfg = cfg;
	this.matcher = matcher;
	this.hiddenTokenHelper = hiddenTokenHelper;
	this.preserveSpaces = preserveSpaces;
	this.indentationPrefix = initialIndentation == null ? "" : initialIndentation;
}
 
Example 13
/**
 * @since 2.3
 */
@Override
public ITokenStream createFormatterStream(EObject context, String indent, ITokenStream out, boolean preserveWhitespaces) {
	if(context != null && context.eResource() != null && context.eResource().getURI() != null) {
		contextResourceURI = EcoreUtil2.getPlatformResourceOrNormalizedURI(context).trimFragment();
	} else if (context != null && context.eResource() == null) {
		log.error("Model has no XtextResource. This is likely to cause follow-up errors");
	}
	return new FormattingConfigBasedStream(out, indent, getConfig(), createMatcher(), hiddenTokenHelper,
			preserveWhitespaces);
}
 
Example 14
Source Project: xtext-core   Source File: FormatterTest.java    License: Eclipse Public License 2.0 5 votes vote down vote up
protected void serializeToTokenBuffer(String model, ITokenStream out) throws Exception {
	EObject semanticObject = get(ParseHelper.class).parse(model);
	ISerializationDiagnostic.Acceptor errors = ISerializationDiagnostic.EXCEPTION_THROWING_ACCEPTOR;
	ISemanticSequencer semantic = get(ISemanticSequencer.class);
	ISyntacticSequencer syntactic = get(ISyntacticSequencer.class);
	IHiddenTokenSequencer hidden = get(IHiddenTokenSequencer.class);
	TokenStreamSequenceAdapter tokenstream = new TokenStreamSequenceAdapter(out, getGrammarAccess().getGrammar(), errors);
	semantic.init((ISemanticSequenceAcceptor) syntactic, errors);
	ISerializationContext context = new SerializationContext.RuleContext(null, (ParserRule) get(IGrammarAccess.class).getGrammar().getRules().get(0));
	syntactic.init(context, semanticObject, (ISyntacticSequenceAcceptor) hidden, errors);
	hidden.init(context, semanticObject, tokenstream, errors);
	tokenstream.init(context);
	semantic.createSequence(context, semanticObject);
}
 
Example 15
Source Project: dsl-devkit   Source File: ExtendedLine.java    License: Eclipse Public License 1.0 5 votes vote down vote up
/**
 * Write part of this line to a token stream. The part to be written are
 * all line entries from 0 .. endIndex.
 *
 * @param intoStream
 *          - the token stream into which to write this line.
 * @param endIndex
 *          the index of the last entry to flush, exclusive.
 * @throws IOException
 *           - propagated exception from underlying token stream
 */
@Override
protected void flush(final ITokenStream intoStream, final int endIndex) throws IOException {
  if (firstNotIgnored == 0 && getLinewrapEntry() != null) {
    intoStream.writeHidden(getLinewrapEntry().getGrammarElement(), getLinewrapEntry().getValue());
  }
  for (int i = firstNotIgnored; i < endIndex; i++) {
    final ExtendedLineEntry entry = getEntries().get(i);
    int previousEntryIndex = findPredecessorOrLastEntryIndex(entry);
    final ExtendedLineEntry previousEntry = lineEntries.get(previousEntryIndex);
    final boolean shouldBreakTheLine = entry.isFixedLocatorOpening() && entry.isFixed() && (getAbsoluteLineLength(entry) >= entry.getColumnIndent());
    final boolean isPreviousDifferentThanCurrent = (previousEntryIndex == 0 && !previousEntry.equals(entry));
    final boolean isPreviousColumnAlignedAndNotUtmost = (previousEntry.getColumnIndent() > 0 && !previousEntry.isFixedLocatorClosing()
        && !previousEntry.isFixedLocatorOpening());
    final boolean isCurrentBreakableAndColumnAligned = !entry.isNoBreak() && entry.isFixedLocatorOpening() && entry.isFixedLocatorClosing();

    if (shouldBreakTheLine || (isCurrentBreakableAndColumnAligned && isPreviousDifferentThanCurrent && isPreviousColumnAlignedAndNotUtmost)) { // nobreak
      intoStream.writeHidden(entry.getGrammarElement(), "\n" + SpaceEntry.createPadding(entry.getColumnIndent())); //$NON-NLS-1$
    } else {
      intoStream.writeHidden(entry.getGrammarElement(), entry.getLeadingWS());
    }
    if (entry.isHidden()) {
      intoStream.writeHidden(entry.getGrammarElement(), entry.getValue());
    } else {
      intoStream.writeSemantic(entry.getGrammarElement(), entry.getValue());
    }
  }
  firstNotIgnored = 0; // Reset
}
 
Example 16
public ExtendedFormattingConfigBasedStream(final ITokenStream out, final String indentation, final FormattingConfig cfg, final IElementMatcher<ElementPattern> matcher,
                                      final IHiddenTokenHelper hiddenTokenHelper, final boolean preserveSpaces, final AbstractExtendedFormatter formatter) {
// @Format-On
  super(out, indentation, cfg, matcher, hiddenTokenHelper, preserveSpaces);
  this.storedPreserveSpacesValue = preserveSpaces;
  this.formatter = formatter;
  this.noFormatMemento = new NoFormatLocator(cfg);
}
 
Example 17
@Override
protected void writeSemantic(final ITokenStream out, final ICompositeNode node) throws IOException {
  final String text = tokenUtil.serializeNode(node);
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeSemantic(node.getGrammarElement(), text);
}
 
Example 18
@Override
protected void writeHidden(final ITokenStream out, final ILeafNode node) throws IOException {
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeHidden(node.getGrammarElement(), node.getText());
}
 
Example 19
@Override
protected void writeSemantic(final ITokenStream out, final ILeafNode node) throws IOException {
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeSemantic(node.getGrammarElement(), node.getText());
}
 
Example 20
Source Project: gama   Source File: GamlFormatter.java    License: GNU General Public License v3.0 5 votes vote down vote up
@Override
public ITokenStream createFormatterStream(final EObject context, final String indent, final ITokenStream out,
		final boolean preserveWhitespaces) {
	if (context == null || !context.eResource().getErrors().isEmpty()) {
		// Fixes #2018
		return out;
	}
	return super.createFormatterStream(context, indent, out, preserveWhitespaces);
}
 
Example 21
Source Project: n4js   Source File: AccessibleSerializer.java    License: Eclipse Public License 1.0 4 votes vote down vote up
@Override
public void serialize(EObject obj, ITokenStream tokenStream, SaveOptions options) throws IOException {
	super.serialize(obj, tokenStream, options);
}
 
Example 22
public WsMergerStream(ITokenStream out) {
	super();
	this.out = out;
}
 
Example 23
public TokenStreamSequenceAdapter(ITokenStream out, Grammar grammar, ISerializationDiagnostic.Acceptor errorAcceptor) {
	this.out = out;
	this.grammar = grammar;
	this.errorAcceptor = errorAcceptor;
}
 
Example 24
protected FilterFirstWhitespaceStream(ITokenStream out) {
	super(out);
}
 
Example 25
Source Project: xtext-core   Source File: NodeModelStreamer.java    License: Eclipse Public License 2.0 4 votes vote down vote up
protected void writeHidden(ITokenStream out, ILeafNode node) throws IOException {
	out.writeHidden(node.getGrammarElement(), node.getText());
}
 
Example 26
Source Project: xtext-core   Source File: NodeModelStreamer.java    License: Eclipse Public License 2.0 4 votes vote down vote up
protected void writeHiddenEmpty(ITokenStream out) throws IOException {
	out.writeHidden(hiddenTokenHelper.getWhitespaceRuleFor(null, ""), "");
}
 
Example 27
Source Project: xtext-core   Source File: NodeModelStreamer.java    License: Eclipse Public License 2.0 4 votes vote down vote up
protected void writeSemantic(ITokenStream out, ILeafNode node) throws IOException {
	out.writeSemantic(node.getGrammarElement(), node.getText());
}
 
Example 28
Source Project: xtext-core   Source File: NullFormatter.java    License: Eclipse Public License 2.0 4 votes vote down vote up
@Override
public ITokenStream createFormatterStream(String initalIndentation,
		ITokenStream outputStream, boolean preserveWhitespaces) {
	return outputStream;
}
 
Example 29
Source Project: xtext-core   Source File: OneWhitespaceFormatter.java    License: Eclipse Public License 2.0 4 votes vote down vote up
public OneWhitespaceFormatterStream(ITokenStream out, boolean preserve) {
	super(out);
	this.hasStarted = false;
	this.wsWritten = false;
	this.preserveWS = preserve;
}
 
Example 30
Source Project: xtext-core   Source File: OneWhitespaceFormatter.java    License: Eclipse Public License 2.0 4 votes vote down vote up
@Override
public ITokenStream createFormatterStream(String initalIndentation,
		ITokenStream outputStream, boolean preserveWhitespaces) {
	return new OneWhitespaceFormatterStream(outputStream,
			preserveWhitespaces);
}