org.eclipse.xtext.parsetree.reconstr.ITokenStream Java Examples

@Override
public TreeConstructionReport serializeSubtree(EObject object, ITokenStream out) throws IOException {
	TreeConstructionReportImpl report = createReport(object);
	AbstractToken root = serialize(object, report);
	Set<ICompositeNode> roots = Sets.newHashSet();
	Map<EObject, AbstractToken> obj2token = Maps.newHashMap();
	collectRootsAndEObjects(root, obj2token, roots);
	//		dump("", root);
	Map<ILeafNode, EObject> comments = commentAssociater.associateCommentsWithSemanticEObjects(object, roots);
	for (ICompositeNode r : roots)
		assignNodesByMatching(obj2token, r, comments);
	WsMergerStream wsout = new WsMergerStream(out);
	//		dump("", root);
	//		System.out.println(EmfFormatter.objToStr(roots.iterator().next(),
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_LENGTH,
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_OFFSET,
	//				ParsetreePackage.Literals.ABSTRACT_NODE__TOTAL_LINE, ParsetreePackage.Literals.ABSTRACT_NODE__PARENT));
	ITextRegion previousLocation = ITextRegion.EMPTY_REGION;
	initStream(root, wsout);
	previousLocation = write(root, wsout, previousLocation);
	wsout.flush();
	report.setPreviousLocation(previousLocation);
	return report;
}
 

@Override
public IFormattedRegion format(final ICompositeNode root, final int offset, final int length) {
  final TokenStringBuffer buf = new TokenStringBuffer();
  final ITokenStream out = offset == 0 ? buf : new FilterStream(buf);
  final ITokenStream formatStream = formatter.createFormatterStream(null, out, false);
  if (!(formatStream instanceof IDelegatingTokenStream)) {
    return super.format(root, offset, length);
  }
  try {
    ITextRegion range;
    if (offset > 0) {
      int noffset = root.getText().lastIndexOf('\n', offset) + 1; // Keep the new line
      int nlength = length + (offset - noffset); // Always start in the beginning of the line
      range = nodeModelStreamer.feedTokenStream(formatStream, root, noffset, nlength);
    } else {
      range = nodeModelStreamer.feedTokenStream(formatStream, root, offset, length);
    }
    return new FormattedRegion(range.getOffset(), range.getLength(), buf.toString());
  } catch (IOException e) {
    // this should never happen since TokenStringBuffer doesn't throw IOEs.
    throw new RuntimeException(e); // NOPMD
  }
}
 

@Override
public IFormattedRegion format(ICompositeNode root, int offset, int length) {
	String indent = getIndentation(root, offset);
	TokenStringBuffer buf = new TokenStringBuffer();
	ITokenStream out = offset == 0 ? buf : new FilterFirstWhitespaceStream(buf);
	ITokenStream fmt;
	if (formatter instanceof IFormatterExtension) {
		EObject semanticElement = NodeModelUtils.findActualSemanticObjectFor(root);
		if (semanticElement != null)
			fmt = ((IFormatterExtension) formatter).createFormatterStream(semanticElement, indent, out, false);
		else {
			// see https://bugs.eclipse.org/bugs/show_bug.cgi?id=380406
			ITextRegion rootRegion = root.getTextRegion();
			return new FormattedRegion(rootRegion.getOffset(), rootRegion.getLength(), root.getText());
		}
	} else
		fmt = formatter.createFormatterStream(indent, out, false);
	try {
		ITextRegion range = nodeModelStreamer.feedTokenStream(fmt, root, offset, length);
		return new FormattedRegion(range.getOffset(), range.getLength(), buf.toString());
	} catch (IOException e) {
		// this should never happen since TokenStringBuffer doesn't throw IOEs.
		throw new RuntimeException(e);
	}
}
 

/**
 * @param endIndex
 *            the index of the last entry to flush, exclusive.
 */
protected void flush(ITokenStream intoStream, int endIndex) throws IOException {
	for (int i = 0; i < endIndex; i++) {
		LineEntry e = this.entries.get(i);
		Pair<AbstractRule, String> spaces = getSpaces(e, i == 0);
		//				if (i == 0 && startWithNL && !e.isBreakable())
		//					throw new IllegalStateException("break for non-breakable item");
		//				else if (i != 0 && e.isBreak())
		//					throw new IllegalStateException("break within line");
		//				if (i == 0 && startWithNL && spaces.getSecond().indexOf('\n') < 0)
		//					throw new IllegalStateException("missing newline before " + e.value);
		// System.out.println("Spaces: '" + sp + "' before '" + e.val
		// + "'");
		if (spaces != null)
			intoStream.writeHidden(spaces.getFirst(), spaces.getSecond());
		if (e.isHidden)
			intoStream.writeHidden(e.grammarElement, e.value);
		else
			intoStream.writeSemantic(e.grammarElement, e.value);
	}
}
 

public ITokenStream createFormatterStream(EObject context, String indent,
		ITokenStream out, boolean preserveWhitespaces) {
	if (context != null && context.eResource() != null
			&& context.eResource().getURI() != null) {
		contextResourceURI = EcoreUtil2
				.getPlatformResourceOrNormalizedURI(context).trimFragment();
	}
	return new DotFormattingConfigBasedStream(out, indent, getConfig(),
			createMatcher(), hiddenTokenHelper, preserveWhitespaces);
}
 

@Override
public ITokenStream createFormatterStream(final EObject context, final String indent, final ITokenStream out,
		final boolean preserveWhitespaces) {
	if (context == null || !context.eResource().getErrors().isEmpty()) {
		// Fixes #2018
		return out;
	}
	return super.createFormatterStream(context, indent, out, preserveWhitespaces);
}
 

@Override
protected void writeSemantic(final ITokenStream out, final ILeafNode node) throws IOException {
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeSemantic(node.getGrammarElement(), node.getText());
}
 

@Override
protected void writeHidden(final ITokenStream out, final ILeafNode node) throws IOException {
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeHidden(node.getGrammarElement(), node.getText());
}
 

@Override
protected void writeSemantic(final ITokenStream out, final ICompositeNode node) throws IOException {
  final String text = tokenUtil.serializeNode(node);
  if (out instanceof ExtendedFormattingConfigBasedStream) {
    ((ExtendedFormattingConfigBasedStream) out).setNode(node);
  }
  out.writeSemantic(node.getGrammarElement(), text);
}
 

public ExtendedFormattingConfigBasedStream(final ITokenStream out, final String indentation, final FormattingConfig cfg, final IElementMatcher<ElementPattern> matcher,
                                      final IHiddenTokenHelper hiddenTokenHelper, final boolean preserveSpaces, final AbstractExtendedFormatter formatter) {
// @Format-On
  super(out, indentation, cfg, matcher, hiddenTokenHelper, preserveSpaces);
  this.storedPreserveSpacesValue = preserveSpaces;
  this.formatter = formatter;
  this.noFormatMemento = new NoFormatLocator(cfg);
}
 

/**
 * Write part of this line to a token stream. The part to be written are
 * all line entries from 0 .. endIndex.
 *
 * @param intoStream
 *          - the token stream into which to write this line.
 * @param endIndex
 *          the index of the last entry to flush, exclusive.
 * @throws IOException
 *           - propagated exception from underlying token stream
 */
@Override
protected void flush(final ITokenStream intoStream, final int endIndex) throws IOException {
  if (firstNotIgnored == 0 && getLinewrapEntry() != null) {
    intoStream.writeHidden(getLinewrapEntry().getGrammarElement(), getLinewrapEntry().getValue());
  }
  for (int i = firstNotIgnored; i < endIndex; i++) {
    final ExtendedLineEntry entry = getEntries().get(i);
    int previousEntryIndex = findPredecessorOrLastEntryIndex(entry);
    final ExtendedLineEntry previousEntry = lineEntries.get(previousEntryIndex);
    final boolean shouldBreakTheLine = entry.isFixedLocatorOpening() && entry.isFixed() && (getAbsoluteLineLength(entry) >= entry.getColumnIndent());
    final boolean isPreviousDifferentThanCurrent = (previousEntryIndex == 0 && !previousEntry.equals(entry));
    final boolean isPreviousColumnAlignedAndNotUtmost = (previousEntry.getColumnIndent() > 0 && !previousEntry.isFixedLocatorClosing()
        && !previousEntry.isFixedLocatorOpening());
    final boolean isCurrentBreakableAndColumnAligned = !entry.isNoBreak() && entry.isFixedLocatorOpening() && entry.isFixedLocatorClosing();

    if (shouldBreakTheLine || (isCurrentBreakableAndColumnAligned && isPreviousDifferentThanCurrent && isPreviousColumnAlignedAndNotUtmost)) { // nobreak
      intoStream.writeHidden(entry.getGrammarElement(), "\n" + SpaceEntry.createPadding(entry.getColumnIndent())); //$NON-NLS-1$
    } else {
      intoStream.writeHidden(entry.getGrammarElement(), entry.getLeadingWS());
    }
    if (entry.isHidden()) {
      intoStream.writeHidden(entry.getGrammarElement(), entry.getValue());
    } else {
      intoStream.writeSemantic(entry.getGrammarElement(), entry.getValue());
    }
  }
  firstNotIgnored = 0; // Reset
}
 

protected void serializeToTokenBuffer(String model, ITokenStream out) throws Exception {
	EObject semanticObject = get(ParseHelper.class).parse(model);
	ISerializationDiagnostic.Acceptor errors = ISerializationDiagnostic.EXCEPTION_THROWING_ACCEPTOR;
	ISemanticSequencer semantic = get(ISemanticSequencer.class);
	ISyntacticSequencer syntactic = get(ISyntacticSequencer.class);
	IHiddenTokenSequencer hidden = get(IHiddenTokenSequencer.class);
	TokenStreamSequenceAdapter tokenstream = new TokenStreamSequenceAdapter(out, getGrammarAccess().getGrammar(), errors);
	semantic.init((ISemanticSequenceAcceptor) syntactic, errors);
	ISerializationContext context = new SerializationContext.RuleContext(null, (ParserRule) get(IGrammarAccess.class).getGrammar().getRules().get(0));
	syntactic.init(context, semanticObject, (ISyntacticSequenceAcceptor) hidden, errors);
	hidden.init(context, semanticObject, tokenstream, errors);
	tokenstream.init(context);
	semantic.createSequence(context, semanticObject);
}
 

/**
 * @since 2.3
 */
@Override
public ITokenStream createFormatterStream(EObject context, String indent, ITokenStream out, boolean preserveWhitespaces) {
	if(context != null && context.eResource() != null && context.eResource().getURI() != null) {
		contextResourceURI = EcoreUtil2.getPlatformResourceOrNormalizedURI(context).trimFragment();
	} else if (context != null && context.eResource() == null) {
		log.error("Model has no XtextResource. This is likely to cause follow-up errors");
	}
	return new FormattingConfigBasedStream(out, indent, getConfig(), createMatcher(), hiddenTokenHelper,
			preserveWhitespaces);
}
 

public FormattingConfigBasedStream(ITokenStream out, String initialIndentation, FormattingConfig cfg,
		IElementMatcher<ElementPattern> matcher, IHiddenTokenHelper hiddenTokenHelper, boolean preserveSpaces) {
	super(out);
	this.cfg = cfg;
	this.matcher = matcher;
	this.hiddenTokenHelper = hiddenTokenHelper;
	this.preserveSpaces = preserveSpaces;
	this.indentationPrefix = initialIndentation == null ? "" : initialIndentation;
}
 

protected void writeSemantic(ITokenStream out, ICompositeNode node) throws IOException {
	AbstractRule rule = tokenUtil.getTokenRule(node);
	String text = tokenUtil.serializeNode(node);
	try {
		// there and back again - the value converter is used to obtain a canonical representation of the value
		text = getFormattedDatatypeValue(node, rule, text);
	} catch(ValueConverterException e) {
		// which may fail - fall back to plain text value
	}
	out.writeSemantic(node.getGrammarElement(), text);
}
 

/**
 * Instantiates a formatting configuration based stream
 */
DotFormattingConfigBasedStream(ITokenStream out,
		String initialIndentation, FormattingConfig cfg,
		IElementMatcher<ElementPattern> matcher,
		IHiddenTokenHelper hiddenTokenHelper, boolean preserveSpaces) {
	super(out, initialIndentation, cfg, matcher, hiddenTokenHelper,
			preserveSpaces);
}
 

@Override
public ITextRegion feedTokenStream(ITokenStream out, ICompositeNode in, int offset, int length) throws IOException {
	List<INode> nodes = getLeafs(in, offset, offset + length);
	if (nodes.isEmpty())
		return new TextRegion(in.getOffset(), 0);
	if (out instanceof ITokenStreamExtension)
		((ITokenStreamExtension) out).init(findRootRuleForRegion(nodes.get(0)));
	boolean lastIsTokenOrComment = false;
	for (INode node : nodes) {
		boolean currentIsTokenOrComment = tokenUtil.isCommentNode(node) || tokenUtil.isToken(node);
		if (lastIsTokenOrComment && currentIsTokenOrComment)
			writeHiddenEmpty(out);
		lastIsTokenOrComment = currentIsTokenOrComment;
		if (node instanceof ILeafNode) {
			ILeafNode leaf = (ILeafNode) node;
			if (leaf.isHidden())
				writeHidden(out, leaf);
			else
				writeSemantic(out, leaf);
		} else if (node instanceof ICompositeNode)
			writeSemantic(out, (ICompositeNode) node);
	}
	out.flush();
	int rStart = nodes.get(0).getOffset();
	int rLength = nodes.get(nodes.size() - 1).getEndOffset() - rStart;
	return new TextRegion(rStart, rLength);
}
 

@Override
protected void writeSemantic(ITokenStream out, ICompositeNode node)
		throws IOException {
	AbstractRule rule = tokenUtil.getTokenRule(node);
	Object val = valueConverter.toValue(tokenUtil.serializeNode(node),
			rule.getName(), node);

	if (val instanceof ID && ((ID) val).getType() == ID.Type.HTML_STRING) {
		writeHTMLStringSemantic(rule,
				(DotFormatter.DotFormattingConfigBasedStream) out, node);
	} else {
		super.writeSemantic(out, node);
	}
}
 

private void writeHTMLStringSemantic(AbstractRule rule,
		DotFormattingConfigBasedStream out, ICompositeNode node)
		throws IOException {
	Injector htmlLabelInjector = new DotHtmlLabelStandaloneSetup()
			.createInjectorAndDoEMFRegistration();
	IFormatter dotHtmlLabelFormatter = htmlLabelInjector
			.getInstance(IFormatter.class);
	ITokenStream htmlLabelOut = new TokenStringBuffer();
	// TODO: calculate initial indentation properly
	ITokenStream fmt = dotHtmlLabelFormatter.createFormatterStream("\t\t",
			htmlLabelOut, false);
	INodeModelStreamer dothtmlLabelNodeModelStreamer = htmlLabelInjector
			.getInstance(INodeModelStreamer.class);

	IParser dotHtmlLabelParser = htmlLabelInjector
			.getInstance(IParser.class);
	// cut off the leading and the trailing white spaces
	String trimmedNodeText = node.getText().trim();
	String htmlLabelText = trimmedNodeText.substring(1,
			trimmedNodeText.length() - 1);
	IParseResult parseResult = dotHtmlLabelParser
			.parse(new StringReader(htmlLabelText));
	ICompositeNode htmlLabelRootNode = parseResult.getRootNode();

	dothtmlLabelNodeModelStreamer.feedTokenStream(fmt, htmlLabelRootNode, 0,
			htmlLabelText.length());
	out.writeSemantic(null, "<");
	out.addNewLine();
	out.addLineEntry(node.getGrammarElement(), htmlLabelOut.toString(),
			false);
	out.addNewLine();
	out.writeSemantic(null, ">");
}
 

protected void serialize(EObject obj, ITokenStream tokenStream, SaveOptions options) throws IOException {
	ISerializationDiagnostic.Acceptor errors = ISerializationDiagnostic.EXCEPTION_THROWING_ACCEPTOR;
	ITokenStream formatterTokenStream;
	if (formatter instanceof IFormatterExtension)
		formatterTokenStream = ((IFormatterExtension) formatter).createFormatterStream(obj, null, tokenStream,
				!options.isFormatting());
	else
		formatterTokenStream = formatter.createFormatterStream(null, tokenStream, !options.isFormatting());
	ISerializationContext context = getIContext(obj);
	ISequenceAcceptor acceptor = new TokenStreamSequenceAdapter(formatterTokenStream, grammar.getGrammar(), errors);
	serialize(context, obj, acceptor, errors);
	formatterTokenStream.flush();
}
 

protected void writeSemantic(ITokenStream out, ILeafNode node) throws IOException {
	out.writeSemantic(node.getGrammarElement(), node.getText());
}
 

@Override
public void serialize(EObject obj, ITokenStream tokenStream, SaveOptions options) throws IOException {
	super.serialize(obj, tokenStream, options);
}
 

public ITokenStream getOutStream() {
  return out;
}
 

/** {@inheritDoc} */
@Override
public void setDelegateStream(final ITokenStream delegate) {
  this.out = delegate;
}
 

/** {@inheritDoc} */
@Override
public ITokenStream getDelegateStream() {
  return this.out;
}
 

public WsMergerStream(ITokenStream out) {
	super();
	this.out = out;
}
 

public TokenStreamSequenceAdapter(ITokenStream out, Grammar grammar, ISerializationDiagnostic.Acceptor errorAcceptor) {
	this.out = out;
	this.grammar = grammar;
	this.errorAcceptor = errorAcceptor;
}
 

protected FilterStream(final ITokenStream out) {
  super(out);
  enabled = false;
}
 

@Override
public ITokenStream createFormatterStream(final EObject context, final String indent, final ITokenStream out, final boolean preserveWhitespaces) {
  // TODO set context (see superclass)
  return createFormatterStream(indent, out, preserveWhitespaces);
}
 

/**
 * {@inheritDoc}
 */
@Override
public ITokenStream createFormatterStream(final String initalIndentation, final ITokenStream outputStream, final boolean preserveWhitespaces) {
  return new ExtendedFormattingConfigBasedStream(outputStream, initalIndentation, getConfig(), createMatcher(), getHiddenTokenHelper(), preserveWhitespaces, this);
}