/* * Copyright 2018 Google LLC. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.google.cloud.tools.opensource.dashboard; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.cloud.tools.opensource.classpath.ClassFile; import com.google.cloud.tools.opensource.classpath.ClassPathBuilder; import com.google.cloud.tools.opensource.classpath.ClassPathEntry; import com.google.cloud.tools.opensource.classpath.ClassPathResult; import com.google.cloud.tools.opensource.classpath.LinkageChecker; import com.google.cloud.tools.opensource.classpath.SymbolProblem; import com.google.cloud.tools.opensource.dependencies.Artifacts; import com.google.cloud.tools.opensource.dependencies.Bom; import com.google.cloud.tools.opensource.dependencies.DependencyGraph; import com.google.cloud.tools.opensource.dependencies.DependencyGraphBuilder; import com.google.cloud.tools.opensource.dependencies.DependencyPath; import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; import com.google.cloud.tools.opensource.dependencies.RepositoryUtility; import com.google.cloud.tools.opensource.dependencies.Update; import com.google.cloud.tools.opensource.dependencies.VersionComparator; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimaps; import com.google.common.collect.Sets; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.DefaultObjectWrapperBuilder; import freemarker.template.Template; import freemarker.template.TemplateException; import freemarker.template.TemplateHashModel; import freemarker.template.Version; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; import org.apache.commons.cli.ParseException; import org.eclipse.aether.RepositoryException; import org.eclipse.aether.RepositorySystem; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.graph.Dependency; public class DashboardMain { public static final String TEST_NAME_LINKAGE_CHECK = "Linkage Errors"; public static final String TEST_NAME_UPPER_BOUND = "Upper Bounds"; public static final String TEST_NAME_GLOBAL_UPPER_BOUND = "Global Upper Bounds"; public static final String TEST_NAME_DEPENDENCY_CONVERGENCE = "Dependency Convergence"; private static final Configuration freemarkerConfiguration = configureFreemarker(); private static final DependencyGraphBuilder dependencyGraphBuilder = new DependencyGraphBuilder(); private static final ClassPathBuilder classPathBuilder = new ClassPathBuilder(dependencyGraphBuilder); /** * Generates a code hygiene dashboard for a BOM. This tool takes a path to pom.xml of the BOM as * an argument or Maven coordinates to a BOM. * * <p>Generated dashboard is at {@code target/$groupId/$artifactId/$version/index.html}, where * each value is from BOM coordinates except {@code $version} is "snapshot" if the BOM has * snapshot version. */ public static void main(String[] arguments) throws IOException, TemplateException, RepositoryException, URISyntaxException, ParseException, MavenRepositoryException { DashboardArguments dashboardArguments = DashboardArguments.readCommandLine(arguments); if (dashboardArguments.hasVersionlessCoordinates()) { generateAllVersions(dashboardArguments.getVersionlessCoordinates()); } else if (dashboardArguments.hasFile()) { generate(dashboardArguments.getBomFile()); } else { generate(dashboardArguments.getBomCoordinates()); } } private static void generateAllVersions(String versionlessCoordinates) throws IOException, TemplateException, RepositoryException, URISyntaxException, MavenRepositoryException { List<String> elements = Splitter.on(':').splitToList(versionlessCoordinates); checkArgument( elements.size() == 2, "The versionless coordinates should have one colon character: " + versionlessCoordinates); String groupId = elements.get(0); String artifactId = elements.get(1); RepositorySystem repositorySystem = RepositoryUtility.newRepositorySystem(); ImmutableList<String> versions = RepositoryUtility.findVersions(repositorySystem, groupId, artifactId); for (String version : versions) { generate(String.format("%s:%s:%s", groupId, artifactId, version)); } generateVersionIndex(groupId, artifactId, versions); } @VisibleForTesting static Path generateVersionIndex(String groupId, String artifactId, List<String> versions) throws IOException, TemplateException, URISyntaxException { Path directory = outputDirectory(groupId, artifactId, "snapshot").getParent(); directory.toFile().mkdirs(); Path page = directory.resolve("index.html"); Map<String, Object> templateData = new HashMap<>(); templateData.put("versions", versions); templateData.put("groupId", groupId); templateData.put("artifactId", artifactId); File dashboardFile = page.toFile(); try (Writer out = new OutputStreamWriter(new FileOutputStream(dashboardFile), StandardCharsets.UTF_8)) { Template dashboard = freemarkerConfiguration.getTemplate("/templates/version_index.ftl"); dashboard.process(templateData, out); } copyResource(directory, "css/dashboard.css"); return page; } @VisibleForTesting static Path generate(String bomCoordinates) throws IOException, TemplateException, RepositoryException, URISyntaxException { Path output = generate(Bom.readBom(bomCoordinates)); System.out.println("Wrote dashboard for " + bomCoordinates + " to " + output); return output; } @VisibleForTesting static Path generate(Path bomFile) throws IOException, TemplateException, URISyntaxException, MavenRepositoryException { checkArgument(Files.isRegularFile(bomFile), "The input BOM %s is not a regular file", bomFile); checkArgument(Files.isReadable(bomFile), "The input BOM %s is not readable", bomFile); Path output = generate(Bom.readBom(bomFile)); System.out.println("Wrote dashboard for " + bomFile + " to " + output); return output; } private static Path generate(Bom bom) throws IOException, TemplateException, URISyntaxException { ImmutableList<Artifact> managedDependencies = bom.getManagedDependencies(); ClassPathResult classPathResult = classPathBuilder.resolve(managedDependencies); ImmutableList<ClassPathEntry> classpath = classPathResult.getClassPath(); LinkageChecker linkageChecker = LinkageChecker.create(classpath); ImmutableSetMultimap<SymbolProblem, ClassFile> symbolProblems = linkageChecker.findSymbolProblems(); ArtifactCache cache = loadArtifactInfo(managedDependencies); Path output = generateHtml(bom, cache, classPathResult, symbolProblems); return output; } private static Path outputDirectory(String groupId, String artifactId, String version) { String versionPathElement = version.contains("-SNAPSHOT") ? "snapshot" : version; return Paths.get("target", groupId, artifactId, versionPathElement); } private static Path generateHtml( Bom bom, ArtifactCache cache, ClassPathResult classPathResult, ImmutableSetMultimap<SymbolProblem, ClassFile> symbolProblems) throws IOException, TemplateException, URISyntaxException { Artifact bomArtifact = new DefaultArtifact(bom.getCoordinates()); Path relativePath = outputDirectory( bomArtifact.getGroupId(), bomArtifact.getArtifactId(), bomArtifact.getVersion()); Path output = Files.createDirectories(relativePath); copyResource(output, "css/dashboard.css"); copyResource(output, "js/dashboard.js"); ImmutableMap<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> symbolProblemTable = indexByJar(symbolProblems); List<ArtifactResults> table = generateReports( freemarkerConfiguration, output, cache, symbolProblemTable, classPathResult, bom); generateDashboard( freemarkerConfiguration, output, table, cache.getGlobalDependencies(), symbolProblemTable, classPathResult, bom); return output; } private static void copyResource(Path output, String resourceName) throws IOException, URISyntaxException { ClassLoader classLoader = DashboardMain.class.getClassLoader(); Path input = Paths.get(classLoader.getResource(resourceName).toURI()).toAbsolutePath(); Path copy = output.resolve(input.getFileName()); if (!Files.exists(copy)) { Files.copy(input, copy); } } @VisibleForTesting static Configuration configureFreemarker() { Configuration configuration = new Configuration(new Version("2.3.28")); configuration.setDefaultEncoding("UTF-8"); configuration.setClassForTemplateLoading(DashboardMain.class, "/"); return configuration; } @VisibleForTesting static List<ArtifactResults> generateReports( Configuration configuration, Path output, ArtifactCache cache, ImmutableMap<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> symbolProblemTable, ClassPathResult classPathResult, Bom bom) throws TemplateException { Map<Artifact, ArtifactInfo> artifacts = cache.getInfoMap(); List<ArtifactResults> table = new ArrayList<>(); for (Entry<Artifact, ArtifactInfo> entry : artifacts.entrySet()) { ArtifactInfo info = entry.getValue(); try { if (info.getException() != null) { ArtifactResults unavailable = new ArtifactResults(entry.getKey()); unavailable.setExceptionMessage(info.getException().getMessage()); table.add(unavailable); } else { Artifact artifact = entry.getKey(); ImmutableSet<ClassPathEntry> jarsInDependencyTree = classPathResult.getClassPathEntries(Artifacts.toCoordinates(artifact)); Map<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> relevantSymbolProblemTable = Maps.filterKeys(symbolProblemTable, jarsInDependencyTree::contains); ArtifactResults results = generateArtifactReport( configuration, output, artifact, entry.getValue(), cache.getGlobalDependencies(), ImmutableMap.copyOf(relevantSymbolProblemTable), classPathResult, bom); table.add(results); } } catch (IOException ex) { ArtifactResults unavailableTestResult = new ArtifactResults(entry.getKey()); unavailableTestResult.setExceptionMessage(ex.getMessage()); // Even when there's a problem generating test result, show the error in the dashboard table.add(unavailableTestResult); } } return table; } /** * This is the only method that queries the Maven repository. */ private static ArtifactCache loadArtifactInfo(List<Artifact> artifacts) { Map<Artifact, ArtifactInfo> infoMap = new LinkedHashMap<>(); List<DependencyGraph> globalDependencies = new ArrayList<>(); for (Artifact artifact : artifacts) { DependencyGraph completeDependencies = dependencyGraphBuilder.buildFullDependencyGraph(ImmutableList.of(artifact)); globalDependencies.add(completeDependencies); // picks versions according to Maven rules DependencyGraph transitiveDependencies = dependencyGraphBuilder.buildMavenDependencyGraph(new Dependency(artifact, "compile")); ArtifactInfo info = new ArtifactInfo(completeDependencies, transitiveDependencies); infoMap.put(artifact, info); } ArtifactCache cache = new ArtifactCache(); cache.setInfoMap(infoMap); cache.setGlobalDependencies(globalDependencies); return cache; } private static ArtifactResults generateArtifactReport( Configuration configuration, Path output, Artifact artifact, ArtifactInfo artifactInfo, List<DependencyGraph> globalDependencies, ImmutableMap<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> symbolProblemTable, ClassPathResult classPathResult, Bom bom) throws IOException, TemplateException { String coordinates = Artifacts.toCoordinates(artifact); File outputFile = output.resolve(coordinates.replace(':', '_') + ".html").toFile(); try (Writer out = new OutputStreamWriter( new FileOutputStream(outputFile), StandardCharsets.UTF_8)) { // includes all versions DependencyGraph graph = artifactInfo.getCompleteDependencies(); List<Update> convergenceIssues = graph.findUpdates(); // picks versions according to Maven rules DependencyGraph transitiveDependencies = artifactInfo.getTransitiveDependencies(); Map<Artifact, Artifact> upperBoundFailures = findUpperBoundsFailures(graph.getHighestVersionMap(), transitiveDependencies); Map<Artifact, Artifact> globalUpperBoundFailures = findUpperBoundsFailures( collectLatestVersions(globalDependencies), transitiveDependencies); long totalLinkageErrorCount = symbolProblemTable.values().stream() .flatMap(problemToClasses -> problemToClasses.keySet().stream()) .distinct() .count(); Template report = configuration.getTemplate("/templates/component.ftl"); Map<String, Object> templateData = new HashMap<>(); templateData.put("artifact", artifact); templateData.put("updates", convergenceIssues); templateData.put("upperBoundFailures", upperBoundFailures); templateData.put("globalUpperBoundFailures", globalUpperBoundFailures); templateData.put("lastUpdated", LocalDateTime.now()); templateData.put("dependencyGraph", graph); templateData.put("symbolProblems", symbolProblemTable); templateData.put("classPathResult", classPathResult); templateData.put("totalLinkageErrorCount", totalLinkageErrorCount); templateData.put("coordinates", bom.getCoordinates()); report.process(templateData, out); ArtifactResults results = new ArtifactResults(artifact); results.addResult(TEST_NAME_UPPER_BOUND, upperBoundFailures.size()); results.addResult(TEST_NAME_GLOBAL_UPPER_BOUND, globalUpperBoundFailures.size()); results.addResult(TEST_NAME_DEPENDENCY_CONVERGENCE, convergenceIssues.size()); results.addResult(TEST_NAME_LINKAGE_CHECK, (int) totalLinkageErrorCount); return results; } } private static Map<Artifact, Artifact> findUpperBoundsFailures( Map<String, String> expectedVersionMap, DependencyGraph transitiveDependencies) { Map<String, String> actualVersionMap = transitiveDependencies.getHighestVersionMap(); VersionComparator comparator = new VersionComparator(); Map<Artifact, Artifact> upperBoundFailures = new LinkedHashMap<>(); for (String id : expectedVersionMap.keySet()) { String expectedVersion = expectedVersionMap.get(id); String actualVersion = actualVersionMap.get(id); // Check that the actual version is not null because it is // possible for dependencies to appear or disappear from the tree // depending on which version of another dependency is loaded. // In both cases, no action is needed. if (actualVersion != null && comparator.compare(actualVersion, expectedVersion) < 0) { // Maven did not choose highest version DefaultArtifact lower = new DefaultArtifact(id + ":" + actualVersion); DefaultArtifact upper = new DefaultArtifact(id + ":" + expectedVersion); upperBoundFailures.put(lower, upper); } } return upperBoundFailures; } /** * Partitions {@code symbolProblems} by the JAR file that contains the {@link ClassFile}. * * <p>For example, {@code classes = result.get(JarX).get(SymbolProblemY)} where {@code classes} * are not null means that {@code JarX} has {@code SymbolProblemY} and that {@code JarX} contains * {@code classes} which reference {@code SymbolProblemY.getSymbol()}. */ private static ImmutableMap<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> indexByJar(ImmutableSetMultimap<SymbolProblem, ClassFile> symbolProblems) { ImmutableMap<ClassPathEntry, Collection<Entry<SymbolProblem, ClassFile>>> jarMap = Multimaps.index(symbolProblems.entries(), entry -> entry.getValue().getClassPathEntry()) .asMap(); return ImmutableMap.copyOf( Maps.transformValues( jarMap, entries -> ImmutableSetMultimap.copyOf( Multimaps.transformValues( ImmutableSetMultimap.copyOf(entries), ClassFile::getBinaryName)))); } @VisibleForTesting static void generateDashboard( Configuration configuration, Path output, List<ArtifactResults> table, List<DependencyGraph> globalDependencies, ImmutableMap<ClassPathEntry, ImmutableSetMultimap<SymbolProblem, String>> symbolProblemTable, ClassPathResult classPathResult, Bom bom) throws IOException, TemplateException { Map<String, String> latestArtifacts = collectLatestVersions(globalDependencies); Map<String, Object> templateData = new HashMap<>(); templateData.put("table", table); templateData.put("lastUpdated", LocalDateTime.now()); templateData.put("latestArtifacts", latestArtifacts); templateData.put("symbolProblems", symbolProblemTable); templateData.put("classPathResult", classPathResult); templateData.put("dependencyPathRootCauses", findRootCauses(classPathResult)); templateData.put("coordinates", bom.getCoordinates()); templateData.put("dependencyGraphs", globalDependencies); // Accessing static methods from Freemarker template // https://freemarker.apache.org/docs/pgui_misc_beanwrapper.html#autoid_60 DefaultObjectWrapper wrapper = new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_28) .build(); TemplateHashModel staticModels = wrapper.getStaticModels(); templateData.put("dashboardMain", staticModels.get(DashboardMain.class.getName())); templateData.put("pieChart", staticModels.get(PieChart.class.getName())); File dashboardFile = output.resolve("index.html").toFile(); try (Writer out = new OutputStreamWriter( new FileOutputStream(dashboardFile), StandardCharsets.UTF_8)) { Template dashboard = configuration.getTemplate("/templates/index.ftl"); dashboard.process(templateData, out); } File detailsFile = output.resolve("artifact_details.html").toFile(); try (Writer out = new OutputStreamWriter( new FileOutputStream(detailsFile), StandardCharsets.UTF_8)) { Template details = configuration.getTemplate("/templates/artifact_details.ftl"); details.process(templateData, out); } File unstable = output.resolve("unstable_artifacts.html").toFile(); try (Writer out = new OutputStreamWriter( new FileOutputStream(unstable), StandardCharsets.UTF_8)) { Template details = configuration.getTemplate("/templates/unstable_artifacts.ftl"); details.process(templateData, out); } File dependencyTrees = output.resolve("dependency_trees.html").toFile(); try (Writer out = new OutputStreamWriter(new FileOutputStream(dependencyTrees), StandardCharsets.UTF_8)) { Template details = configuration.getTemplate("/templates/dependency_trees.ftl"); details.process(templateData, out); } } private static Map<String, String> collectLatestVersions( List<DependencyGraph> globalDependencies) { Map<String, String> latestArtifacts = new TreeMap<>(); VersionComparator comparator = new VersionComparator(); if (globalDependencies != null) { for (DependencyGraph graph : globalDependencies) { Map<String, String> map = graph.getHighestVersionMap(); for (String key : map.keySet()) { String newVersion = map.get(key); String oldVersion = latestArtifacts.get(key); if (oldVersion == null || comparator.compare(newVersion, oldVersion) > 0) { latestArtifacts.put(key, map.get(key)); } } } } return latestArtifacts; } /** * Returns the number of rows in {@code table} that show unavailable ({@code null} result) or some * failures for {@code columnName}. */ public static long countFailures(List<ArtifactResults> table, String columnName) { return table.stream() .filter(row -> row.getResult(columnName) == null || row.getFailureCount(columnName) > 0) .count(); } private static final int MINIMUM_NUMBER_DEPENDENCY_PATHS = 5; /** * Returns mapping from jar files to summaries of the root problem in their {@link * DependencyPath}s. The summary explains common patterns ({@code groupId:artifactId}) in the path * elements. The returned map does not have a key for a jar file when it has fewer than {@link * #MINIMUM_NUMBER_DEPENDENCY_PATHS} dependency paths or a common pattern is not found among the * elements in the paths. * * <p>Example summary: "Artifacts 'com.google.http-client:google-http-client > * commons-logging:commons-logging > log4j:log4j' exist in all 994 dependency paths. Example * path: com.google.cloud:google-cloud-core:1.59.0 ..." * * <p>Using this summary in the BOM dashboard avoids repetitive items in the {@link * DependencyPath} list that share the same root problem caused by widely-used libraries, for * example, {@code commons-logging:commons-logging}, {@code * com.google.http-client:google-http-client} and {@code log4j:log4j}. */ private static ImmutableMap<String, String> findRootCauses(ClassPathResult classPathResult) { // Freemarker is not good at handling non-string keys. Path object in .ftl is automatically // converted to String. https://freemarker.apache.org/docs/app_faq.html#faq_nonstring_keys ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); for (ClassPathEntry entry : classPathResult.getClassPath()) { List<DependencyPath> dependencyPaths = classPathResult.getDependencyPaths(entry); ImmutableList<String> commonVersionlessArtifacts = commonVersionlessArtifacts(dependencyPaths); if (dependencyPaths.size() > MINIMUM_NUMBER_DEPENDENCY_PATHS && commonVersionlessArtifacts.size() > 1) { // The last paths elements are always same builder.put( entry.toString(), summaryMessage( dependencyPaths.size(), commonVersionlessArtifacts, dependencyPaths.get(0))); } } return builder.build(); } private static ImmutableList<String> commonVersionlessArtifacts( List<DependencyPath> dependencyPaths) { ImmutableList<String> initialVersionlessCoordinates = versionlessCoordinates(dependencyPaths.get(0)); // LinkedHashSet remembers insertion order LinkedHashSet<String> versionlessCoordinatesIntersection = Sets.newLinkedHashSet(initialVersionlessCoordinates); for (DependencyPath dependencyPath : dependencyPaths) { // List of versionless coordinates ("groupId:artifactId") ImmutableList<String> versionlessCoordinatesInPath = versionlessCoordinates(dependencyPath); // intersection of elements in DependencyPaths versionlessCoordinatesIntersection.retainAll(versionlessCoordinatesInPath); } return ImmutableList.copyOf(versionlessCoordinatesIntersection); } private static ImmutableList<String> versionlessCoordinates(DependencyPath dependencyPath) { return dependencyPath.getArtifacts().stream() .map(Artifacts::makeKey) .collect(toImmutableList()); } private static String summaryMessage( int dependencyPathCount, List<String> coordinates, DependencyPath examplePath) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append("Dependency path '"); messageBuilder.append(Joiner.on(" > ").join(coordinates)); messageBuilder.append("' exists in all " + dependencyPathCount + " dependency paths. "); messageBuilder.append("Example path: "); messageBuilder.append(examplePath); return messageBuilder.toString(); } }