/*
* Copyright 2015-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.deployer.spi.yarn.appdeployer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.StringUtils;
import org.springframework.yarn.am.ContainerLauncherInterceptor;
import org.springframework.yarn.am.cluster.ContainerCluster;
import org.springframework.yarn.am.cluster.ManagedContainerClusterAppmaster;
import org.springframework.yarn.am.container.AbstractLauncher;
import org.springframework.yarn.am.grid.GridMember;
import org.springframework.yarn.am.grid.support.ProjectionData;
import org.springframework.yarn.fs.LocalResourcesFactoryBean;
import org.springframework.yarn.fs.LocalResourcesFactoryBean.CopyEntry;
import org.springframework.yarn.fs.LocalResourcesFactoryBean.TransferEntry;
import org.springframework.yarn.fs.ResourceLocalizer;
import org.springframework.yarn.listener.ContainerMonitorListener;
/**
* Custom yarn appmaster tweaking container launch settings.
*
* @author Janne Valkealahti
*
*/
public class StreamAppmaster extends ManagedContainerClusterAppmaster {
private final static Log log = LogFactory.getLog(StreamAppmaster.class);
private final Map<String, ResourceLocalizer> artifactLocalizers = new HashMap<>();
private final ContainerIndexTracker indexTracker = new ContainerIndexTracker();
/** ContainerId to ContainerCluster id map */
private final Map<ContainerId, String> containerIdMap = new HashMap<>();
@Autowired
private StreamAppmasterProperties streamAppmasterProperties;
@Override
protected void onInit() throws Exception {
super.onInit();
// TODO: we want to have a proper support in base classes to gracefully
// shutdown appmaster when it has nothing to do. this trick
// here is solely a workaround not being able to access internal
// structures of base classes. this is pretty much all we can do
// from a subclass.
// potentially we want to make it configurable with a grace period, etc.
getMonitor().addContainerMonitorStateListener(new ContainerMonitorListener() {
@Override
public void state(ContainerMonitorState state) {
if (log.isDebugEnabled()) {
log.info("Received monitor state " + state + " and container clusters size is " + getContainerClusters().size());
}
if (state.getRunning() == 0 && getContainerClusters().size() == 0) {
// this state is valid at start but we know it's not gonna
// get called until we have had at least one container running
log.info("No running containers and no container clusters, initiate app shutdown");
notifyCompleted();
}
}
});
if (getLauncher() instanceof AbstractLauncher) {
((AbstractLauncher)getLauncher()).addInterceptor(new IndexAddingContainerLauncherInterceptor());
}
}
@Override
public ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projectionData,
Map<String, Object> extraProperties) {
log.info("intercept createContainerCluster " + clusterId);
String artifactPath = streamAppmasterProperties.getArtifact();
try {
LocalResourcesFactoryBean lrfb = new LocalResourcesFactoryBean();
lrfb.setConfiguration(getConfiguration());
String containerArtifact = (String) extraProperties.get("containerArtifact");
TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifactPath + "/" + containerArtifact, false);
ArrayList<TransferEntry> hdfsEntries = new ArrayList<TransferEntry>();
hdfsEntries.add(te);
lrfb.setHdfsEntries(hdfsEntries);
lrfb.setCopyEntries(new ArrayList<CopyEntry>());
lrfb.afterPropertiesSet();
ResourceLocalizer rl = lrfb.getObject();
log.info("Adding localizer for " + clusterId + " / " + rl);
artifactLocalizers.put(clusterId, rl);
} catch (Exception e) {
log.error("Error creating localizer", e);
}
return super.createContainerCluster(clusterId, clusterDef, projectionData, extraProperties);
}
@Override
protected List<String> onContainerLaunchCommands(Container container, ContainerCluster cluster,
List<String> commands) {
log.info("onContainerLaunchCommands commands=" + StringUtils.collectionToCommaDelimitedString(commands));
ArrayList<String> list = new ArrayList<String>();
Map<String, Object> extraProperties = cluster.getExtraProperties();
String artifact = (String) extraProperties.get("containerArtifact");
for (String command : commands) {
if (command.contains("placeholder.jar")) {
list.add(command.replace("placeholder.jar", artifact));
} else {
list.add(command);
}
}
if (extraProperties != null) {
for (Entry<String, Object> entry : extraProperties.entrySet()) {
if (entry.getKey().startsWith("containerArg")) {
log.info("onContainerLaunchCommands adding command=--" + entry.getValue().toString());
list.add(Math.max(list.size() - 2, 0), "--" + entry.getValue().toString());
}
}
}
return list;
}
@Override
protected Map<String, LocalResource> buildLocalizedResources(ContainerCluster cluster) {
Map<String, LocalResource> resources = super.buildLocalizedResources(cluster);
ResourceLocalizer rl = artifactLocalizers.get(cluster.getId());
log.info("Localizer for " + cluster.getId() + " is " + rl);
resources.putAll(rl.getResources());
return resources;
}
@Override
protected void onContainerCompleted(ContainerStatus status) {
super.onContainerCompleted(status);
String containerClusterId = containerIdMap.get(status.getContainerId());
if (containerClusterId != null) {
synchronized (indexTracker) {
indexTracker.freeIndex(status.getContainerId(), containerClusterId);
}
}
}
private ContainerCluster findContainerClusterByContainerId(ContainerId containerId) {
for (Entry<String, ContainerCluster> entry : getContainerClusters().entrySet()) {
for (GridMember member : entry.getValue().getGridProjection().getMembers()) {
if (member.getId().equals(containerId)) {
return entry.getValue();
}
}
}
return null;
}
/**
* Interceptor adding INSTANCE_INDEX as env variable based on ContainerIndexTracker.
*/
private class IndexAddingContainerLauncherInterceptor implements ContainerLauncherInterceptor {
@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
ContainerCluster containerCluster = findContainerClusterByContainerId(container.getId());
if (containerCluster == null) {
return context;
}
containerIdMap.put(container.getId(), containerCluster.getId());
Map<String, String> environment = context.getEnvironment();
Map<String, String> indexEnv = new HashMap<>();
indexEnv.putAll(environment);
Integer reservedIndex;
synchronized (indexTracker) {
reservedIndex = indexTracker.reserveIndex(container.getId(), containerCluster);
}
indexEnv.put("SPRING_CLOUD_APPLICATION_GUID", container.getId().toString());
indexEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(reservedIndex));
indexEnv.put("INSTANCE_INDEX", Integer.toString(reservedIndex));
context.setEnvironment(indexEnv);
return context;
}
}
/**
* Support class tracking containers per group and reserving an index sequence.
*/
private static class ContainerIndexTracker {
// TODO: move this feature to spring-yarn where scaling can be
// implemented accurately. scaling up/down is anyway not
// supported in sc stream at this moment.
Map<String, ArrayList<ContainerId>> reservationsMap = new HashMap<>();
Integer reserveIndex(ContainerId containerId, ContainerCluster containerCluster) {
ArrayList<ContainerId> reservationList = reservationsMap.get(containerCluster.getId());
if (reservationList == null) {
reservationList = new ArrayList<>();
reservationsMap.put(containerCluster.getId(), reservationList);
}
// we always increment index at least once
Iterator<ContainerId> iterator = reservationList.iterator();
int index = -1;
ContainerId n = null;
while(iterator.hasNext()) {
ContainerId nn = iterator.next();
index++;
if (nn == null) {
// we found existing nullified reservation, use that
n = containerId;
break;
}
}
// all resevations in use, add new
if (n == null) {
reservationList.add(containerId);
index++;
}
return index;
}
void freeIndex(ContainerId containerId, String containerClusterId) {
ArrayList<ContainerId> reservationList = reservationsMap.get(containerClusterId);
if (reservationList != null) {
for (int index = 0; index < reservationList.size(); index++) {
if (containerId.equals(reservationList.get(index))) {
// nullify existing reservation
reservationList.set(index, null);
return;
}
}
}
}
}
}
