org.springframework.cloud.deployer.resource.maven.MavenResource Java Examples

/**
 * Extracts the version from the resource. Supported resource types are
 * MavenResource, {@link DockerResource}, and {@link UrlResource}.
 * @param resource the resource to use.
 * @return the version the resource.
 */
public String getResourceVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		return mavenResource.getVersion();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageTag(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 

private static AppDeploymentRequest createAppDeploymentRequest(String app, String stream) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.stream.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	if (app.contains("-source-")) {
		properties.put("spring.cloud.stream.bindings.output.destination", stream);
	}
	else {
		properties.put("spring.cloud.stream.bindings.input.destination", stream);
		properties.put("spring.cloud.stream.bindings.input.group", "default");
	}
	AppDefinition definition = new AppDefinition(app, properties);
	Map<String, String> deploymentProperties = new HashMap<>();
	deploymentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, stream);
	/*
	 * This will allow output to be logged to the output of the process that started
	 * the application.
	 */
	deploymentProperties.put(LocalDeployerProperties.INHERIT_LOGGING, "true");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, deploymentProperties);
	return request;
}
 

@Test
public void deployMavenArtifactShouldNotDeleteIfConfigured() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));

	CloudFoundryDeploymentProperties deploymentProperties = new CloudFoundryDeploymentProperties();
	deploymentProperties.setAutoDeleteMavenArtifacts(false);
	CloudFoundryAppDeployer deployer = new CloudFoundryAppDeployer(this.applicationNameGenerator, deploymentProperties,
			this.operations, this.runtimeEnvironmentInfo);

	deployResource(deployer, resource);
	assertTrue(mavenArtifact.getParentFile().exists());
}
 

/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-scheduler-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-scheduler-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-scheduler-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 

/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 

@Test
public void testMavenResourceProcessing() {
	MavenResource mavenResource = new MavenResource.Builder()
			.artifactId("timestamp-task")
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.RELEASE")
			.build();
	String resourceWithoutVersion = appResourceService.getResourceWithoutVersion(mavenResource);
	assertThat(resourceWithoutVersion).isEqualTo("maven://org.springframework.cloud.task.app:timestamp-task:jar");
	assertThat(appResourceService.getResourceVersion(mavenResource)).isEqualTo("1.0.0.RELEASE");
}
 

@Test
public void testGetResourceWithoutVersion() {
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit::exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar");
}
 

/**
 * Returns a string representing the resource with version subtracted
 * @param resource to be represented as string.
 * @return String representation of the resource.
 */
public String getResourceWithoutVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		StringBuilder mavenResourceStringBuilder = new StringBuilder();
		mavenResourceStringBuilder.append(String.format("maven://%s:%s",
				mavenResource.getGroupId(),
				mavenResource.getArtifactId()));
		if (StringUtils.hasText(mavenResource.getExtension())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getExtension());
		}
		else {
			mavenResourceStringBuilder.append(":jar");
		}
		if (StringUtils.hasText(mavenResource.getClassifier())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getClassifier());
		}
		return mavenResourceStringBuilder.toString();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageWithoutVersion(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceWithoutVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 

private static AppDeploymentRequest createAppDeploymentRequest(String app) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	AppDefinition definition = new AppDefinition(app, properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	return request;
}
 

@Test
public void deployMavenArtifactShouldDeleteByDefault() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));
	assertTrue(this.deploymentProperties.isAutoDeleteMavenArtifacts());
	deployResource(this.deployer, resource);
	assertFalse(mavenArtifact.getParentFile().exists());

}
 

@Test
public void testTaskTimestampAsHdfsResource() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource base = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();
	copyFile(base, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource resource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/timestamp-task-1.0.0.BUILD-SNAPSHOT-exec.jar");

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 

@Test
public void testTaskTimestamp() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 

@Test
public void testTaskTimestampCancel() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> properties = new HashMap<String, String>();
	// let it run max 60 sec to get status and cancel.
	properties.put("repeat", "60");
	AppDefinition definition = new AppDefinition("timestamp-task", properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Sleeping");

	assertThat(deployer.status(id).getState(), is(LaunchState.running));
	deployer.cancel(id);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Stopping beans");
	assertThat(deployer.status(id).getState(), is(LaunchState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 

@Test
public void testTaskTimestampCommandlineArgs() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	List<String> commandlineArgs = new ArrayList<String>();
	commandlineArgs.add("--format=yyyyMMdd yyyy");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, null, commandlineArgs);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		}
		if (file.getName().endsWith("Container.stdout")) {
			SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd yyyy");
			String expect = format.format(new Date());
			assertThat(content, containsString(expect));
		}
	}
}
 

@Test
public void testSmokeDeployer() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());
	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	for (int i = 0; i < 10; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(timeId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(logId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}

	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 

@Test
public void testStreamTimeHdfs() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);
	String fsUri = getConfiguration().get("fs.defaultFS");

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource hdfsResource = new MavenResource.Builder(m2Properties)
			.artifactId("hdfs-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "timehdfs.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> hdfsProperties = new HashMap<>();
	hdfsProperties.put("spring.cloud.stream.bindings.input.destination", "timehdfs.0");
	hdfsProperties.put("spring.hadoop.fsUri", fsUri);
	AppDefinition hdfsDefinition = new AppDefinition("log", hdfsProperties);
	Map<String, String> hdfsEnvironmentProperties = new HashMap<>();
	hdfsEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	hdfsEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest hdfsRequest = new AppDeploymentRequest(hdfsDefinition, hdfsResource, hdfsEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String hdfsId = deployer.deploy(hdfsRequest);
	assertThat(hdfsId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started HdfsSinkApplication");
	assertThat(deployer.status(hdfsId).getState(), is(DeploymentState.deployed));

	waitHdfsFile("/tmp/hdfs-sink/data-0.txt", 2, TimeUnit.MINUTES);

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.timehdfs.0");
	deployer.undeploy(hdfsId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.timehdfs.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));
}
 

@Test
public void testStreamTimeLogAsHdfsResource() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	copyFile(timeResourceBase, "/dataflow/artifacts/repo/");
	copyFile(logResourceBase, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource timeResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/time-source-1.0.0.BUILD-SNAPSHOT-exec.jar");
	Resource logResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/log-sink-1.0.0.BUILD-SNAPSHOT-exec.jar");

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 

@Test
public void testGetResource() {
	String mavenUri = "maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE";
	Resource resource = appResourceCommon.getResource(mavenUri);
	assertThat(resource).isInstanceOf(MavenResource.class);
}
 

@Test
public void testEscapeBackslashProperties() throws IOException {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);


	HashMap<String, String> timeAppProps = new HashMap<>();
	timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");

	// Set the properties for:
	// time --foo.expression=\d --bar=\d --complex.expression="#jsonPath(payload,'$.name') matches '\d*'"
	timeAppProps.put("foo.expression", "\\d");
	timeAppProps.put("bar", "\\d");
	timeAppProps.put("complex.expression", "#jsonPath(payload,'$.name') matches '\\d*'");
	timeAppProps.put("bar.expression", "\\d \\\\ \\0 \\a \\b \\t \\n \\v \\f \\r \\e \\N \\_ \\L \\P \\");

	AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	Map<String, String> skipperDeployerProperties = new HashMap<>();
	skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1",
			"t1: time | log",
			appDeploymentRequests,
			skipperDeployerProperties);

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));
	skipperStreamDeployer.deployStream(streamDeploymentRequest);

	ArgumentCaptor<UploadRequest> uploadRequestCaptor = ArgumentCaptor.forClass(UploadRequest.class);
	verify(skipperClient).upload(uploadRequestCaptor.capture());
	assertThat(uploadRequestCaptor.getValue()).isNotNull();
	assertThat(uploadRequestCaptor.getValue().getName()).isEqualTo("test1");
	assertThat(uploadRequestCaptor.getValue().getVersion()).isEqualTo("1.0.1");

	Package pkg = SkipperPackageUtils.loadPackageFromBytes(uploadRequestCaptor);
	Package timePackage = pkg.getDependencies().get(0);

	assertThat(timePackage).isNotNull();
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"foo.expression\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"complex.expression\": \"#jsonPath(payload,'$.name') matches '\\\\d*'\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar.expression\": \"\\\\d \\\\\\\\ \\\\0 \\\\a \\\\b \\\\t \\\\n \\\\v \\\\f \\\\r \\\\e \\\\N \\\\_ \\\\L \\\\P \\\\\"");
}
 

private void testAppRegisteredOnStreamDeploy(AppRegistryService appRegistryService) {

		HashMap<String, String> timeAppProps = new HashMap<>();
		timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");
		AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
		MavenResource timeResource = new MavenResource.Builder()
				.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
		AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

		HashMap<String, String> logAppProps = new HashMap<>();
		logAppProps.put("spring.cloud.dataflow.stream.app.type", "sink");
		AppDefinition logAppDefinition = new AppDefinition("log", logAppProps);
		MavenResource logResource = new MavenResource.Builder()
				.artifactId("log-sink-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(logResource)).thenReturn(logResource.getVersion());
		AppDeploymentRequest logAppDeploymentRequest = new AppDeploymentRequest(logAppDefinition, logResource);

		List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(logAppDeploymentRequest, timeAppDeploymentRequest);

		Map<String, String> skipperDeployerProperties = new HashMap<>();
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_NAME, "package1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PLATFORM_NAME, "testPlatform");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_REPO_NAME, "mylocal-repo1");

		StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1", "time | log",
				appDeploymentRequests, skipperDeployerProperties);

		SkipperClient skipperClient = MockUtils.createSkipperClientMock();

		StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
		SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
				streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

		when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));

		skipperStreamDeployer.deployStream(streamDeploymentRequest);
	}
 

@Test
public void testStreamDeployWithLongAppName() {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);

	AppDefinition timeAppDefinition = new AppDefinition("time", new HashMap<>());
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	String streamName = "asdfkdunfdnereerejrerkjelkraerkldjkfdjfkdsjflkjdflkdjflsdflsdjfldlfdlsfjdlfjdlfjdslfdnmdfndfmdsfmndsdfafdsfmdnfdske";

	String streamDSL = "time | log";

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest(streamName, streamDSL,
							appDeploymentRequests,
			new HashMap<>());

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById(streamName)).thenReturn(Optional.of(new StreamDefinition(streamName, streamDSL)));
	try {
		skipperStreamDeployer.deployStream(streamDeploymentRequest);
		fail("Expected InvalidStreamDefinitionException");
	}
	catch (Exception e) {
		assertThat(e instanceof InvalidStreamDefinitionException).isTrue();
		assertThat(e.getMessage().equals("The runtime application name for the app time in the stream "+streamName+" should not exceed 63 in length. Currently it is: "+streamName+"-time-v{version-2digits}"));
	}
}
 

@Test
public void testStreamTimeLog() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}