Java Code Examples for org.springframework.cloud.deployer.resource.maven.MavenResource

The following examples show how to use org.springframework.cloud.deployer.resource.maven.MavenResource. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-scheduler-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-scheduler-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-scheduler-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 
Example 2
/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 
Example 3
@Test
public void deployMavenArtifactShouldNotDeleteIfConfigured() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));

	CloudFoundryDeploymentProperties deploymentProperties = new CloudFoundryDeploymentProperties();
	deploymentProperties.setAutoDeleteMavenArtifacts(false);
	CloudFoundryAppDeployer deployer = new CloudFoundryAppDeployer(this.applicationNameGenerator, deploymentProperties,
			this.operations, this.runtimeEnvironmentInfo);

	deployResource(deployer, resource);
	assertTrue(mavenArtifact.getParentFile().exists());
}
 
Example 4
Source Project: spring-cloud-deployer-local   Source File: TickTock.java    License: Apache License 2.0 6 votes vote down vote up
private static AppDeploymentRequest createAppDeploymentRequest(String app, String stream) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.stream.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	if (app.contains("-source-")) {
		properties.put("spring.cloud.stream.bindings.output.destination", stream);
	}
	else {
		properties.put("spring.cloud.stream.bindings.input.destination", stream);
		properties.put("spring.cloud.stream.bindings.input.group", "default");
	}
	AppDefinition definition = new AppDefinition(app, properties);
	Map<String, String> deploymentProperties = new HashMap<>();
	deploymentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, stream);
	/*
	 * This will allow output to be logged to the output of the process that started
	 * the application.
	 */
	deploymentProperties.put(LocalDeployerProperties.INHERIT_LOGGING, "true");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, deploymentProperties);
	return request;
}
 
Example 5
Source Project: spring-cloud-dataflow   Source File: AppResourceCommon.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Extracts the version from the resource. Supported resource types are
 * MavenResource, {@link DockerResource}, and {@link UrlResource}.
 * @param resource the resource to use.
 * @return the version the resource.
 */
public String getResourceVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		return mavenResource.getVersion();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageTag(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 
Example 6
Source Project: spring-cloud-deployer-yarn   Source File: TaskLauncherIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestamp() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example 7
Source Project: spring-cloud-deployer-yarn   Source File: TaskLauncherIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestampAsHdfsResource() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource base = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();
	copyFile(base, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource resource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/timestamp-task-1.0.0.BUILD-SNAPSHOT-exec.jar");

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example 8
@Test
public void deployMavenArtifactShouldDeleteByDefault() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));
	assertTrue(this.deploymentProperties.isAutoDeleteMavenArtifacts());
	deployResource(this.deployer, resource);
	assertFalse(mavenArtifact.getParentFile().exists());

}
 
Example 9
Source Project: spring-cloud-deployer-local   Source File: TimeStamp.java    License: Apache License 2.0 5 votes vote down vote up
private static AppDeploymentRequest createAppDeploymentRequest(String app) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	AppDefinition definition = new AppDefinition(app, properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	return request;
}
 
Example 10
Source Project: spring-cloud-dataflow   Source File: AppResourceCommon.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns a string representing the resource with version subtracted
 * @param resource to be represented as string.
 * @return String representation of the resource.
 */
public String getResourceWithoutVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		StringBuilder mavenResourceStringBuilder = new StringBuilder();
		mavenResourceStringBuilder.append(String.format("maven://%s:%s",
				mavenResource.getGroupId(),
				mavenResource.getArtifactId()));
		if (StringUtils.hasText(mavenResource.getExtension())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getExtension());
		}
		else {
			mavenResourceStringBuilder.append(":jar");
		}
		if (StringUtils.hasText(mavenResource.getClassifier())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getClassifier());
		}
		return mavenResourceStringBuilder.toString();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageWithoutVersion(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceWithoutVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 
Example 11
@Test
public void testGetResourceWithoutVersion() {
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit::exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar");
}
 
Example 12
Source Project: spring-cloud-dataflow   Source File: ResourceUtilsTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMavenResourceProcessing() {
	MavenResource mavenResource = new MavenResource.Builder()
			.artifactId("timestamp-task")
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.RELEASE")
			.build();
	String resourceWithoutVersion = appResourceService.getResourceWithoutVersion(mavenResource);
	assertThat(resourceWithoutVersion).isEqualTo("maven://org.springframework.cloud.task.app:timestamp-task:jar");
	assertThat(appResourceService.getResourceVersion(mavenResource)).isEqualTo("1.0.0.RELEASE");
}
 
Example 13
Source Project: spring-cloud-deployer-yarn   Source File: AppDeployerIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLog() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example 14
Source Project: spring-cloud-deployer-yarn   Source File: AppDeployerIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLogAsHdfsResource() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	copyFile(timeResourceBase, "/dataflow/artifacts/repo/");
	copyFile(logResourceBase, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource timeResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/time-source-1.0.0.BUILD-SNAPSHOT-exec.jar");
	Resource logResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/log-sink-1.0.0.BUILD-SNAPSHOT-exec.jar");

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example 15
Source Project: spring-cloud-deployer-yarn   Source File: AppDeployerIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeHdfs() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);
	String fsUri = getConfiguration().get("fs.defaultFS");

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource hdfsResource = new MavenResource.Builder(m2Properties)
			.artifactId("hdfs-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "timehdfs.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> hdfsProperties = new HashMap<>();
	hdfsProperties.put("spring.cloud.stream.bindings.input.destination", "timehdfs.0");
	hdfsProperties.put("spring.hadoop.fsUri", fsUri);
	AppDefinition hdfsDefinition = new AppDefinition("log", hdfsProperties);
	Map<String, String> hdfsEnvironmentProperties = new HashMap<>();
	hdfsEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	hdfsEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest hdfsRequest = new AppDeploymentRequest(hdfsDefinition, hdfsResource, hdfsEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String hdfsId = deployer.deploy(hdfsRequest);
	assertThat(hdfsId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started HdfsSinkApplication");
	assertThat(deployer.status(hdfsId).getState(), is(DeploymentState.deployed));

	waitHdfsFile("/tmp/hdfs-sink/data-0.txt", 2, TimeUnit.MINUTES);

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.timehdfs.0");
	deployer.undeploy(hdfsId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.timehdfs.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));
}
 
Example 16
Source Project: spring-cloud-deployer-yarn   Source File: AppDeployerIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSmokeDeployer() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());
	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	for (int i = 0; i < 10; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(timeId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(logId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}

	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example 17
Source Project: spring-cloud-deployer-yarn   Source File: TaskLauncherIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCommandlineArgs() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	List<String> commandlineArgs = new ArrayList<String>();
	commandlineArgs.add("--format=yyyyMMdd yyyy");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, null, commandlineArgs);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		}
		if (file.getName().endsWith("Container.stdout")) {
			SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd yyyy");
			String expect = format.format(new Date());
			assertThat(content, containsString(expect));
		}
	}
}
 
Example 18
Source Project: spring-cloud-deployer-yarn   Source File: TaskLauncherIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCancel() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> properties = new HashMap<String, String>();
	// let it run max 60 sec to get status and cancel.
	properties.put("repeat", "60");
	AppDefinition definition = new AppDefinition("timestamp-task", properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Sleeping");

	assertThat(deployer.status(id).getState(), is(LaunchState.running));
	deployer.cancel(id);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Stopping beans");
	assertThat(deployer.status(id).getState(), is(LaunchState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example 19
@Test
public void testGetResource() {
	String mavenUri = "maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE";
	Resource resource = appResourceCommon.getResource(mavenUri);
	assertThat(resource).isInstanceOf(MavenResource.class);
}
 
Example 20
@Test
public void testEscapeBackslashProperties() throws IOException {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);


	HashMap<String, String> timeAppProps = new HashMap<>();
	timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");

	// Set the properties for:
	// time --foo.expression=\d --bar=\d --complex.expression="#jsonPath(payload,'$.name') matches '\d*'"
	timeAppProps.put("foo.expression", "\\d");
	timeAppProps.put("bar", "\\d");
	timeAppProps.put("complex.expression", "#jsonPath(payload,'$.name') matches '\\d*'");
	timeAppProps.put("bar.expression", "\\d \\\\ \\0 \\a \\b \\t \\n \\v \\f \\r \\e \\N \\_ \\L \\P \\");

	AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	Map<String, String> skipperDeployerProperties = new HashMap<>();
	skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1",
			"t1: time | log",
			appDeploymentRequests,
			skipperDeployerProperties);

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));
	skipperStreamDeployer.deployStream(streamDeploymentRequest);

	ArgumentCaptor<UploadRequest> uploadRequestCaptor = ArgumentCaptor.forClass(UploadRequest.class);
	verify(skipperClient).upload(uploadRequestCaptor.capture());
	assertThat(uploadRequestCaptor.getValue()).isNotNull();
	assertThat(uploadRequestCaptor.getValue().getName()).isEqualTo("test1");
	assertThat(uploadRequestCaptor.getValue().getVersion()).isEqualTo("1.0.1");

	Package pkg = SkipperPackageUtils.loadPackageFromBytes(uploadRequestCaptor);
	Package timePackage = pkg.getDependencies().get(0);

	assertThat(timePackage).isNotNull();
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"foo.expression\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"complex.expression\": \"#jsonPath(payload,'$.name') matches '\\\\d*'\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar.expression\": \"\\\\d \\\\\\\\ \\\\0 \\\\a \\\\b \\\\t \\\\n \\\\v \\\\f \\\\r \\\\e \\\\N \\\\_ \\\\L \\\\P \\\\\"");
}
 
Example 21
private void testAppRegisteredOnStreamDeploy(AppRegistryService appRegistryService) {

		HashMap<String, String> timeAppProps = new HashMap<>();
		timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");
		AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
		MavenResource timeResource = new MavenResource.Builder()
				.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
		AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

		HashMap<String, String> logAppProps = new HashMap<>();
		logAppProps.put("spring.cloud.dataflow.stream.app.type", "sink");
		AppDefinition logAppDefinition = new AppDefinition("log", logAppProps);
		MavenResource logResource = new MavenResource.Builder()
				.artifactId("log-sink-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(logResource)).thenReturn(logResource.getVersion());
		AppDeploymentRequest logAppDeploymentRequest = new AppDeploymentRequest(logAppDefinition, logResource);

		List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(logAppDeploymentRequest, timeAppDeploymentRequest);

		Map<String, String> skipperDeployerProperties = new HashMap<>();
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_NAME, "package1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PLATFORM_NAME, "testPlatform");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_REPO_NAME, "mylocal-repo1");

		StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1", "time | log",
				appDeploymentRequests, skipperDeployerProperties);

		SkipperClient skipperClient = MockUtils.createSkipperClientMock();

		StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
		SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
				streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

		when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));

		skipperStreamDeployer.deployStream(streamDeploymentRequest);
	}
 
Example 22
@Test
public void testStreamDeployWithLongAppName() {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);

	AppDefinition timeAppDefinition = new AppDefinition("time", new HashMap<>());
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	String streamName = "asdfkdunfdnereerejrerkjelkraerkldjkfdjfkdsjflkjdflkdjflsdflsdjfldlfdlsfjdlfjdlfjdslfdnmdfndfmdsfmndsdfafdsfmdnfdske";

	String streamDSL = "time | log";

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest(streamName, streamDSL,
							appDeploymentRequests,
			new HashMap<>());

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById(streamName)).thenReturn(Optional.of(new StreamDefinition(streamName, streamDSL)));
	try {
		skipperStreamDeployer.deployStream(streamDeploymentRequest);
		fail("Expected InvalidStreamDefinitionException");
	}
	catch (Exception e) {
		assertThat(e instanceof InvalidStreamDefinitionException).isTrue();
		assertThat(e.getMessage().equals("The runtime application name for the app time in the stream "+streamName+" should not exceed 63 in length. Currently it is: "+streamName+"-time-v{version-2digits}"));
	}
}