org.springframework.cloud.deployer.resource.maven.MavenResource Java Examples

The following examples show how to use org.springframework.cloud.deployer.resource.maven.MavenResource. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AppResourceCommon.java    From spring-cloud-dataflow with Apache License 2.0 6 votes vote down vote up
/**
 * Extracts the version from the resource. Supported resource types are
 * MavenResource, {@link DockerResource}, and {@link UrlResource}.
 * @param resource the resource to use.
 * @return the version the resource.
 */
public String getResourceVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		return mavenResource.getVersion();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageTag(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 
Example #2
Source File: TickTock.java    From spring-cloud-deployer-local with Apache License 2.0 6 votes vote down vote up
private static AppDeploymentRequest createAppDeploymentRequest(String app, String stream) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.stream.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	if (app.contains("-source-")) {
		properties.put("spring.cloud.stream.bindings.output.destination", stream);
	}
	else {
		properties.put("spring.cloud.stream.bindings.input.destination", stream);
		properties.put("spring.cloud.stream.bindings.input.group", "default");
	}
	AppDefinition definition = new AppDefinition(app, properties);
	Map<String, String> deploymentProperties = new HashMap<>();
	deploymentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, stream);
	/*
	 * This will allow output to be logged to the output of the process that started
	 * the application.
	 */
	deploymentProperties.put(LocalDeployerProperties.INHERIT_LOGGING, "true");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, deploymentProperties);
	return request;
}
 
Example #3
Source File: CloudFoundryAppDeployerTests.java    From spring-cloud-deployer-cloudfoundry with Apache License 2.0 6 votes vote down vote up
@Test
public void deployMavenArtifactShouldNotDeleteIfConfigured() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));

	CloudFoundryDeploymentProperties deploymentProperties = new CloudFoundryDeploymentProperties();
	deploymentProperties.setAutoDeleteMavenArtifacts(false);
	CloudFoundryAppDeployer deployer = new CloudFoundryAppDeployer(this.applicationNameGenerator, deploymentProperties,
			this.operations, this.runtimeEnvironmentInfo);

	deployResource(deployer, resource);
	assertTrue(mavenArtifact.getParentFile().exists());
}
 
Example #4
Source File: AbstractSchedulerIntegrationTests.java    From spring-cloud-deployer with Apache License 2.0 6 votes vote down vote up
/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-scheduler-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-scheduler-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-scheduler-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 
Example #5
Source File: AbstractIntegrationTests.java    From spring-cloud-deployer with Apache License 2.0 6 votes vote down vote up
/**
 * Return a resource corresponding to the spring-cloud-deployer-spi-test-app app suitable for the target runtime.
 *
 * The default implementation returns an uber-jar fetched via Maven. Subclasses may override.
 */
protected Resource testApplication() {
	Properties properties = new Properties();
	try {
		properties.load(new ClassPathResource("integration-test-app.properties").getInputStream());
	}
	catch (IOException e) {
		throw new RuntimeException("Failed to determine which version of spring-cloud-deployer-spi-test-app to use", e);
	}
	return new MavenResource.Builder(mavenProperties)
			.groupId("org.springframework.cloud")
			.artifactId("spring-cloud-deployer-spi-test-app")
			.classifier("exec")
			.version(properties.getProperty("version"))
			.extension("jar")
			.build();
}
 
Example #6
Source File: ResourceUtilsTests.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
@Test
public void testMavenResourceProcessing() {
	MavenResource mavenResource = new MavenResource.Builder()
			.artifactId("timestamp-task")
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.RELEASE")
			.build();
	String resourceWithoutVersion = appResourceService.getResourceWithoutVersion(mavenResource);
	assertThat(resourceWithoutVersion).isEqualTo("maven://org.springframework.cloud.task.app:timestamp-task:jar");
	assertThat(appResourceService.getResourceVersion(mavenResource)).isEqualTo("1.0.0.RELEASE");
}
 
Example #7
Source File: AppResourceCommonTests.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetResourceWithoutVersion() {
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:war:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit::exec:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar:exec");
	assertThat(appResourceCommon.getResourceWithoutVersion(
			MavenResource.parse("org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE")))
			.isEqualTo("maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:jar");
}
 
Example #8
Source File: AppResourceCommon.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a string representing the resource with version subtracted
 * @param resource to be represented as string.
 * @return String representation of the resource.
 */
public String getResourceWithoutVersion(Resource resource) {
	Assert.notNull(resource, "resource must not be null");
	if (resource instanceof MavenResource) {
		MavenResource mavenResource = (MavenResource) resource;
		StringBuilder mavenResourceStringBuilder = new StringBuilder();
		mavenResourceStringBuilder.append(String.format("maven://%s:%s",
				mavenResource.getGroupId(),
				mavenResource.getArtifactId()));
		if (StringUtils.hasText(mavenResource.getExtension())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getExtension());
		}
		else {
			mavenResourceStringBuilder.append(":jar");
		}
		if (StringUtils.hasText(mavenResource.getClassifier())) {
			mavenResourceStringBuilder.append(":" + mavenResource.getClassifier());
		}
		return mavenResourceStringBuilder.toString();
	}
	else if (resource instanceof DockerResource) {
		DockerResource dockerResource = (DockerResource) resource;
		return getDockerImageWithoutVersion(dockerResource);
	}
	else if (resource instanceof UrlResource) {
		return getUrlResourceWithoutVersion((UrlResource) resource);
	}
	else {
		throw new IllegalArgumentException("Do not support extracting resource from Resource of type "
				+ resource.getClass().getSimpleName());
	}
}
 
Example #9
Source File: TimeStamp.java    From spring-cloud-deployer-local with Apache License 2.0 5 votes vote down vote up
private static AppDeploymentRequest createAppDeploymentRequest(String app) {
	MavenResource resource = new MavenResource.Builder()
			.artifactId(app)
			.groupId("org.springframework.cloud.task.app")
			.version("1.0.0.BUILD-SNAPSHOT")
			.build();
	Map<String, String> properties = new HashMap<>();
	properties.put("server.port", "0");
	AppDefinition definition = new AppDefinition(app, properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	return request;
}
 
Example #10
Source File: CloudFoundryAppDeployerTests.java    From spring-cloud-deployer-cloudfoundry with Apache License 2.0 5 votes vote down vote up
@Test
public void deployMavenArtifactShouldDeleteByDefault() throws IOException, URISyntaxException {

	MavenResource resource = mock(MavenResource.class);

	folder.newFolder("maven");
	File mavenArtifact = folder.newFile("maven/artifact.jar");
	given(resource.getFile()).willReturn(mavenArtifact);
	given(resource.getURI()).willReturn(new URI("maven://test:demo:0.0.1"));
	assertTrue(this.deploymentProperties.isAutoDeleteMavenArtifacts());
	deployResource(this.deployer, resource);
	assertFalse(mavenArtifact.getParentFile().exists());

}
 
Example #11
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestampAsHdfsResource() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource base = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();
	copyFile(base, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource resource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/timestamp-task-1.0.0.BUILD-SNAPSHOT-exec.jar");

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example #12
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestamp() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example #13
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCancel() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> properties = new HashMap<String, String>();
	// let it run max 60 sec to get status and cancel.
	properties.put("repeat", "60");
	AppDefinition definition = new AppDefinition("timestamp-task", properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Sleeping");

	assertThat(deployer.status(id).getState(), is(LaunchState.running));
	deployer.cancel(id);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Stopping beans");
	assertThat(deployer.status(id).getState(), is(LaunchState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #14
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCommandlineArgs() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	List<String> commandlineArgs = new ArrayList<String>();
	commandlineArgs.add("--format=yyyyMMdd yyyy");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, null, commandlineArgs);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		}
		if (file.getName().endsWith("Container.stdout")) {
			SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd yyyy");
			String expect = format.format(new Date());
			assertThat(content, containsString(expect));
		}
	}
}
 
Example #15
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testSmokeDeployer() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());
	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	for (int i = 0; i < 10; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(timeId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(logId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}

	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #16
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeHdfs() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);
	String fsUri = getConfiguration().get("fs.defaultFS");

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource hdfsResource = new MavenResource.Builder(m2Properties)
			.artifactId("hdfs-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "timehdfs.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> hdfsProperties = new HashMap<>();
	hdfsProperties.put("spring.cloud.stream.bindings.input.destination", "timehdfs.0");
	hdfsProperties.put("spring.hadoop.fsUri", fsUri);
	AppDefinition hdfsDefinition = new AppDefinition("log", hdfsProperties);
	Map<String, String> hdfsEnvironmentProperties = new HashMap<>();
	hdfsEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	hdfsEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest hdfsRequest = new AppDeploymentRequest(hdfsDefinition, hdfsResource, hdfsEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String hdfsId = deployer.deploy(hdfsRequest);
	assertThat(hdfsId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started HdfsSinkApplication");
	assertThat(deployer.status(hdfsId).getState(), is(DeploymentState.deployed));

	waitHdfsFile("/tmp/hdfs-sink/data-0.txt", 2, TimeUnit.MINUTES);

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.timehdfs.0");
	deployer.undeploy(hdfsId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.timehdfs.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));
}
 
Example #17
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLogAsHdfsResource() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	copyFile(timeResourceBase, "/dataflow/artifacts/repo/");
	copyFile(logResourceBase, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource timeResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/time-source-1.0.0.BUILD-SNAPSHOT-exec.jar");
	Resource logResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/log-sink-1.0.0.BUILD-SNAPSHOT-exec.jar");

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #18
Source File: AppResourceCommonTests.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
@Test
public void testGetResource() {
	String mavenUri = "maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.3.0.RELEASE";
	Resource resource = appResourceCommon.getResource(mavenUri);
	assertThat(resource).isInstanceOf(MavenResource.class);
}
 
Example #19
Source File: SkipperStreamDeployerTests.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
@Test
public void testEscapeBackslashProperties() throws IOException {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);


	HashMap<String, String> timeAppProps = new HashMap<>();
	timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");

	// Set the properties for:
	// time --foo.expression=\d --bar=\d --complex.expression="#jsonPath(payload,'$.name') matches '\d*'"
	timeAppProps.put("foo.expression", "\\d");
	timeAppProps.put("bar", "\\d");
	timeAppProps.put("complex.expression", "#jsonPath(payload,'$.name') matches '\\d*'");
	timeAppProps.put("bar.expression", "\\d \\\\ \\0 \\a \\b \\t \\n \\v \\f \\r \\e \\N \\_ \\L \\P \\");

	AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	Map<String, String> skipperDeployerProperties = new HashMap<>();
	skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1",
			"t1: time | log",
			appDeploymentRequests,
			skipperDeployerProperties);

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));
	skipperStreamDeployer.deployStream(streamDeploymentRequest);

	ArgumentCaptor<UploadRequest> uploadRequestCaptor = ArgumentCaptor.forClass(UploadRequest.class);
	verify(skipperClient).upload(uploadRequestCaptor.capture());
	assertThat(uploadRequestCaptor.getValue()).isNotNull();
	assertThat(uploadRequestCaptor.getValue().getName()).isEqualTo("test1");
	assertThat(uploadRequestCaptor.getValue().getVersion()).isEqualTo("1.0.1");

	Package pkg = SkipperPackageUtils.loadPackageFromBytes(uploadRequestCaptor);
	Package timePackage = pkg.getDependencies().get(0);

	assertThat(timePackage).isNotNull();
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"foo.expression\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar\": \"\\\\d\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"complex.expression\": \"#jsonPath(payload,'$.name') matches '\\\\d*'\"");
	assertThat(timePackage.getConfigValues().getRaw()).contains("\"bar.expression\": \"\\\\d \\\\\\\\ \\\\0 \\\\a \\\\b \\\\t \\\\n \\\\v \\\\f \\\\r \\\\e \\\\N \\\\_ \\\\L \\\\P \\\\\"");
}
 
Example #20
Source File: SkipperStreamDeployerTests.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
private void testAppRegisteredOnStreamDeploy(AppRegistryService appRegistryService) {

		HashMap<String, String> timeAppProps = new HashMap<>();
		timeAppProps.put("spring.cloud.dataflow.stream.app.type", "source");
		AppDefinition timeAppDefinition = new AppDefinition("time", timeAppProps);
		MavenResource timeResource = new MavenResource.Builder()
				.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
		AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

		HashMap<String, String> logAppProps = new HashMap<>();
		logAppProps.put("spring.cloud.dataflow.stream.app.type", "sink");
		AppDefinition logAppDefinition = new AppDefinition("log", logAppProps);
		MavenResource logResource = new MavenResource.Builder()
				.artifactId("log-sink-rabbit").groupId("org.springframework.cloud.stream.app")
				.version("1.2.0.RELEASE").build();
		when(appRegistryService.getResourceVersion(logResource)).thenReturn(logResource.getVersion());
		AppDeploymentRequest logAppDeploymentRequest = new AppDeploymentRequest(logAppDefinition, logResource);

		List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(logAppDeploymentRequest, timeAppDeploymentRequest);

		Map<String, String> skipperDeployerProperties = new HashMap<>();
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_NAME, "package1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PACKAGE_VERSION, "1.0.1");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_PLATFORM_NAME, "testPlatform");
		skipperDeployerProperties.put(SkipperStream.SKIPPER_REPO_NAME, "mylocal-repo1");

		StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest("test1", "time | log",
				appDeploymentRequests, skipperDeployerProperties);

		SkipperClient skipperClient = MockUtils.createSkipperClientMock();

		StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
		SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
				streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

		when(streamDefinitionRepository.findById("test1")).thenReturn(Optional.of(new StreamDefinition("test1", "t1: time | log")));

		skipperStreamDeployer.deployStream(streamDeploymentRequest);
	}
 
Example #21
Source File: SkipperStreamDeployerTests.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamDeployWithLongAppName() {

	AppRegistryService appRegistryService = mock(AppRegistryService.class);

	when(appRegistryService.appExist(eq("time"), eq(ApplicationType.source), eq("1.2.0.RELEASE")))
			.thenReturn(true);
	when(appRegistryService.appExist(eq("log"), eq(ApplicationType.sink), eq("1.2.0.RELEASE")))
			.thenReturn(true);

	AppDefinition timeAppDefinition = new AppDefinition("time", new HashMap<>());
	MavenResource timeResource = new MavenResource.Builder()
			.artifactId("time-source-rabbit").groupId("org.springframework.cloud.stream.app")
			.version("1.2.0.RELEASE").build();
	when(appRegistryService.getResourceVersion(timeResource)).thenReturn(timeResource.getVersion());
	AppDeploymentRequest timeAppDeploymentRequest = new AppDeploymentRequest(timeAppDefinition, timeResource);

	List<AppDeploymentRequest> appDeploymentRequests = Arrays.asList(timeAppDeploymentRequest);

	String streamName = "asdfkdunfdnereerejrerkjelkraerkldjkfdjfkdsjflkjdflkdjflsdflsdjfldlfdlsfjdlfjdlfjdslfdnmdfndfmdsfmndsdfafdsfmdnfdske";

	String streamDSL = "time | log";

	StreamDeploymentRequest streamDeploymentRequest = new StreamDeploymentRequest(streamName, streamDSL,
							appDeploymentRequests,
			new HashMap<>());

	SkipperClient skipperClient = MockUtils.createSkipperClientMock();

	StreamDefinitionRepository streamDefinitionRepository = mock(StreamDefinitionRepository.class);
	SkipperStreamDeployer skipperStreamDeployer = new SkipperStreamDeployer(skipperClient,
			streamDefinitionRepository, appRegistryService, mock(ForkJoinPool.class), new DefaultStreamDefinitionService());

	when(streamDefinitionRepository.findById(streamName)).thenReturn(Optional.of(new StreamDefinition(streamName, streamDSL)));
	try {
		skipperStreamDeployer.deployStream(streamDeploymentRequest);
		fail("Expected InvalidStreamDefinitionException");
	}
	catch (Exception e) {
		assertThat(e instanceof InvalidStreamDefinitionException).isTrue();
		assertThat(e.getMessage().equals("The runtime application name for the app time in the stream "+streamName+" should not exceed 63 in length. Currently it is: "+streamName+"-time-v{version-2digits}"));
	}
}
 
Example #22
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLog() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}