org.springframework.cloud.deployer.resource.maven.MavenProperties Java Examples

The following examples show how to use org.springframework.cloud.deployer.resource.maven.MavenProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JdbcDataflowTaskExecutionMetadataDao.java    From spring-cloud-dataflow with Apache License 2.0 6 votes vote down vote up
public JdbcDataflowTaskExecutionMetadataDao(DataSource dataSource,
		DataFieldMaxValueIncrementer incrementer) {

	this.incrementer = incrementer;

	this.jdbcTemplate = new NamedParameterJdbcTemplate(dataSource);

	this.objectMapper = new ObjectMapper();
	SimpleModule module = new SimpleModule();
	module.addDeserializer(Resource.class,
			new ResourceDeserializer(new AppResourceCommon(new MavenProperties(), new DefaultResourceLoader())));
	this.objectMapper.registerModule(module);
	this.objectMapper.addMixIn(Resource.class, ResourceMixin.class);
	this.objectMapper.addMixIn(AppDefinition.class, AppDefinitionMixin.class);
	this.objectMapper.addMixIn(AppDeploymentRequest.class, AppDeploymentRequestMixin.class);
	this.objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);

	this.dataSource = dataSource;
}
 
Example #2
Source File: ServerDependencies.java    From spring-cloud-skipper with Apache License 2.0 5 votes vote down vote up
@Bean
public DelegatingResourceLoader delegatingResourceLoader(MavenProperties mavenProperties) {
	DockerResourceLoader dockerLoader = new DockerResourceLoader();
	MavenResourceLoader mavenResourceLoader = new MavenResourceLoader(mavenProperties);
	Map<String, ResourceLoader> loaders = new HashMap<>();
	loaders.put("docker", dockerLoader);
	loaders.put("maven", mavenResourceLoader);
	return new DelegatingResourceLoader(loaders);
}
 
Example #3
Source File: JobDependencies.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
@Bean
@ConditionalOnMissingBean
public AppRegistryService appRegistryService(AppRegistrationRepository appRegistrationRepository,
		AuditRecordService auditRecordService) {
	return new DefaultAppRegistryService(appRegistrationRepository,
			new AppResourceCommon(new MavenProperties(), new DefaultResourceLoader()), auditRecordService);
}
 
Example #4
Source File: TestDependencies.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
@Bean
public DelegatingResourceLoader resourceLoader(MavenProperties mavenProperties) {
	Map<String, ResourceLoader> resourceLoaders = new HashMap<>();
	resourceLoaders.put("maven", new MavenResourceLoader(mavenProperties));
	resourceLoaders.put("file", new FileSystemResourceLoader());

	DelegatingResourceLoader delegatingResourceLoader = new DelegatingResourceLoader(resourceLoaders);
	return delegatingResourceLoader;
}
 
Example #5
Source File: DockerRegistryValidator.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
public DockerRegistryValidator(DockerValidatorProperties dockerValidatorProperties,
		DockerResource dockerResource) {
	this.dockerValidatiorProperties = dockerValidatorProperties;
	this.dockerResource = dockerResource;
	this.restTemplate = configureRestTemplate();
	this.dockerAuth = getDockerAuth();
	this.appResourceCommon =  new AppResourceCommon(new MavenProperties(), null);
}
 
Example #6
Source File: LocalDataFlowServerAutoConfiguration.java    From spring-cloud-dataflow with Apache License 2.0 5 votes vote down vote up
@Bean
public DelegatingResourceLoader delegatingResourceLoader(MavenProperties mavenProperties) {
	DockerResourceLoader dockerLoader = new DockerResourceLoader();
	MavenResourceLoader mavenResourceLoader = new MavenResourceLoader(mavenProperties);
	Map<String, ResourceLoader> loaders = new HashMap<>();
	loaders.put("docker", dockerLoader);
	loaders.put("maven", mavenResourceLoader);
	return new DelegatingResourceLoader(loaders);
}
 
Example #7
Source File: ResourceLoadingAutoConfigurationTests.java    From spring-cloud-deployer with Apache License 2.0 5 votes vote down vote up
@Test
public void testMavenProperties() {
	this.contextRunner
			.withPropertyValues("maven.offline=true")
			.run((context) -> {
				assertThat(context).getBean(MavenProperties.class).has(offlineCondition);
			});
}
 
Example #8
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestampAsHdfsResource() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource base = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();
	copyFile(base, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource resource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/timestamp-task-1.0.0.BUILD-SNAPSHOT-exec.jar");

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example #9
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskTimestamp() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));
}
 
Example #10
Source File: SkipperServerConfiguration.java    From spring-cloud-skipper with Apache License 2.0 5 votes vote down vote up
@Bean
public DelegatingResourceLoader delegatingResourceLoader(MavenProperties mavenProperties) {
	DockerResourceLoader dockerLoader = new DockerResourceLoader();
	MavenResourceLoader mavenResourceLoader = new MavenResourceLoader(mavenProperties);
	Map<String, ResourceLoader> loaders = new HashMap<>();
	loaders.put("docker", dockerLoader);
	loaders.put("maven", mavenResourceLoader);
	return new DelegatingResourceLoader(loaders);
}
 
Example #11
Source File: TestDependencies.java    From spring-cloud-dashboard with Apache License 2.0 5 votes vote down vote up
@Bean
public ResourceLoader resourceLoader() {
	MavenProperties mavenProperties = new MavenProperties();
	mavenProperties.setRemoteRepositories(new HashMap<>(Collections.singletonMap("springRepo",
			new MavenProperties.RemoteRepository("https://repo.spring.io/libs-snapshot"))));

	Map<String, ResourceLoader> resourceLoaders = new HashMap<>();
	resourceLoaders.put("maven", new MavenResourceLoader(mavenProperties));
	resourceLoaders.put("file", new FileSystemResourceLoader());

	DelegatingResourceLoader delegatingResourceLoader = new DelegatingResourceLoader(resourceLoaders);
	return delegatingResourceLoader;
}
 
Example #12
Source File: AppResourceCommon.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
public AppResourceCommon(MavenProperties mavenProperties, ResourceLoader resourceLoader) {
	Assert.notNull(mavenProperties, "Non null Maven Properties are required!");
	this.mavenProperties = mavenProperties;
	this.metadataResourceLoader = resourceLoader;
}
 
Example #13
Source File: DeployerConfiguration.java    From spring-cloud-cli with Apache License 2.0 4 votes vote down vote up
@Bean
public MavenResourceLoader mavenResourceLoader(MavenProperties mavenProperties) {
	return new MavenResourceLoader(mavenProperties);
}
 
Example #14
Source File: DeployerConfiguration.java    From spring-cloud-cli with Apache License 2.0 4 votes vote down vote up
@ConfigurationProperties(prefix = "spring.cloud.maven")
@Bean
public MavenProperties mavenProperties() {
	return new MavenProperties();
}
 
Example #15
Source File: TestDependencies.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
@Bean
public AppResourceCommon appResourceService(MavenProperties mavenProperties,
		DelegatingResourceLoader delegatingResourceLoader) {
	return new AppResourceCommon(mavenProperties, delegatingResourceLoader);
}
 
Example #16
Source File: DataFlowControllerAutoConfiguration.java    From spring-cloud-dashboard with Apache License 2.0 4 votes vote down vote up
@Bean
public MavenResourceLoader mavenResourceLoader(MavenProperties properties) {
	return new MavenResourceLoader(properties);
}
 
Example #17
Source File: DataFlowControllerAutoConfiguration.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
@Bean
public AppResourceCommon appResourceCommon(@Nullable MavenProperties mavenProperties,
		DelegatingResourceLoader delegatingResourceLoader) {
	return new AppResourceCommon(mavenProperties, delegatingResourceLoader);
}
 
Example #18
Source File: AbstractMigrateUriRegistrySqlCommand.java    From spring-cloud-dataflow with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a migration data from URI_REGISTRY table to get inserted
 * into app_registration table. We're working on a raw data level
 * thus no use hibernate, spring repositories nor entity classes.
 *
 * @param jdbcTemplate the jdbc template
 * @return the list
 */
protected List<AppRegistrationMigrationData> createAppRegistrationMigrationData(JdbcTemplate jdbcTemplate) {
	Map<String, AppRegistrationMigrationData> data = new HashMap<>();
	// to get version using same existing logic
	AppResourceCommon arc = new AppResourceCommon(new MavenProperties(), new DelegatingResourceLoader());
	// track that we only add default version for one by just giving it to first one
	// just to be on a safe side
	Collection<String> defaultVersions = new HashSet<>();

	// format for old URI_REGISTRY is:
	// +-------------------------------+-------------------------------------------------------------------------------------------------+
	// | NAME                          | URI                                                                                             |
	// +-------------------------------+-------------------------------------------------------------------------------------------------+
	// | processor.bridge              | maven://org.springframework.cloud.stream.app:bridge-processor-rabbit:2.0.1.RELEASE              |
	// | processor.bridge.metadata     | maven://org.springframework.cloud.stream.app:bridge-processor-rabbit:jar:metadata:2.0.1.RELEASE |
	//
	// this needs to go to a new app_registration table structure where
	// uri and metadata and other fields are within same record.
	// AppRegistrationMigrationData is base of that info which then
	// gets actually updates with updateAppRegistration method.

	jdbcTemplate.query("select NAME, URI from URI_REGISTRY", rs -> {
		AppRegistrationMigrationData armd;
		String name = rs.getString(1);
		String uri = rs.getString(2);
		// we assume i.e processor.bridge and processor.bridge.metadata belong together
		String[] split = name.split("\\.");
		if (split.length == 2 || split.length == 3) {
			String key = split[0] + split[1];
			armd = data.getOrDefault(key, new AppRegistrationMigrationData());
			if (!defaultVersions.contains(key)) {
				armd.setDefaultVersion(true);
				defaultVersions.add(key);
			}
			if (split.length == 2) {
				// we got *.* first
				armd.setName(split[1]);
				armd.setType(ApplicationType.valueOf(split[0]).ordinal());
				armd.setUri(uri);
				try {
					armd.setVersion(arc.getResourceVersion(arc.getResource(uri)));
				}
				catch (Exception e) {
					logger.warn("Skipping URI_REGISTRY item {} migration with URI {} due to lack of version number in the URI", name, uri, e);
				}
			}
			else {
				// we got *.*.metadata first
				armd.setName(split[1]);
				armd.setType(ApplicationType.valueOf(split[0]).ordinal());
				armd.setMetadataUri(uri);
			}
			// either *.* or *.*.metadata, put it to map, either initial
			// insert or update
			data.put(key, armd);
		}
	});
	// if we skipper version errors, filter those out
	return data.values().stream().filter(i -> !ObjectUtils.isEmpty(i.getVersion())).collect(Collectors.toList());
}
 
Example #19
Source File: DataFlowControllerAutoConfiguration.java    From spring-cloud-dashboard with Apache License 2.0 4 votes vote down vote up
@Bean
public MavenProperties mavenProperties() {
	return new MavenConfigurationProperties();
}
 
Example #20
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testSmokeDeployer() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());
	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));
	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	for (int i = 0; i < 10; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(timeId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}
	deployer.undeploy(logId);
	for (int i = 0; i < 500; i++) {
		deployer.status(timeId);
		deployer.status(logId);
	}

	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #21
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLog() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResource = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #22
Source File: AbstractIntegrationTests.java    From spring-cloud-deployer with Apache License 2.0 4 votes vote down vote up
@Bean
@ConfigurationProperties("maven")
public MavenProperties mavenProperties() {
	return new MavenProperties();
}
 
Example #23
Source File: AbstractSchedulerIntegrationTests.java    From spring-cloud-deployer with Apache License 2.0 4 votes vote down vote up
@Bean
@ConfigurationProperties("maven")
public MavenProperties mavenProperties() {
	return new MavenProperties();
}
 
Example #24
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeLogAsHdfsResource() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource logResourceBase = new MavenResource.Builder(m2Properties)
			.artifactId("log-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	copyFile(timeResourceBase, "/dataflow/artifacts/repo/");
	copyFile(logResourceBase, "/dataflow/artifacts/repo/");

	@SuppressWarnings("resource")
	HdfsResourceLoader resourceLoader = new HdfsResourceLoader(getConfiguration());
	resourceLoader.setHandleNoprefix(true);
	Resource timeResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/time-source-1.0.0.BUILD-SNAPSHOT-exec.jar");
	Resource logResource = resourceLoader.getResource("hdfs:/dataflow/artifacts/repo/log-sink-1.0.0.BUILD-SNAPSHOT-exec.jar");

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "ticktock.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> logProperties = new HashMap<>();
	logProperties.put("spring.cloud.stream.bindings.input.destination", "ticktock.0");
	logProperties.put("expression", "new String(payload + ' hello')");
	AppDefinition logDefinition = new AppDefinition("log", logProperties);
	Map<String, String> logEnvironmentProperties = new HashMap<>();
	logEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	logEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "ticktock");
	AppDeploymentRequest logRequest = new AppDeploymentRequest(logDefinition, logResource, logEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String logId = deployer.deploy(logRequest);
	assertThat(logId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started LogSinkApplication");
	assertThat(deployer.status(logId).getState(), is(DeploymentState.deployed));

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "hello");

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.ticktock.0");
	deployer.undeploy(logId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.ticktock.0");

	assertThat(deployer.status(timeId).getState(), is(DeploymentState.unknown));
	assertThat(deployer.status(logId).getState(), is(DeploymentState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #25
Source File: ResourceLoadingAutoConfiguration.java    From spring-cloud-deployer with Apache License 2.0 4 votes vote down vote up
@Bean
@Order(0)
public DelegatingResourceLoaderBuilderCustomizer mavenDelegatingResourceLoaderBuilderCustomizer(MavenProperties mavenProperties) {
	return customizer -> customizer.loader("maven", new MavenResourceLoader(mavenProperties));
}
 
Example #26
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCancel() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> properties = new HashMap<String, String>();
	// let it run max 60 sec to get status and cancel.
	properties.put("repeat", "60");
	AppDefinition definition = new AppDefinition("timestamp-task", properties);
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Sleeping");

	assertThat(deployer.status(id).getState(), is(LaunchState.running));
	deployer.cancel(id);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Stopping beans");
	assertThat(deployer.status(id).getState(), is(LaunchState.unknown));

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		} else if (file.getName().endsWith("Container.stderr")) {
			assertThat("stderr with content: " + content, file.length(), is(0l));
		}
	}
}
 
Example #27
Source File: AppDeployerIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testStreamTimeHdfs() throws Exception {
	assertThat(context.containsBean("appDeployer"), is(true));
	assertThat(context.getBean("appDeployer"), instanceOf(YarnAppDeployer.class));
	AppDeployer deployer = context.getBean("appDeployer", AppDeployer.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);
	String fsUri = getConfiguration().get("fs.defaultFS");

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource timeResource = new MavenResource.Builder(m2Properties)
			.artifactId("time-source")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	MavenResource hdfsResource = new MavenResource.Builder(m2Properties)
			.artifactId("hdfs-sink")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	Map<String, String> timeProperties = new HashMap<>();
	timeProperties.put("spring.cloud.stream.bindings.output.destination", "timehdfs.0");
	AppDefinition timeDefinition = new AppDefinition("time", timeProperties);
	Map<String, String> timeEnvironmentProperties = new HashMap<>();
	timeEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	timeEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest timeRequest = new AppDeploymentRequest(timeDefinition, timeResource, timeEnvironmentProperties);

	Map<String, String> hdfsProperties = new HashMap<>();
	hdfsProperties.put("spring.cloud.stream.bindings.input.destination", "timehdfs.0");
	hdfsProperties.put("spring.hadoop.fsUri", fsUri);
	AppDefinition hdfsDefinition = new AppDefinition("log", hdfsProperties);
	Map<String, String> hdfsEnvironmentProperties = new HashMap<>();
	hdfsEnvironmentProperties.put(AppDeployer.COUNT_PROPERTY_KEY, "1");
	hdfsEnvironmentProperties.put(AppDeployer.GROUP_PROPERTY_KEY, "timehdfs");
	AppDeploymentRequest hdfsRequest = new AppDeploymentRequest(hdfsDefinition, hdfsResource, hdfsEnvironmentProperties);


	String timeId = deployer.deploy(timeRequest);
	assertThat(timeId, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TimeSourceApplication");
	assertThat(deployer.status(timeId).getState(), is(DeploymentState.deployed));

	String hdfsId = deployer.deploy(hdfsRequest);
	assertThat(hdfsId, notNullValue());

	assertWaitFileContent(1, TimeUnit.MINUTES, applicationId, "Started HdfsSinkApplication");
	assertThat(deployer.status(hdfsId).getState(), is(DeploymentState.deployed));

	waitHdfsFile("/tmp/hdfs-sink/data-0.txt", 2, TimeUnit.MINUTES);

	deployer.undeploy(timeId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped outbound.timehdfs.0");
	deployer.undeploy(hdfsId);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "stopped inbound.timehdfs.0");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(6));
}
 
Example #28
Source File: TaskLauncherIT.java    From spring-cloud-deployer-yarn with Apache License 2.0 4 votes vote down vote up
@Test
public void testTaskTimestampCommandlineArgs() throws Exception {
	assertThat(context.containsBean("taskLauncher"), is(true));
	assertThat(context.getBean("taskLauncher"), instanceOf(YarnTaskLauncher.class));
	TaskLauncher deployer = context.getBean("taskLauncher", TaskLauncher.class);
	YarnCloudAppService yarnCloudAppService = context.getBean(YarnCloudAppService.class);

	MavenProperties m2Properties = new MavenProperties();
	Map<String, RemoteRepository> remoteRepositories = new HashMap<>();
	remoteRepositories.put("default", new RemoteRepository("https://repo.spring.io/libs-snapshot-local"));
	m2Properties.setRemoteRepositories(remoteRepositories);

	MavenResource resource = new MavenResource.Builder(m2Properties)
			.artifactId("timestamp-task")
			.groupId(GROUP_ID)
			.version(artifactVersion)
			.extension("jar")
			.classifier("exec")
			.build();

	AppDefinition definition = new AppDefinition("timestamp-task", null);
	List<String> commandlineArgs = new ArrayList<String>();
	commandlineArgs.add("--format=yyyyMMdd yyyy");
	AppDeploymentRequest request = new AppDeploymentRequest(definition, resource, null, commandlineArgs);
	String id = deployer.launch(request);
	assertThat(id, notNullValue());

	ApplicationId applicationId = assertWaitApp(2, TimeUnit.MINUTES, yarnCloudAppService);
	assertWaitFileContent(2, TimeUnit.MINUTES, applicationId, "Started TaskApplication");

	List<Resource> resources = ContainerLogUtils.queryContainerLogs(
			getYarnCluster(), applicationId);

	assertThat(resources, notNullValue());
	assertThat(resources.size(), is(4));

	for (Resource res : resources) {
		File file = res.getFile();
		String content = ContainerLogUtils.getFileContent(file);
		if (file.getName().endsWith("stdout")) {
			assertThat(file.length(), greaterThan(0l));
		}
		if (file.getName().endsWith("Container.stdout")) {
			SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd yyyy");
			String expect = format.format(new Date());
			assertThat(content, containsString(expect));
		}
	}
}