Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResourceType#FILE

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResourceType#FILE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestResourceRetention.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService(),null));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
Example 2
Source File: StreamAppmaster.java    From spring-cloud-deployer-yarn with Apache License 2.0 6 votes vote down vote up 
@Override
public ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projectionData,
		Map<String, Object> extraProperties) {
	log.info("intercept createContainerCluster " + clusterId);
	String artifactPath = streamAppmasterProperties.getArtifact();
	try {
		LocalResourcesFactoryBean lrfb = new LocalResourcesFactoryBean();
		lrfb.setConfiguration(getConfiguration());

		String containerArtifact = (String) extraProperties.get("containerArtifact");
		TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifactPath + "/" + containerArtifact, false);
		ArrayList<TransferEntry> hdfsEntries = new ArrayList<TransferEntry>();
		hdfsEntries.add(te);
		lrfb.setHdfsEntries(hdfsEntries);
		lrfb.setCopyEntries(new ArrayList<CopyEntry>());
		lrfb.afterPropertiesSet();
		ResourceLocalizer rl = lrfb.getObject();
		log.info("Adding localizer for " + clusterId + " / " + rl);
		artifactLocalizers.put(clusterId, rl);
	} catch (Exception e) {
		log.error("Error creating localizer", e);
	}

	return super.createContainerCluster(clusterId, clusterDef, projectionData, extraProperties);
}
Example 3
Source File: TestResourceRetention.java    From big-c with Apache License 2.0 6 votes vote down vote up 
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
Example 4
Source File: TestLocalResourcesTrackerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private LocalResourceRequest createLocalResourceRequest(String user, int i,
    long ts, LocalResourceVisibility vis) {
  final LocalResourceRequest req =
      new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
          ts + i * 2000, LocalResourceType.FILE, vis, null);
  return req;
}
Example 5
Source File: TaskAppmaster.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up 
private ResourceLocalizer buildArtifactResourceLocalizer() throws Exception {
	String artifact = taskAppmasterProperties.getArtifact();
	log.info("Building localizer for artifact " + artifact);
	LocalResourcesFactoryBean fb = new LocalResourcesFactoryBean();
	fb.setConfiguration(getConfiguration());
	TransferEntry te = new TransferEntry(LocalResourceType.FILE, null, artifact, false);
	ArrayList<TransferEntry> hdfsEntries = new ArrayList<TransferEntry>();
	hdfsEntries.add(te);
	fb.setHdfsEntries(hdfsEntries);
	fb.setCopyEntries(new ArrayList<CopyEntry>());
	fb.afterPropertiesSet();
	return fb.getObject();
}
Example 6
Source File: TestLocalResourcesTrackerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private LocalResourceRequest createLocalResourceRequest(String user, int i,
    long ts, LocalResourceVisibility vis) {
  final LocalResourceRequest req =
      new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
          ts + i * 2000, LocalResourceType.FILE, vis, null);
  return req;
}
Example 7
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
  switch(type){
  case ARCHIVE : return LocalResourceType.ARCHIVE;
  case FILE : return LocalResourceType.FILE;
  case PATTERN : return LocalResourceType.PATTERN;
  default : throw new IllegalArgumentException("unknown 'type': " + type);
  }
}
Example 8
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
  switch(type){
  case ARCHIVE : return LocalResourceType.ARCHIVE;
  case FILE : return LocalResourceType.FILE;
  case PATTERN : return LocalResourceType.PATTERN;
  default : throw new IllegalArgumentException("unknown 'type': " + type);
  }
}