org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalResourceStatusPBImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void mergeLocalToBuilder() {
  if (this.resource != null &&
      !((LocalResourcePBImpl)this.resource).getProto()
        .equals(builder.getResource())) {
    builder.setResource(convertToProtoFormat(this.resource));
  }
  if (this.localPath != null &&
      !((URLPBImpl)this.localPath).getProto()
        .equals(builder.getLocalPath())) {
    builder.setLocalPath(convertToProtoFormat(this.localPath));
  }
  if (this.exception != null &&
      !((SerializedExceptionPBImpl)this.exception).getProto()
        .equals(builder.getException())) {
    builder.setException(convertToProtoFormat(this.exception));
  }
}
 
Example #2
Source File: ResourceLocalizationSpecPBImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void mergeLocalToBuilder() {
  ResourceLocalizationSpecProtoOrBuilder l = viaProto ? proto : builder;
  if (this.resource != null
      && !(l.getResource()
        .equals(((LocalResourcePBImpl) resource).getProto()))) {
    maybeInitBuilder();
    builder.setResource(((LocalResourcePBImpl) resource).getProto());
  }
  if (this.destinationDirectory != null
      && !(l.getDestinationDirectory()
        .equals(((URLPBImpl) destinationDirectory).getProto()))) {
    maybeInitBuilder();
    builder.setDestinationDirectory(((URLPBImpl) destinationDirectory)
      .getProto());
  }
}
 
Example #3
Source File: LocalResourceStatusPBImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void mergeLocalToBuilder() {
  if (this.resource != null &&
      !((LocalResourcePBImpl)this.resource).getProto()
        .equals(builder.getResource())) {
    builder.setResource(convertToProtoFormat(this.resource));
  }
  if (this.localPath != null &&
      !((URLPBImpl)this.localPath).getProto()
        .equals(builder.getLocalPath())) {
    builder.setLocalPath(convertToProtoFormat(this.localPath));
  }
  if (this.exception != null &&
      !((SerializedExceptionPBImpl)this.exception).getProto()
        .equals(builder.getException())) {
    builder.setException(convertToProtoFormat(this.exception));
  }
}
 
Example #4
Source File: ResourceLocalizationSpecPBImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void mergeLocalToBuilder() {
  ResourceLocalizationSpecProtoOrBuilder l = viaProto ? proto : builder;
  if (this.resource != null
      && !(l.getResource()
        .equals(((LocalResourcePBImpl) resource).getProto()))) {
    maybeInitBuilder();
    builder.setResource(((LocalResourcePBImpl) resource).getProto());
  }
  if (this.destinationDirectory != null
      && !(l.getDestinationDirectory()
        .equals(((URLPBImpl) destinationDirectory).getProto()))) {
    maybeInitBuilder();
    builder.setDestinationDirectory(((URLPBImpl) destinationDirectory)
      .getProto());
  }
}
 
Example #5
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
    List<PlanLocalResource> localResourcesList) {
  Map<String, LocalResource> map = new HashMap<String, LocalResource>();
  for(PlanLocalResource res : localResourcesList){
    LocalResource r = new LocalResourcePBImpl();

    //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
    //else we will receive a default value back, eg ""
    if(res.hasPattern()){
      r.setPattern(res.getPattern());
    }
    r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri())));  // see above notes on HDFS URL handling
    r.setSize(res.getSize());
    r.setTimestamp(res.getTimeStamp());
    r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
    r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
    map.put(res.getName(), r);
  }
  return map;
}
 
Example #6
Source File: DagTypeConverters.java    From tez with Apache License 2.0 6 votes vote down vote up
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
    List<PlanLocalResource> localResourcesList) {
  Map<String, LocalResource> map = new HashMap<String, LocalResource>();
  for(PlanLocalResource res : localResourcesList){
    LocalResource r = new LocalResourcePBImpl();

    //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
    //else we will receive a default value back, eg ""
    if(res.hasPattern()){
      r.setPattern(res.getPattern());
    }
    r.setResource(convertToYarnURL(res.getUri()));
    r.setSize(res.getSize());
    r.setTimestamp(res.getTimeStamp());
    r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
    r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
    map.put(res.getName(), r);
  }
  return map;
}
 
Example #7
Source File: TestPBRecordImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
static LocalResource createResource() {
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  assertTrue(ret instanceof LocalResourcePBImpl);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
    "hdfs://y.ak:8020/foo/bar")));
  ret.setSize(4344L);
  ret.setTimestamp(3141592653589793L);
  ret.setVisibility(LocalResourceVisibility.PUBLIC);
  return ret;
}
 
Example #8
Source File: TestPBRecordImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LocalResource createResource() {
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  assertTrue(ret instanceof LocalResourcePBImpl);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
    "hdfs://y.ak:8020/foo/bar")));
  ret.setSize(4344L);
  ret.setTimestamp(3141592653589793L);
  ret.setVisibility(LocalResourceVisibility.PUBLIC);
  return ret;
}
 
Example #9
Source File: LocalResourceStatusPBImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private LocalResourceProto convertToProtoFormat(LocalResource rsrc) {
  return ((LocalResourcePBImpl)rsrc).getProto();
}
 
Example #10
Source File: LocalResourceStatusPBImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto rsrc) {
  return new LocalResourcePBImpl(rsrc);
}
 
Example #11
Source File: TestLocalResourcesTrackerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  DeletionService mockDelService = mock(DeletionService.class);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    // Simulate lr1 getting localized
    ResourceLocalizedEvent rle1 =
        new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
    tracker.handle(rle1);
    dispatcher.await();

    ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
        ArgumentCaptor.forClass(LocalizedResourceProto.class);
    verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
        localizedProtoCaptor.capture());
    LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
    Assert.assertEquals(lr1, new LocalResourceRequest(
        new LocalResourcePBImpl(localizedProto.getResource())));
    Assert.assertEquals(localizedPath1.toString(),
        localizedProto.getLocalPath());
    LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
    Assert.assertNotNull(localizedRsrc1);

    // simulate release and retention processing
    tracker.handle(new ResourceReleaseEvent(lr1, cId1));
    dispatcher.await();
    boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);

    Assert.assertTrue(removeResult);
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example #12
Source File: TestLocalResourcesTrackerImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    ResourceFailedLocalizationEvent rfe1 =
        new ResourceFailedLocalizationEvent(
            lr1, new Exception("Test").toString());
    tracker.handle(rfe1);
    dispatcher.await();
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example #13
Source File: LocalResourceStatusPBImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
private LocalResourceProto convertToProtoFormat(LocalResource rsrc) {
  return ((LocalResourcePBImpl)rsrc).getProto();
}
 
Example #14
Source File: LocalResourceStatusPBImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto rsrc) {
  return new LocalResourcePBImpl(rsrc);
}
 
Example #15
Source File: TestLocalResourcesTrackerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  DeletionService mockDelService = mock(DeletionService.class);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    // Simulate lr1 getting localized
    ResourceLocalizedEvent rle1 =
        new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
    tracker.handle(rle1);
    dispatcher.await();

    ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
        ArgumentCaptor.forClass(LocalizedResourceProto.class);
    verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
        localizedProtoCaptor.capture());
    LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
    Assert.assertEquals(lr1, new LocalResourceRequest(
        new LocalResourcePBImpl(localizedProto.getResource())));
    Assert.assertEquals(localizedPath1.toString(),
        localizedProto.getLocalPath());
    LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
    Assert.assertNotNull(localizedRsrc1);

    // simulate release and retention processing
    tracker.handle(new ResourceReleaseEvent(lr1, cId1));
    dispatcher.await();
    boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);

    Assert.assertTrue(removeResult);
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example #16
Source File: TestLocalResourcesTrackerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    ResourceFailedLocalizationEvent rfe1 =
        new ResourceFailedLocalizationEvent(
            lr1, new Exception("Test").toString());
    tracker.handle(rfe1);
    dispatcher.await();
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}