org.apache.hadoop.hdfs.inotify.Event Java Examples

The following examples show how to use org.apache.hadoop.hdfs.inotify.Event. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void makeSureHappyPathForProcessingEventsSendsFlowFilesToCorrectRelationship() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path(/)?.*");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(3, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #2
Source File: PBHelper.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static InotifyProtos.MetadataUpdateType metadataUpdateTypeConvert(
    Event.MetadataUpdateEvent.MetadataType type) {
  switch (type) {
  case TIMES:
    return InotifyProtos.MetadataUpdateType.META_TYPE_TIMES;
  case REPLICATION:
    return InotifyProtos.MetadataUpdateType.META_TYPE_REPLICATION;
  case OWNER:
    return InotifyProtos.MetadataUpdateType.META_TYPE_OWNER;
  case PERMS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_PERMS;
  case ACLS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_ACLS;
  case XATTRS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_XATTRS;
  default:
    return null;
  }
}
 
Example #3
Source File: PBHelper.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static Event.MetadataUpdateEvent.MetadataType metadataUpdateTypeConvert(
    InotifyProtos.MetadataUpdateType type) {
  switch (type) {
  case META_TYPE_TIMES:
    return Event.MetadataUpdateEvent.MetadataType.TIMES;
  case META_TYPE_REPLICATION:
    return Event.MetadataUpdateEvent.MetadataType.REPLICATION;
  case META_TYPE_OWNER:
    return Event.MetadataUpdateEvent.MetadataType.OWNER;
  case META_TYPE_PERMS:
    return Event.MetadataUpdateEvent.MetadataType.PERMS;
  case META_TYPE_ACLS:
    return Event.MetadataUpdateEvent.MetadataType.ACLS;
  case META_TYPE_XATTRS:
    return Event.MetadataUpdateEvent.MetadataType.XATTRS;
  default:
    return null;
  }
}
 
Example #4
Source File: PBHelper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static Event.MetadataUpdateEvent.MetadataType metadataUpdateTypeConvert(
    InotifyProtos.MetadataUpdateType type) {
  switch (type) {
  case META_TYPE_TIMES:
    return Event.MetadataUpdateEvent.MetadataType.TIMES;
  case META_TYPE_REPLICATION:
    return Event.MetadataUpdateEvent.MetadataType.REPLICATION;
  case META_TYPE_OWNER:
    return Event.MetadataUpdateEvent.MetadataType.OWNER;
  case META_TYPE_PERMS:
    return Event.MetadataUpdateEvent.MetadataType.PERMS;
  case META_TYPE_ACLS:
    return Event.MetadataUpdateEvent.MetadataType.ACLS;
  case META_TYPE_XATTRS:
    return Event.MetadataUpdateEvent.MetadataType.XATTRS;
  default:
    return null;
  }
}
 
Example #5
Source File: PBHelper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static InotifyProtos.MetadataUpdateType metadataUpdateTypeConvert(
    Event.MetadataUpdateEvent.MetadataType type) {
  switch (type) {
  case TIMES:
    return InotifyProtos.MetadataUpdateType.META_TYPE_TIMES;
  case REPLICATION:
    return InotifyProtos.MetadataUpdateType.META_TYPE_REPLICATION;
  case OWNER:
    return InotifyProtos.MetadataUpdateType.META_TYPE_OWNER;
  case PERMS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_PERMS;
  case ACLS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_ACLS;
  case XATTRS:
    return InotifyProtos.MetadataUpdateType.META_TYPE_XATTRS;
  default:
    return null;
  }
}
 
Example #6
Source File: EventTypeValidator.java    From nifi with Apache License 2.0 6 votes vote down vote up
private String isValidEventType(String input) {
    if (input != null && !"".equals(input.trim())) {
        final String[] events = input.split(",");
        final List<String> invalid = new ArrayList<>();
        for (String event : events) {
            try {
                Event.EventType.valueOf(event.trim().toUpperCase());
            } catch (IllegalArgumentException e) {
                invalid.add(event.trim());
            }
        }

        return invalid.isEmpty() ? null : "The following are not valid event types: " + invalid;
    }

    return "Empty event types are not allowed.";
}
 
Example #7
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldProperlyHandleAnEmptyEventBatch() throws Exception {
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(new Event[]{});

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path");
    runner.setProperty(GetHDFSEvents.NUMBER_OF_RETRIES_FOR_POLL, "5");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #8
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldOnlyProcessEventsWithSpecificPath() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/create(/)?");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #9
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldOnlyProcessEventsWithSpecificPath() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/create(/)?");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #10
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void makeSureHappyPathForProcessingEventsSendsFlowFilesToCorrectRelationship() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path(/)?.*");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(3, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #11
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void onTriggerShouldProperlyHandleAnEmptyEventBatch() throws Exception {
    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(new Event[]{});

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.POLL_DURATION, "1 second");
    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path");
    runner.setProperty(GetHDFSEvents.NUMBER_OF_RETRIES_FOR_POLL, "5");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(0, successfulFlowFiles.size());
    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #12
Source File: EventTypeValidator.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
private String isValidEventType(String input) {
    if (input != null && !"".equals(input.trim())) {
        final String[] events = input.split(",");
        final List<String> invalid = new ArrayList<>();
        for (String event : events) {
            try {
                Event.EventType.valueOf(event.trim().toUpperCase());
            } catch (IllegalArgumentException e) {
                invalid.add(event.trim());
            }
        }

        return invalid.isEmpty() ? null : "The following are not valid event types: " + invalid;
    }

    return "Empty event types are not allowed.";
}
 
Example #13
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch() throws Exception {
    Event[] events = new Event[] {
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/4/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/.t.txt").build()
    };

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/${literal(1)}/${literal(2)}/${literal(3)}/.*.txt");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create");
    runner.setProperty(GetHDFSEvents.IGNORE_HIDDEN_FILES, "true");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());

    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(eventType.equals("CREATE"));
    }

    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #14
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void eventsProcessorShouldProperlyFilterEventTypes() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path(/.*)?");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create, metadata");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(2, successfulFlowFiles.size());

    List<String> expectedEventTypes = Arrays.asList("CREATE", "METADATA");
    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(expectedEventTypes.contains(eventType));
    }

    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #15
Source File: TestGetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
private Event[] getEvents() {
    return new Event[]{
                EventTestUtils.createCreateEvent(),
                EventTestUtils.createCloseEvent(),
                EventTestUtils.createMetadataUpdateEvent()
        };
}
 
Example #16
Source File: TestDFSInotifyEventInputStream.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testReadEventsWithTimeout() throws IOException,
    InterruptedException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    final DFSClient client = new DFSClient(cluster.getDfsCluster()
        .getNameNode(0).getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    ScheduledExecutorService ex = Executors
        .newSingleThreadScheduledExecutor();
    ex.schedule(new Runnable() {
      @Override
      public void run() {
        try {
          client.mkdirs("/dir", null, false);
        } catch (IOException e) {
          // test will fail
          LOG.error("Unable to create /dir", e);
        }
      }
    }, 1, TimeUnit.SECONDS);
    // a very generous wait period -- the edit will definitely have been
    // processed by the time this is up
    EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
    Assert.assertNotNull(batch);
    Assert.assertEquals(1, batch.getEvents().length);
    Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
    Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
  } finally {
    cluster.shutdown();
  }
}
 
Example #17
Source File: EventTestUtils.java    From nifi with Apache License 2.0 5 votes vote down vote up
public static Event.CreateEvent createCreateEvent() {
    return new Event.CreateEvent.Builder()
            .ctime(new Date().getTime())
            .groupName("group_name")
            .iNodeType(Event.CreateEvent.INodeType.DIRECTORY)
            .overwrite(false)
            .ownerName("ownerName")
            .path("/some/path/create")
            .perms(new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE))
            .replication(1)
            .symlinkTarget("/some/symlink/target")
            .build();
}
 
Example #18
Source File: GetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
private String getPath(Event event) {
    if (event == null || event.getEventType() == null) {
        throw new IllegalArgumentException("Event and event type must not be null.");
    }

    switch (event.getEventType()) {
        case CREATE: return ((Event.CreateEvent) event).getPath();
        case CLOSE: return ((Event.CloseEvent) event).getPath();
        case APPEND: return ((Event.AppendEvent) event).getPath();
        case RENAME: return ((Event.RenameEvent) event).getSrcPath();
        case METADATA: return ((Event.MetadataUpdateEvent) event).getPath();
        case UNLINK: return ((Event.UnlinkEvent) event).getPath();
        default: throw new IllegalArgumentException("Unsupported event type.");
    }
}
 
Example #19
Source File: GetHDFSEvents.java    From nifi with Apache License 2.0 5 votes vote down vote up
private boolean toProcessEvent(ProcessContext context, Event event) {
    final String[] eventTypes = context.getProperty(EVENT_TYPES).getValue().split(",");
    for (String name : eventTypes) {
        if (name.trim().equalsIgnoreCase(event.getEventType().name())) {
            return notificationConfig.getPathFilter().accept(new Path(getPath(event)));
        }
    }

    return false;
}
 
Example #20
Source File: TestDFSInotifyEventInputStream.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testReadEventsWithTimeout() throws IOException,
    InterruptedException, MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    final DFSClient client = new DFSClient(cluster.getDfsCluster()
        .getNameNode(0).getNameNodeAddress(), conf);
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    ScheduledExecutorService ex = Executors
        .newSingleThreadScheduledExecutor();
    ex.schedule(new Runnable() {
      @Override
      public void run() {
        try {
          client.mkdirs("/dir", null, false);
        } catch (IOException e) {
          // test will fail
          LOG.error("Unable to create /dir", e);
        }
      }
    }, 1, TimeUnit.SECONDS);
    // a very generous wait period -- the edit will definitely have been
    // processed by the time this is up
    EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
    Assert.assertNotNull(batch);
    Assert.assertEquals(1, batch.getEvents().length);
    Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
    Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
  } finally {
    cluster.shutdown();
  }
}
 
Example #21
Source File: TestDFSInotifyEventInputStream.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 120000)
public void testNNFailover() throws IOException, URISyntaxException,
    MissingEventsException {
  Configuration conf = new HdfsConfiguration();
  MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).build();

  try {
    cluster.getDfsCluster().waitActive();
    cluster.getDfsCluster().transitionToActive(0);
    DFSClient client = ((DistributedFileSystem) HATestUtil.configureFailoverFs
        (cluster.getDfsCluster(), conf)).dfs;
    DFSInotifyEventInputStream eis = client.getInotifyEventStream();
    for (int i = 0; i < 10; i++) {
      client.mkdirs("/dir" + i, null, false);
    }
    cluster.getDfsCluster().shutdownNameNode(0);
    cluster.getDfsCluster().transitionToActive(1);
    EventBatch batch = null;
    // we can read all of the edits logged by the old active from the new
    // active
    for (int i = 0; i < 10; i++) {
      batch = waitForNextEvents(eis);
      Assert.assertEquals(1, batch.getEvents().length);
      Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
      Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
          i));
    }
    Assert.assertTrue(eis.poll() == null);
  } finally {
    cluster.shutdown();
  }
}
 
Example #22
Source File: PBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INodeType
    type) {
  switch (type) {
  case DIRECTORY:
    return InotifyProtos.INodeType.I_TYPE_DIRECTORY;
  case FILE:
    return InotifyProtos.INodeType.I_TYPE_FILE;
  case SYMLINK:
    return InotifyProtos.INodeType.I_TYPE_SYMLINK;
  default:
    return null;
  }
}
 
Example #23
Source File: EventTestUtils.java    From nifi with Apache License 2.0 5 votes vote down vote up
public static Event.MetadataUpdateEvent createMetadataUpdateEvent() {
    return new Event.MetadataUpdateEvent.Builder()
            .replication(0)
            .perms(new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE))
            .path("/some/path/metadata")
            .ownerName("owner")
            .acls(Collections.singletonList(new AclEntry.Builder().setName("schema").setPermission(FsAction.ALL).setScope(AclEntryScope.ACCESS).setType(AclEntryType.GROUP).build()))
            .atime(new Date().getTime())
            .groupName("groupName")
            .metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
            .mtime(1L)
            .xAttrs(Collections.singletonList(new XAttr.Builder().setName("name").setNameSpace(XAttr.NameSpace.USER).setValue(new byte[0]).build()))
            .xAttrsRemoved(false)
            .build();
}
 
Example #24
Source File: PBHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static Event.CreateEvent.INodeType createTypeConvert(InotifyProtos.INodeType
    type) {
  switch (type) {
  case I_TYPE_DIRECTORY:
    return Event.CreateEvent.INodeType.DIRECTORY;
  case I_TYPE_FILE:
    return Event.CreateEvent.INodeType.FILE;
  case I_TYPE_SYMLINK:
    return Event.CreateEvent.INodeType.SYMLINK;
  default:
    return null;
  }
}
 
Example #25
Source File: GetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
private String getPath(Event event) {
    if (event == null || event.getEventType() == null) {
        throw new IllegalArgumentException("Event and event type must not be null.");
    }

    switch (event.getEventType()) {
        case CREATE: return ((Event.CreateEvent) event).getPath();
        case CLOSE: return ((Event.CloseEvent) event).getPath();
        case APPEND: return ((Event.AppendEvent) event).getPath();
        case RENAME: return ((Event.RenameEvent) event).getSrcPath();
        case METADATA: return ((Event.MetadataUpdateEvent) event).getPath();
        case UNLINK: return ((Event.UnlinkEvent) event).getPath();
        default: throw new IllegalArgumentException("Unsupported event type.");
    }
}
 
Example #26
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void eventsProcessorShouldProperlyFilterEventTypes() throws Exception {
    Event[] events = getEvents();

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path(/.*)?");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create, metadata");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(2, successfulFlowFiles.size());

    List<String> expectedEventTypes = Arrays.asList("CREATE", "METADATA");
    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(expectedEventTypes.contains(eventType));
    }

    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #27
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void makeSureExpressionLanguageIsWorkingProperlyWithinTheHdfsPathToWatch() throws Exception {
    Event[] events = new Event[] {
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/4/t.txt").build(),
            new Event.CreateEvent.Builder().path("/some/path/1/2/3/.t.txt").build()
    };

    EventBatch eventBatch = mock(EventBatch.class);
    when(eventBatch.getEvents()).thenReturn(events);

    when(inotifyEventInputStream.poll(1000000L, TimeUnit.MICROSECONDS)).thenReturn(eventBatch);
    when(hdfsAdmin.getInotifyEventStream()).thenReturn(inotifyEventInputStream);
    when(eventBatch.getTxid()).thenReturn(100L);

    GetHDFSEvents processor = new TestableGetHDFSEvents(kerberosProperties, hdfsAdmin);
    TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(GetHDFSEvents.HDFS_PATH_TO_WATCH, "/some/path/${literal(1)}/${literal(2)}/${literal(3)}/.*.txt");
    runner.setProperty(GetHDFSEvents.EVENT_TYPES, "create");
    runner.setProperty(GetHDFSEvents.IGNORE_HIDDEN_FILES, "true");
    runner.run();

    List<MockFlowFile> successfulFlowFiles = runner.getFlowFilesForRelationship(GetHDFSEvents.REL_SUCCESS);
    assertEquals(1, successfulFlowFiles.size());

    for (MockFlowFile f : successfulFlowFiles) {
        String eventType = f.getAttribute(EventAttributes.EVENT_TYPE);
        assertTrue(eventType.equals("CREATE"));
    }

    verify(eventBatch).getTxid();
    assertEquals("100", runner.getProcessContext().getStateManager().getState(Scope.CLUSTER).get("last.tx.id"));
}
 
Example #28
Source File: TestGetHDFSEvents.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
private Event[] getEvents() {
    return new Event[]{
                EventTestUtils.createCreateEvent(),
                EventTestUtils.createCloseEvent(),
                EventTestUtils.createMetadataUpdateEvent()
        };
}
 
Example #29
Source File: EventTestUtils.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public static Event.CreateEvent createCreateEvent() {
    return new Event.CreateEvent.Builder()
            .ctime(new Date().getTime())
            .groupName("group_name")
            .iNodeType(Event.CreateEvent.INodeType.DIRECTORY)
            .overwrite(false)
            .ownerName("ownerName")
            .path("/some/path/create")
            .perms(new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE))
            .replication(1)
            .symlinkTarget("/some/symlink/target")
            .build();
}
 
Example #30
Source File: EventTestUtils.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
public static Event.MetadataUpdateEvent createMetadataUpdateEvent() {
    return new Event.MetadataUpdateEvent.Builder()
            .replication(0)
            .perms(new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE))
            .path("/some/path/metadata")
            .ownerName("owner")
            .acls(Collections.singletonList(new AclEntry.Builder().setName("schema").setPermission(FsAction.ALL).setScope(AclEntryScope.ACCESS).setType(AclEntryType.GROUP).build()))
            .atime(new Date().getTime())
            .groupName("groupName")
            .metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
            .mtime(1L)
            .xAttrs(Collections.singletonList(new XAttr.Builder().setName("name").setNameSpace(XAttr.NameSpace.USER).setValue(new byte[0]).build()))
            .xAttrsRemoved(false)
            .build();
}