org.apache.nifi.util.FileUtils Java Examples

The following examples show how to use org.apache.nifi.util.FileUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NarUnpacker.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Unpacks the NAR to the specified directory. Creates a checksum file that
 * used to determine if future expansion is necessary.
 *
 * @param workingDirectory
 *            the root directory to which the NAR should be unpacked.
 * @throws IOException
 *             if the NAR could not be unpacked.
 */
private static void unpack(final File nar, final File workingDirectory, final byte[] hash)
        throws IOException {

    try (JarFile jarFile = new JarFile(nar)) {
        Enumeration<JarEntry> jarEntries = jarFile.entries();
        while (jarEntries.hasMoreElements()) {
            JarEntry jarEntry = jarEntries.nextElement();
            String name = jarEntry.getName();
            File f = new File(workingDirectory, name);
            if (jarEntry.isDirectory()) {
                FileUtils.ensureDirectoryExistAndCanAccess(f);
            } else {
                makeFile(jarFile.getInputStream(jarEntry), f);
            }
        }
    }

    final File hashFile = new File(workingDirectory, HASH_FILENAME);
    try (final FileOutputStream fos = new FileOutputStream(hashFile)) {
        fos.write(hash);
    }
}
 
Example #2
Source File: NarUnpacker.java    From nifi-minifi with Apache License 2.0 6 votes vote down vote up
/**
 * Unpacks the NAR to the specified directory. Creates a checksum file that
 * used to determine if future expansion is necessary.
 *
 * @param workingDirectory
 *            the root directory to which the NAR should be unpacked.
 * @throws IOException
 *             if the NAR could not be unpacked.
 */
private static void unpack(final File nar, final File workingDirectory, final byte[] hash)
        throws IOException {

    try (JarFile jarFile = new JarFile(nar)) {
        Enumeration<JarEntry> jarEntries = jarFile.entries();
        while (jarEntries.hasMoreElements()) {
            JarEntry jarEntry = jarEntries.nextElement();
            String name = jarEntry.getName();
            File f = new File(workingDirectory, name);
            if (jarEntry.isDirectory()) {
                FileUtils.ensureDirectoryExistAndCanAccess(f);
            } else {
                makeFile(jarFile.getInputStream(jarEntry), f);
            }
        }
    }

    final File hashFile = new File(workingDirectory, HASH_FILENAME);
    try (final FileOutputStream fos = new FileOutputStream(hashFile)) {
        fos.write(hash);
    }
}
 
Example #3
Source File: NarUnpacker.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Unpacks the NAR to the specified directory. Creates a checksum file that
 * used to determine if future expansion is necessary.
 *
 * @param workingDirectory the root directory to which the NAR should be unpacked.
 * @throws IOException if the NAR could not be unpacked.
 */
private static void unpack(final File nar, final File workingDirectory, final byte[] hash) throws IOException {

    try (JarFile jarFile = new JarFile(nar)) {
        Enumeration<JarEntry> jarEntries = jarFile.entries();
        while (jarEntries.hasMoreElements()) {
            JarEntry jarEntry = jarEntries.nextElement();
            String name = jarEntry.getName();
            if(name.contains("META-INF/bundled-dependencies")){
                name = name.replace("META-INF/bundled-dependencies", BUNDLED_DEPENDENCIES_DIRECTORY);
            }
            File f = new File(workingDirectory, name);
            if (jarEntry.isDirectory()) {
                FileUtils.ensureDirectoryExistAndCanReadAndWrite(f);
            } else {
                makeFile(jarFile.getInputStream(jarEntry), f);
            }
        }
    }

    final File hashFile = new File(workingDirectory, HASH_FILENAME);
    try (final FileOutputStream fos = new FileOutputStream(hashFile)) {
        fos.write(hash);
    }
}
 
Example #4
Source File: NarUnpacker.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar
 *            the nar to unpack
 * @param baseWorkingDirectory
 *            the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException
 *             if unable to explode nar
 */
private static File unpackNar(final File nar, final File baseWorkingDirectory)
        throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.",
                        new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 
Example #5
Source File: NarUnpacker.java    From nifi-minifi with Apache License 2.0 5 votes vote down vote up
/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar
 *            the nar to unpack
 * @param baseWorkingDirectory
 *            the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException
 *             if unable to explode nar
 */
private static File unpackNar(final File nar, final File baseWorkingDirectory)
        throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.",
                        new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 
Example #6
Source File: NarAutoLoader.java    From nifi with Apache License 2.0 5 votes vote down vote up
public synchronized void start() throws IOException {
    if (started) {
        return;
    }

    FileUtils.ensureDirectoryExistAndCanReadAndWrite(autoLoadDir);

    final WatchService watcher = FileSystems.getDefault().newWatchService();

    final Path autoLoadPath = autoLoadDir.toPath();
    autoLoadPath.register(watcher, StandardWatchEventKinds.ENTRY_CREATE);

    narAutoLoaderTask = new NarAutoLoaderTask.Builder()
            .autoLoadPath(autoLoadPath)
            .watchService(watcher)
            .pollIntervalMillis(POLL_INTERVAL_MS)
            .narLoader(narLoader)
            .build();

    LOGGER.info("Starting NAR Auto-Loader for directory {} ...", new Object[]{autoLoadPath});

    final Thread thread = new Thread(narAutoLoaderTask);
    thread.setName("NAR Auto-Loader");
    thread.setDaemon(true);
    thread.start();

    LOGGER.info("NAR Auto-Loader started");
    started = true;
}
 
Example #7
Source File: FrameworkIntegrationTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
private void deleteDirectory(final File dir) throws IOException {
    if (!dir.exists()) {
        return;
    }

    FileUtils.deleteFile(dir, true);
}
 
Example #8
Source File: NarUnpacker.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Unpacks the specified nar into the specified base working directory.
 *
 * @param nar the nar to unpack
 * @param baseWorkingDirectory the directory to unpack to
 * @return the directory to the unpacked NAR
 * @throws IOException if unable to explode nar
 */
public static File unpackNar(final File nar, final File baseWorkingDirectory) throws IOException {
    final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");

    // if the working directory doesn't exist, unpack the nar
    if (!narWorkingDirectory.exists()) {
        unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
    } else {
        // the working directory does exist. Run MD5 sum against the nar
        // file and check if the nar has changed since it was deployed.
        final byte[] narMd5 = calculateMd5sum(nar);
        final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
        if (!workingHashFile.exists()) {
            FileUtils.deleteFile(narWorkingDirectory, true);
            unpack(nar, narWorkingDirectory, narMd5);
        } else {
            final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
            if (!Arrays.equals(hashFileContents, narMd5)) {
                logger.info("Contents of nar {} have changed. Reloading.", new Object[] { nar.getAbsolutePath() });
                FileUtils.deleteFile(narWorkingDirectory, true);
                unpack(nar, narWorkingDirectory, narMd5);
            }
        }
    }

    return narWorkingDirectory;
}
 
Example #9
Source File: NiFi.java    From nifi with Apache License 2.0 4 votes vote down vote up
public NiFi(final NiFiProperties properties, ClassLoader rootClassLoader)
        throws ClassNotFoundException, IOException, NoSuchMethodException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {

    // There can only be one krb5.conf for the overall Java process so set this globally during
    // start up so that processors and our Kerberos authentication code don't have to set this
    final File kerberosConfigFile = properties.getKerberosConfigurationFile();
    if (kerberosConfigFile != null) {
        final String kerberosConfigFilePath = kerberosConfigFile.getAbsolutePath();
        LOGGER.info("Setting java.security.krb5.conf to {}", new Object[]{kerberosConfigFilePath});
        System.setProperty("java.security.krb5.conf", kerberosConfigFilePath);
    }

    setDefaultUncaughtExceptionHandler();

    // register the shutdown hook
    addShutdownHook();

    final String bootstrapPort = System.getProperty(BOOTSTRAP_PORT_PROPERTY);
    if (bootstrapPort != null) {
        try {
            final int port = Integer.parseInt(bootstrapPort);

            if (port < 1 || port > 65535) {
                throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535");
            }

            bootstrapListener = new BootstrapListener(this, port);
            bootstrapListener.start();
        } catch (final NumberFormatException nfe) {
            throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535");
        }
    } else {
        LOGGER.info("NiFi started without Bootstrap Port information provided; will not listen for requests from Bootstrap");
        bootstrapListener = null;
    }

    // delete the web working dir - if the application does not start successfully
    // the web app directories might be in an invalid state. when this happens
    // jetty will not attempt to re-extract the war into the directory. by removing
    // the working directory, we can be assured that it will attempt to extract the
    // war every time the application starts.
    File webWorkingDir = properties.getWebWorkingDirectory();
    FileUtils.deleteFilesInDirectory(webWorkingDir, null, LOGGER, true, true);
    FileUtils.deleteFile(webWorkingDir, LOGGER, 3);

    detectTimingIssues();

    // redirect JUL log events
    initLogging();

    final Bundle systemBundle = SystemBundle.create(properties, rootClassLoader);

    // expand the nars
    final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, systemBundle);

    // load the extensions classloaders
    NarClassLoaders narClassLoaders = NarClassLoadersHolder.getInstance();

    narClassLoaders.init(rootClassLoader,
            properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory());

    // load the framework classloader
    final ClassLoader frameworkClassLoader = narClassLoaders.getFrameworkBundle().getClassLoader();
    if (frameworkClassLoader == null) {
        throw new IllegalStateException("Unable to find the framework NAR ClassLoader.");
    }

    final Set<Bundle> narBundles = narClassLoaders.getBundles();

    // load the server from the framework classloader
    Thread.currentThread().setContextClassLoader(frameworkClassLoader);
    Class<?> jettyServer = Class.forName("org.apache.nifi.web.server.JettyServer", true, frameworkClassLoader);
    Constructor<?> jettyConstructor = jettyServer.getConstructor(NiFiProperties.class, Set.class);

    final long startTime = System.nanoTime();
    nifiServer = (NiFiServer) jettyConstructor.newInstance(properties, narBundles);
    nifiServer.setExtensionMapping(extensionMapping);
    nifiServer.setBundles(systemBundle, narBundles);

    if (shutdown) {
        LOGGER.info("NiFi has been shutdown via NiFi Bootstrap. Will not start Controller");
    } else {
        nifiServer.start();

        if (bootstrapListener != null) {
            bootstrapListener.sendStartedStatus(true);
        }

        final long duration = System.nanoTime() - startTime;
        LOGGER.info("Controller initialization took " + duration + " nanoseconds "
                + "(" + (int) TimeUnit.SECONDS.convert(duration, TimeUnit.NANOSECONDS) + " seconds).");
    }
}
 
Example #10
Source File: FrameworkIntegrationTest.java    From nifi with Apache License 2.0 4 votes vote down vote up
protected final void initialize(final NiFiProperties nifiProperties) throws IOException {
    this.nifiProperties = nifiProperties;

    final FlowFileEventRepository flowFileEventRepository = new RingBufferEventRepository(5);

    final BulletinRepository bulletinRepo = new VolatileBulletinRepository();
    flowEngine = new FlowEngine(4, "unit test flow engine");
    extensionManager = new DirectInjectionExtensionManager();

    extensionManager.injectExtensionType(FlowFileRepository.class, WriteAheadFlowFileRepository.class);
    extensionManager.injectExtensionType(ContentRepository.class, FileSystemRepository.class);
    extensionManager.injectExtensionType(ProvenanceRepository.class, WriteAheadProvenanceRepository.class);
    extensionManager.injectExtensionType(StateProvider.class, WriteAheadLocalStateProvider.class);
    extensionManager.injectExtensionType(ComponentStatusRepository.class, VolatileComponentStatusRepository.class);
    extensionManager.injectExtensionType(FlowFileSwapManager.class, FileSystemSwapManager.class);

    extensionManager.injectExtensionType(Processor.class, BiConsumerProcessor.class);
    extensionManager.injectExtensionType(Processor.class, GenerateProcessor.class);
    extensionManager.injectExtensionType(Processor.class, TerminateOnce.class);
    extensionManager.injectExtensionType(Processor.class, TerminateAll.class);
    extensionManager.injectExtensionType(Processor.class, NopProcessor.class);
    extensionManager.injectExtensionType(Processor.class, UsernamePasswordProcessor.class);

    injectExtensionTypes(extensionManager);
    systemBundle = SystemBundle.create(nifiProperties);
    extensionManager.discoverExtensions(systemBundle, Collections.emptySet());

    final StringEncryptor encryptor = StringEncryptor.createEncryptor("PBEWITHMD5AND256BITAES-CBC-OPENSSL", "BC", "unit-test");
    final Authorizer authorizer = new AlwaysAuthorizedAuthorizer();
    final AuditService auditService = new NopAuditService();

    if (isClusteredTest()) {
        final File zookeeperDir = new File("target/state/zookeeper");
        final File version2Dir =  new File(zookeeperDir, "version-2");

        if (!version2Dir.exists()) {
            assertTrue(version2Dir.mkdirs());
        }

        final File[] children = version2Dir.listFiles();
        if (children != null) {
            for (final File file : children) {
                FileUtils.deleteFile(file, true);
            }
        }

        clusterCoordinator = Mockito.mock(ClusterCoordinator.class);
        final HeartbeatMonitor heartbeatMonitor = Mockito.mock(HeartbeatMonitor.class);
        final NodeProtocolSender protocolSender = Mockito.mock(NodeProtocolSender.class);
        final LeaderElectionManager leaderElectionManager = new CuratorLeaderElectionManager(2, nifiProperties);

        final NodeIdentifier localNodeId = new NodeIdentifier(UUID.randomUUID().toString(), "localhost", 8111, "localhost", 8081,
            "localhost", 8082, "localhost", 8083, 8084, false, Collections.emptySet());
        final NodeIdentifier node2Id = new NodeIdentifier(UUID.randomUUID().toString(), "localhost", 8222, "localhost", 8081,
            "localhost", 8082, "localhost", 8083, 8084, false, Collections.emptySet());

        final Set<NodeIdentifier> nodeIdentifiers = new HashSet<>();
        nodeIdentifiers.add(localNodeId);
        nodeIdentifiers.add(node2Id);
        Mockito.when(clusterCoordinator.getNodeIdentifiers()).thenReturn(nodeIdentifiers);
        Mockito.when(clusterCoordinator.getLocalNodeIdentifier()).thenReturn(localNodeId);

        flowController = FlowController.createClusteredInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, protocolSender, bulletinRepo, clusterCoordinator,
            heartbeatMonitor, leaderElectionManager, VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, flowRegistryClient, extensionManager);

        flowController.setClustered(true, UUID.randomUUID().toString());
        flowController.setNodeId(localNodeId);

        flowController.setConnectionStatus(new NodeConnectionStatus(localNodeId, NodeConnectionState.CONNECTED));
    } else {
        flowController = FlowController.createStandaloneInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, bulletinRepo,
            VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, flowRegistryClient, extensionManager);
    }

    processScheduler = new StandardProcessScheduler(flowEngine, flowController, encryptor, flowController.getStateManagerProvider(), nifiProperties);

    final RepositoryContextFactory repositoryContextFactory = flowController.getRepositoryContextFactory();
    final SchedulingAgent timerDrivenSchedulingAgent = new TimerDrivenSchedulingAgent(flowController, flowEngine, repositoryContextFactory, encryptor, nifiProperties);
    processScheduler.setSchedulingAgent(SchedulingStrategy.TIMER_DRIVEN, timerDrivenSchedulingAgent);

    flowFileSwapManager = flowController.createSwapManager();
    resourceClaimManager = flowController.getResourceClaimManager();
}