Java Code Examples for org.apache.commons.vfs2.FileObject#findFiles()

/**
 * Attempts to find a plugin's installation folder on disk within all known plugin folder locations
 *
 * @param pluginFolderName Name of plugin folder
 * @return Tuple of [(FileObject) Location of the first plugin folder found as a direct descendant of one of the known
 * plugin folder locations, (String) Relative path from parent]
 * @throws KettleFileException Error getting plugin folders
 */
protected Object[] findPluginFolder( final String pluginFolderName ) throws KettleFileException {
  List<PluginFolderInterface> pluginFolders = PluginFolder.populateFolders( null );
  if ( pluginFolders != null ) {
    for ( PluginFolderInterface pluginFolder : pluginFolders ) {
      FileObject folder = KettleVFS.getFileObject( pluginFolder.getFolder() );

      try {
        if ( folder.exists() ) {
          FileObject[] files = folder.findFiles( new PluginFolderSelector( pluginFolderName ) );
          if ( files != null && files.length > 0 ) {
            return new Object[] { files[ 0 ],
              folder.getName().getRelativeName( files[ 0 ].getName() ) }; // Return the first match
          }
        }
      } catch ( FileSystemException ex ) {
        throw new KettleFileException( "Error searching for folder '" + pluginFolderName + "'", ex );
      }
    }
  }
  return new Object[] {};
}
 

private DataFileStream<GenericRecord> createDataFileStream() throws Exception {
  DatumReader<GenericRecord> datumReader;
  if ( useFieldAsInputStream ) {
    datumReader = new GenericDatumReader<GenericRecord>();
    inputStream.reset();
    return new DataFileStream<GenericRecord>( inputStream, datumReader );
  }
  if ( schemaFileName != null && schemaFileName.length() > 0 ) {
    Schema schema = new Schema.Parser().parse( KettleVFS.getInputStream( schemaFileName, variableSpace ) );
    datumReader = new GenericDatumReader<GenericRecord>( schema );
  } else {
    datumReader = new GenericDatumReader<GenericRecord>();
  }
  FileObject fileObject = KettleVFS.getFileObject( fileName, variableSpace );
  if ( fileObject.isFile() ) {
    this.inputStream = fileObject.getContent().getInputStream();
    return new DataFileStream<>( inputStream, datumReader );
  } else {
    FileObject[] avroFiles = fileObject.findFiles( new FileExtensionSelector( "avro" ) );
    if ( !Utils.isEmpty( avroFiles ) ) {
      this.inputStream = avroFiles[ 0 ].getContent().getInputStream();
      return new DataFileStream<>( inputStream, datumReader );
    }
    return null;
  }
}
 

private DataFileStream<Object> createNestedDataFileStream() throws Exception {
  DatumReader<Object> datumReader;
  if ( useFieldAsInputStream ) {
    datumReader = new GenericDatumReader<Object>();
    inputStream.reset();
    return new DataFileStream<Object>( inputStream, datumReader );
  }
  if ( schemaFileName != null && schemaFileName.length() > 0 ) {
    Schema schema = new Schema.Parser().parse( KettleVFS.getInputStream( schemaFileName, variableSpace ) );
    datumReader = new GenericDatumReader<Object>( schema );
  } else {
    datumReader = new GenericDatumReader<Object>();
  }
  FileObject fileObject = KettleVFS.getFileObject( fileName, variableSpace );
  if ( fileObject.isFile() ) {
    this.inputStream = fileObject.getContent().getInputStream();
    return new DataFileStream<>( inputStream, datumReader );
  } else {
    FileObject[] avroFiles = fileObject.findFiles( new FileExtensionSelector( "avro" ) );
    if ( !Utils.isEmpty( avroFiles ) ) {
      this.inputStream = avroFiles[ 0 ].getContent().getInputStream();
      return new DataFileStream<>( inputStream, datumReader );
    }
    return null;
  }
}
 

public static void main(final String[] args) throws Exception {

        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));

        // We are interested in files older than one day
        final long cutoff = System.currentTimeMillis() - (24 * 60 * 60 * 1000);
        final AgeFileFilter filter = new AgeFileFilter(cutoff);

        final FileObject[] files = dir.findFiles(new FileFilterSelector(filter));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static Iterable<FileObject> find(FileObject base, FileSelector selector) throws FileSystemException {
    final FileObject[] files = base.findFiles(selector);
    if(files == null) {
        return Iterables2.empty();
    }
    return Iterables2.from(files);
}
 

public static void main(final String[] args) throws Exception {

        // Example, how to print out a list of the current directory's
        // subdirectories
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(DirectoryFileFilter.DIRECTORY));
        for (FileObject file : files) {
            System.out.println(file);
        }
    }
 

public static void main(final String[] args) throws Exception {

        // Example, to retrieve and print all java files where the name matched
        // the regular expression in the current directory
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(new RegexFileFilter(
                "ˆ.*[tT]est(-\\d+)?\\.java$")));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static void main(final String[] args) throws Exception {

        // Example, to print all files and directories in the current directory
        // whose size is greater than 1 MB
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final SizeFileFilter filter = new SizeFileFilter(1024 * 1024);
        final FileObject[] files = dir.findFiles(new FileFilterSelector(filter));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static void main(final String[] args) throws Exception {

        // Example, to retrieve and print all java files that have the
        // expression test in the name in the current directory
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(new WildcardFileFilter(
                "*test*.java")));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static void main(final String[] args) throws Exception {

        // Example, how to print out a list of the real files within the current
        // directory
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(FileFileFilter.FILE));
        for (FileObject file : files) {
            System.out.println(file);
        }
    }
 

public static void main(final String[] args) throws Exception {

        // Example, to print all files and directories in the current directory
        // whose name starts with a {@code .}
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(new PrefixFileFilter(".")));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static void main(final String[] args) throws Exception {

        // Example, to retrieve and print all *.java files in the current
        // directory
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(new SuffixFileFilter(".java")));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

public static void main(final String[] args) throws Exception {

        // Example, to print all files and directories in the current directory
        // whose name is Test
        final FileSystemManager fsManager = VFS.getManager();
        final FileObject dir = fsManager.toFileObject(new File("."));
        final FileObject[] files = dir.findFiles(new FileFilterSelector(new NameFileFilter("Test")));
        for (FileObject file : files) {
            System.out.println(file);
        }

    }
 

/**
 * Test some special file name symbols.
 * <p>
 * Use the RamProvider since it has no character limitations like
 * the (Windows) LocalFileProvider.
 */
@Test
public void testSpecialName() throws FileSystemException
{
    // we test with this file name
    // does not work with '!'
    final String testDir = "/spacialtest/";
    final String testFileName = "test:+-_ \"()<>%#.txt";
    final String expectedName = testDir + testFileName;

    final FileObject dir = prepareSpecialFile(testDir, testFileName);


    // DO: verify you can list it:
    final FileObject[] findFilesResult = dir.findFiles(new AllFileSelector()); // includes dir
    final FileObject[] getChildrenResult = dir.getChildren();
    final FileObject getChildResult = dir.getChild(UriParser.encode(testFileName, ENC));


    // validate findFiles returns expected result
    assertEquals("Unexpected result findFiles: " + Arrays.toString(findFilesResult), 2, findFilesResult.length);
    String resultName = findFilesResult[0].getName().getPathDecoded();
    assertEquals("findFiles Child name does not match", expectedName, resultName);
    assertEquals("Did findFiles but child was no file", FileType.FILE, findFilesResult[0].getType());

    // validate getChildren returns expected result
    assertEquals("Unexpected result getChildren: " + Arrays.toString(getChildrenResult), 1, getChildrenResult.length);
    resultName = getChildrenResult[0].getName().getPathDecoded();
    assertEquals("getChildren Child name does not match", expectedName, resultName);
    assertEquals("Did getChildren but child was no file", FileType.FILE, getChildrenResult[0].getType());

    // validate getChild returns expected child
    assertNotNull("Did not find direct child", getChildResult);
    resultName = getChildResult.getName().getPathDecoded();
    assertEquals("getChild name does not match", expectedName, resultName);
    assertEquals("getChild was no file", FileType.FILE, getChildResult.getType());
}
 

/**
 * Copies another file to this file.
 *
 * @param file The FileObject to copy.
 * @param selector The FileSelector.
 * @throws FileSystemException if an error occurs.
 */
@Override
public void copyFrom(final FileObject file, final FileSelector selector) throws FileSystemException {
    if (!FileObjectUtils.exists(file)) {
        throw new FileSystemException("vfs.provider/copy-missing-file.error", file);
    }

    // Locate the files to copy across
    final ArrayList<FileObject> files = new ArrayList<>();
    file.findFiles(selector, false, files);

    // Copy everything across
    for (final FileObject srcFile : files) {
        // Determine the destination file
        final String relPath = file.getName().getRelativeName(srcFile.getName());
        final FileObject destFile = resolveFile(relPath, NameScope.DESCENDENT_OR_SELF);

        // Clean up the destination file, if necessary
        if (FileObjectUtils.exists(destFile) && destFile.getType() != srcFile.getType()) {
            // The destination file exists, and is not of the same type,
            // so delete it
            // TODO - add a pluggable policy for deleting and overwriting existing files
            destFile.deleteAll();
        }

        // Copy across
        try {
            if (srcFile.getType().hasContent()) {
                FileObjectUtils.writeContent(srcFile, destFile);
            } else if (srcFile.getType().hasChildren()) {
                destFile.createFolder();
            }
        } catch (final IOException e) {
            throw new FileSystemException("vfs.provider/copy-file.error", e, srcFile, destFile);
        }
    }
}
 

@Override
public void run() {
    try {
        long invalidationPeriod = getCacheInvalidationPeriod();
        long currentTime = System.currentTimeMillis();
        // lock the timer in write mode, this will prevent any Task to start during the cleaning process
        if (cacheCleaningRWLock.writeLock().tryLock()) {
            try {
                FileObject rootFO = fileSystemManager.resolveFile(rootCacheUri);
                if (!rootFO.exists()) {
                    rootFO.createFolder();
                }
                FileObject[] files = rootFO.findFiles(Selectors.EXCLUDE_SELF);
                if (files != null) {
                    for (FileObject file : files) {
                        if (currentTime - file.getContent().getLastModifiedTime() > invalidationPeriod) {
                            logger.info("[Cache Space cleaner] deleting " + file);
                            file.delete();
                        }
                    }
                }
            } finally {
                cacheCleaningRWLock.writeLock().unlock();
            }
        }
    } catch (Exception e) {
        logger.error("Error when cleaning files in cache", e);
    }
}
 

private static List<File> findFiles(File root, FileSelector selector) {
    List<File> listFiles = new ArrayList<>();
    try {
        FileObject rootObject = VFS.getManager().toFileObject(root);
        FileObject[] fos = rootObject.findFiles(selector);

        for (FileObject fo : fos) {
            listFiles.add(new File(fo.getName().getPath()));
        }
    } catch (Exception e) {
        logger.error("An error occurred while zipping files: ", e);
    }

    return listFiles;
}
 

private void ProcessFile( String filename, String wildcard ) {

    FileObject filefolder = null;
    String realFilefoldername = environmentSubstitute( filename );
    String realwilcard = environmentSubstitute( wildcard );

    try {
      filefolder = HopVfs.getFileObject( realFilefoldername );
      FileObject[] files = new FileObject[] { filefolder };
      if ( filefolder.exists() ) {
        // the file or folder exists
        if ( filefolder.getType() == FileType.FOLDER ) {
          // It's a folder
          if ( isDetailed() ) {
            logDetailed( BaseMessages.getString(
              PKG, "ActionCheckFilesLocked.ProcessingFolder", realFilefoldername ) );
          }
          // Retrieve all files
          files = filefolder.findFiles( new TextFileSelector( filefolder.toString(), realwilcard ) );

          if ( isDetailed() ) {
            logDetailed( BaseMessages.getString( PKG, "ActionCheckFilesLocked.TotalFilesToCheck", String
              .valueOf( files.length ) ) );
          }
        } else {
          // It's a file
          if ( isDetailed() ) {
            logDetailed( BaseMessages.getString(
              PKG, "ActionCheckFilesLocked.ProcessingFile", realFilefoldername ) );
          }
        }
        // Check files locked
        checkFilesLocked( files );
      } else {
        // We can not find thsi file
        logBasic( BaseMessages.getString( PKG, "ActionCheckFilesLocked.FileNotExist", realFilefoldername ) );
      }
    } catch ( Exception e ) {
      logError( BaseMessages.getString( PKG, "ActionCheckFilesLocked.CouldNotProcess", realFilefoldername, e
        .getMessage() ) );
    } finally {
      if ( filefolder != null ) {
        try {
          filefolder.close();
        } catch ( IOException ex ) {
          // Ignore
        }
      }
    }
  }
 

/**
 * Test if listing files with known scheme prefix works.
 * <p>
 * This test is not RamProvider specific but it uses it as a simple test-bed.
 * Verifies VFS-741.
 */
@Test
public void testSchemePrefix() throws FileSystemException
{
    // use a :-prefix with a known scheme (unknown scheme works since VFS-398)
    final String KNOWN_SCHEME = manager.getSchemes()[0]; // typically "ram"

    // we test with this file name
    final String testDir = "/prefixtest/";
    final String testFileName = KNOWN_SCHEME + ":test:txt";
    final String expectedName = testDir + testFileName;

    final FileObject dir = prepareSpecialFile(testDir, testFileName);


    // verify we can list dir

    // if not it throws:
    // Caused by: org.apache.commons.vfs2.FileSystemException: Invalid descendent file name "ram:data:test.txt".
    //   at org.apache.commons.vfs2.impl.DefaultFileSystemManager.resolveName
    //   at org.apache.commons.vfs2.provider.AbstractFileObject.getChildren
    //   at org.apache.commons.vfs2.provider.AbstractFileObject.traverse
    //   at org.apache.commons.vfs2.provider.AbstractFileObject.findFiles

    // test methods to get the child:
    final FileObject[] findFilesResult = dir.findFiles(new AllFileSelector()); // includes dir
    final FileObject[] getChildrenResult = dir.getChildren();
    final FileObject getChildResult = dir.getChild(testFileName);

    // validate findFiles returns expected result
    assertEquals("Unexpected result findFiles: " + Arrays.toString(findFilesResult), 2, findFilesResult.length);
    String resultName = findFilesResult[0].getName().getPathDecoded();
    assertEquals("findFiles Child name does not match", expectedName, resultName);
    assertEquals("Did findFiles but child was no file", FileType.FILE, findFilesResult[0].getType());

    // validate getChildren returns expected result
    assertEquals("Unexpected result getChildren: " + Arrays.toString(getChildrenResult), 1, getChildrenResult.length);
    resultName = getChildrenResult[0].getName().getPathDecoded();
    assertEquals("getChildren Child name does not match", expectedName, resultName);
    assertEquals("Did getChildren but child was no file", FileType.FILE, getChildrenResult[0].getType());

    // validate getChild returns expected child
    assertNotNull("Did not find direct child", getChildResult);
    resultName = getChildResult.getName().getPathDecoded();
    assertEquals("getChild name does not match", expectedName, resultName);
    assertEquals("getChild was no file", FileType.FILE, getChildResult.getType());
}
 

private void verifyJobSamples( RepositoryDirectoryInterface samplesDirectory ) throws Exception {
  FileObject jobSamplesFolder = KettleVFS.getFileObject( "samples/jobs/" );
  FileObject[] files = jobSamplesFolder.findFiles( new FileSelector() {

    @Override
    public boolean traverseDescendents( FileSelectInfo arg0 ) throws Exception {
      return true;
    }

    @Override
    public boolean includeFile( FileSelectInfo info ) throws Exception {
      return info.getFile().getName().getExtension().equalsIgnoreCase( "kjb" );
    }
  } );

  List<FileObject> filesList = Arrays.asList( files );
  Collections.sort( filesList, new Comparator<FileObject>() {
    @Override
    public int compare( FileObject o1, FileObject o2 ) {
      return o1.getName().getPath().compareTo( o2.getName().getPath() );
    }
  } );

  for ( FileObject file : filesList ) {
    String jobFilename = file.getName().getPath();
    System.out.println( "Storing/Loading/validating job '" + jobFilename + "'" );

    // Load the JobMeta object...
    //
    JobMeta jobMeta = new JobMeta( jobFilename, repository );
    jobMeta.setFilename( null );

    // The name is sometimes empty in the file, duplicates are present too...
    // Replaces slashes and the like as well...
    //
    jobMeta.setName( Const.createName( file.getName().getBaseName() ) );
    jobMeta.setName( jobMeta.getName().replace( '/', '-' ) );

    if ( Utils.isEmpty( jobMeta.getName() ) ) {
      jobMeta.setName( Const.createName( file.getName().getBaseName() ) );
    }
    if ( jobMeta.getName().contains( "/" ) ) {
      jobMeta.setName( jobMeta.getName().replace( '/', '-' ) );
    }

    // Save it in the repository in the samples folder
    //
    jobMeta.setRepositoryDirectory( samplesDirectory );
    repository.save( jobMeta, "unit testing" );
    assertNotNull( jobMeta.getObjectId() );

    // Load it back up again...
    //
    JobMeta repJobMeta = repository.loadJob( jobMeta.getObjectId(), null );
    String oneXml = repJobMeta.getXML();

    // Save & load it again
    //
    repository.save( jobMeta, "unit testing" );
    repJobMeta = repository.loadJob( jobMeta.getObjectId(), null );
    String twoXml = repJobMeta.getXML();

    // The XML needs to be identical after loading
    //
    // storeFile(oneXml, "/tmp/one.ktr");
    // storeFile(twoXml, "/tmp/two.ktr");
    //
    assertEquals( oneXml, twoXml );
  }

  // Verify the number of stored files, see if we can find them all again.
  //
  System.out.println( "Stored " + files.length + " job samples in folder " + samplesDirectory.getPath() );
  String[] jobNames = repository.getJobNames( samplesDirectory.getObjectId(), false );
  assertEquals( files.length, jobNames.length );
}