Java Code Examples for org.apache.commons.compress.utils.IOUtils#copy()

private static void extract(final Path path, final String destDirectory) throws IOException {
  try (TarArchiveInputStream fin =
      new TarArchiveInputStream(
          new GzipCompressorInputStream(new FileInputStream(path.toAbsolutePath().toString())))) {
    TarArchiveEntry entry;
    while ((entry = fin.getNextTarEntry()) != null) {
      if (entry.isDirectory()) {
        continue;
      }
      final File curfile = new File(destDirectory, entry.getName());
      final File parent = curfile.getParentFile();
      if (!parent.exists()) {
        parent.mkdirs();
      }
      IOUtils.copy(fin, new FileOutputStream(curfile));
    }
  }
}
 

/**
 * given tar.gz file will be copied to this tar.gz file.
 * all files will be transferred to new tar.gz file one by one.
 * original directory structure will be kept intact
 *
 * @param zipFile the archive file to be copied to the new archive
 * @param dirPrefixForTar sub path inside the archive
 */
public boolean addZipToArchive(String zipFile, String dirPrefixForTar) {
  try {
    // construct input stream
    ZipFile zipFileObj = new ZipFile(zipFile);
    Enumeration<? extends ZipEntry> entries = zipFileObj.entries();

    // copy the existing entries from source gzip file
    while (entries.hasMoreElements()) {
      ZipEntry nextEntry = entries.nextElement();
      TarArchiveEntry entry = new TarArchiveEntry(dirPrefixForTar + nextEntry.getName());
      entry.setSize(nextEntry.getSize());
      entry.setModTime(nextEntry.getTime());

      tarOutputStream.putArchiveEntry(entry);
      IOUtils.copy(zipFileObj.getInputStream(nextEntry), tarOutputStream);
      tarOutputStream.closeArchiveEntry();
    }

    zipFileObj.close();
    return true;
  } catch (IOException ioe) {
    LOG.log(Level.SEVERE, "Archive File can not be added: " + zipFile, ioe);
    return false;
  }
}
 

public static void unzip(File zip, File dest, Charset charset) throws IOException {
    Path destPath = dest.toPath();
    try (ZipFile zipFile = new ZipFile(zip, ZipFile.OPEN_READ, charset)) {
        Enumeration<? extends ZipEntry> entries = zipFile.entries();
        while (entries.hasMoreElements()) {
            ZipEntry entry = entries.nextElement();
            Path entryPath = destPath.resolve(entry.getName());
            if (!entryPath.normalize().startsWith(dest.toPath()))
                throw new IOException("Zip entry contained path traversal");
            if (entry.isDirectory()) {
                Files.createDirectories(entryPath);
            } else {
                Files.createDirectories(entryPath.getParent());
                try (InputStream in = zipFile.getInputStream(entry)) {
                    try (OutputStream out = new FileOutputStream(entryPath.toFile())) {
                        IOUtils.copy(in, out);
                    }
                }
            }
        }
    }
}
 

public static void untar(
        InputStream tar,
        File parentDir
) throws IOException {
    TarArchiveInputStream tin = new TarArchiveInputStream(tar);
    ArchiveEntry e;
    while ((e = tin.getNextEntry()) != null) {
        File f = new File(parentDir, e.getName());
        f.setLastModified(e.getLastModifiedDate().getTime());
        f.getParentFile().mkdirs();
        if (e.isDirectory()) {
            f.mkdir();
            continue;
        }
        long size = e.getSize();
        checkFileSize(size);
        try (OutputStream out = new FileOutputStream(f)) {
            /* TarInputStream pretends each
               entry's EOF is the stream's EOF */
            IOUtils.copy(tin, out);
        }
    }
}
 

public String doRestPOST(String method, Object parameter) throws IOException, RuntimeException {
    HttpPost request = new HttpPost("http://localhost:16043/rest/v1/" + method);

    if (parameter != null) {
        request.setEntity(new StringEntity(marshal(parameter), ContentType.APPLICATION_JSON));
    }

    HttpResponse response = client.execute(targetHost, request, context);

    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    IOUtils.copy(response.getEntity().getContent(), byteArrayOutputStream);

    String responseString = new String(byteArrayOutputStream.toByteArray(), "UTF8");

    if (response.getStatusLine().getStatusCode() != 200 && response.getStatusLine().getStatusCode() != 203) {
        throw new RuntimeException(responseString);
    }

    return responseString;
}
 

/**
 * add one file to tar.gz file
 * file is created from the given byte array
 *
 * @param filename file to be added to the tar.gz
 */
public boolean addFileToArchive(String filename, byte[] contents) {

  String filePathInTar = JOB_ARCHIVE_DIRECTORY + "/" + filename;
  try {
    TarArchiveEntry entry = new TarArchiveEntry(filePathInTar);
    entry.setSize(contents.length);
    tarOutputStream.putArchiveEntry(entry);
    IOUtils.copy(new ByteArrayInputStream(contents), tarOutputStream);
    tarOutputStream.closeArchiveEntry();

    return true;
  } catch (IOException e) {
    LOG.log(Level.SEVERE, "File can not be added: " + filePathInTar, e);
    return false;
  }
}
 

/**
 * 把一个ZIP文件解压到一个指定的目录中
 * @param zipfilename ZIP文件抽象地址
 * @param outputdir 目录绝对地址
 */
public static void unZipToFolder(String zipfilename, String outputdir) throws IOException {
    File zipfile = new File(zipfilename);
    if (zipfile.exists()) {
        outputdir = outputdir + File.separator;
        FileUtils.forceMkdir(new File(outputdir));

        ZipFile zf = new ZipFile(zipfile, "UTF-8");
        Enumeration zipArchiveEntrys = zf.getEntries();
        while (zipArchiveEntrys.hasMoreElements()) {
            ZipArchiveEntry zipArchiveEntry = (ZipArchiveEntry) zipArchiveEntrys.nextElement();
            if (zipArchiveEntry.isDirectory()) {
                FileUtils.forceMkdir(new File(outputdir + zipArchiveEntry.getName() + File.separator));
            } else {
                IOUtils.copy(zf.getInputStream(zipArchiveEntry), FileUtils.openOutputStream(new File(outputdir + zipArchiveEntry.getName())));
            }
        }
    } else {
        throw new IOException("指定的解压文件不存在：\t" + zipfilename);
    }
}
 

public static byte[] decompress(byte[] bytes, int len) throws IOException
{
	byte[] data = new byte[len + BZIP_HEADER.length];

	// add header
	System.arraycopy(BZIP_HEADER, 0, data, 0, BZIP_HEADER.length);
	System.arraycopy(bytes, 0, data, BZIP_HEADER.length, len);

	ByteArrayOutputStream os = new ByteArrayOutputStream();

	try (InputStream is = new BZip2CompressorInputStream(new ByteArrayInputStream(data)))
	{
		IOUtils.copy(is, os);
	}

	return os.toByteArray();
}
 

private static void addToTar(ArchiveOutputStream tar, File file, String fileNameAndPath) throws IOException {
    if (!file.exists() || !file.canRead()) {
        throw new FileNotFoundException(String.format("Cannot read file %s. Are you sure it exists?",
                file.getAbsolutePath()));
    }
    if (file.isDirectory()) {
        for (File fileInDirectory : file.listFiles()) {
            if (!fileNameAndPath.endsWith("/")) {
                fileNameAndPath = fileNameAndPath + "/";
            }
            addToTar(tar, fileInDirectory, fileNameAndPath + fileInDirectory.getName());
        }
    } else {
        ArchiveEntry entry = tar.createArchiveEntry(file, fileNameAndPath);
        tar.putArchiveEntry(entry);
        try (FileInputStream fis = new FileInputStream(file)) {
            IOUtils.copy(fis, tar);
        }
        tar.closeArchiveEntry();
    }
}
 

public static void unzip(final File zip, final File dest, final Charset charset) throws IOException {
    final Path destPath = dest.toPath();
    try (final ZipFile zipFile = new ZipFile(zip, ZipFile.OPEN_READ, charset)) {
        final Enumeration<? extends ZipEntry> entries = zipFile.entries();
        while (entries.hasMoreElements()) {
            final ZipEntry entry = entries.nextElement();
            final Path entryPath = destPath.resolve(entry.getName());
            if (!entryPath.normalize().startsWith(dest.toPath()))
                throw new IOException("Zip entry contained path traversal");
            if (entry.isDirectory()) {
                Files.createDirectories(entryPath);
            } else {
                Files.createDirectories(entryPath.getParent());
                try (final InputStream in = zipFile.getInputStream(entry)) {
                    try (final OutputStream out = new FileOutputStream(entryPath.toFile())) {
                        IOUtils.copy(in, out);
                    }
                }
            }
        }
    }
}
 

private static List<File> unTar(final File inputFile, final File outputDir) throws FileNotFoundException, IOException, ArchiveException {
    Log.i(TAG, String.format("Untaring %s to dir %s", inputFile.getAbsolutePath(), outputDir.getAbsolutePath()));

    if (!outputDir.exists()) {
        outputDir.mkdirs();
    }
    final List<File> untaredFiles = new LinkedList<File>();
    final InputStream is = new FileInputStream(inputFile);
    final TarArchiveInputStream debInputStream = (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is);
    TarArchiveEntry entry = null;
    while ((entry = (TarArchiveEntry)debInputStream.getNextEntry()) != null) {
        final File outputFile = new File(outputDir, entry.getName());
        if (entry.isDirectory()) {
            if (!outputFile.exists()) {
                if (!outputFile.mkdirs()) {
                    throw new IllegalStateException(String.format("Couldn't create directory %s.", outputFile.getAbsolutePath()));
                }
            }
        } else {
            final OutputStream outputFileStream = new FileOutputStream(outputFile);
            IOUtils.copy(debInputStream, outputFileStream);
            outputFileStream.close();
        }
        untaredFiles.add(outputFile);
    }
    debInputStream.close();

    return untaredFiles;
}
 

/**
 * 
 * ��һ���ļ��鵵
 * 
 * @param sources
 *            Ҫ�鵵��ԭ�ļ�����
 * @param target
 *            �鵵����ļ�
 * @return File ���ع鵵����ļ�
 * @throws Exception
 */
public static File tar(File[] sources, File target) throws Exception {
	FileOutputStream out = new FileOutputStream(target);
	TarArchiveOutputStream os = new TarArchiveOutputStream(out);
	for (File file : sources) {
		os.putArchiveEntry(new TarArchiveEntry(file));
		IOUtils.copy(new FileInputStream(file), os);
		os.closeArchiveEntry();
	}
	if (os != null) {
		os.flush();
		os.close();
	}
	return target;
}
 

/**
 * Bunzip2 a file
 * 
 * @param inputFile
 *            source file
 * @param outputFile
 *            destionation file
 * @return the destionation file
 * @throws ArchiveException
 *             if any error occurs
 */
public File bunzip2(final File inputFile, final File outputFile) {
    LOGGER.info(
            String.format("Ungzipping %s to dir %s.", inputFile.getAbsolutePath(), outputFile.getAbsolutePath()));
    try (BZip2CompressorInputStream in = new BZip2CompressorInputStream(new FileInputStream(inputFile));
            FileOutputStream out = new FileOutputStream(outputFile)) {
        IOUtils.copy(in, out);
        return outputFile;
    } catch (IOException e) {
        throw new ArchiveException("Unable to gunzip file", e);
    }
}
 

private String getReefTarUri(final String jobIdentifier) {
  try {
    // Create REEF_TAR
    final FileOutputStream fileOutputStream = new FileOutputStream(REEF_TAR);
    final TarArchiveOutputStream tarArchiveOutputStream =
        new TarArchiveOutputStream(new GZIPOutputStream(fileOutputStream));
    final File globalFolder = new File(this.fileNames.getGlobalFolderPath());
    final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(globalFolder.toPath());

    for (final Path path : directoryStream) {
      tarArchiveOutputStream.putArchiveEntry(new TarArchiveEntry(path.toFile(),
          globalFolder + "/" + path.getFileName()));

      final BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(path.toFile()));
      IOUtils.copy(bufferedInputStream, tarArchiveOutputStream);
      bufferedInputStream.close();

      tarArchiveOutputStream.closeArchiveEntry();
    }
    directoryStream.close();
    tarArchiveOutputStream.close();
    fileOutputStream.close();

    // Upload REEF_TAR to HDFS
    final FileSystem fileSystem = FileSystem.get(new Configuration());
    final org.apache.hadoop.fs.Path src = new org.apache.hadoop.fs.Path(REEF_TAR);
    final String reefTarUriValue = fileSystem.getUri().toString() + this.jobSubmissionDirectoryPrefix + "/" +
        jobIdentifier + "/" + REEF_TAR;
    final org.apache.hadoop.fs.Path dst = new org.apache.hadoop.fs.Path(reefTarUriValue);
    fileSystem.copyFromLocalFile(src, dst);

    return reefTarUriValue;
  } catch (final IOException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Compresses a file.
 *
 * @param name the compressor name, i.e. "gz", "bzip2", "xz", "pack200", or "deflate".
 * @param source file to compress, may not be null.
 * @param destination compressed file, may not be null.
 * @param deleteSource if true, attempt to delete file on completion. Failure to delete does not cause an exception
 *            to be thrown or affect return value.
 *
 * @return true if source file compressed.
 * @throws IOException on IO exception.
 */
public static boolean execute(final String name, final File source, final File destination,
        final boolean deleteSource) throws IOException {
    if (!source.exists()) {
        return false;
    }
    LOGGER.debug("Starting {} compression of {}", name, source.getPath() );
    try (final FileInputStream input = new FileInputStream(source);
            final BufferedOutputStream output = new BufferedOutputStream(
                    new CompressorStreamFactory().createCompressorOutputStream(name, new FileOutputStream(
                            destination)))) {
        IOUtils.copy(input, output, BUF_SIZE);
        LOGGER.debug("Finished {} compression of {}", name, source.getPath() );
    } catch (final CompressorException e) {
        throw new IOException(e);
    }

    if (deleteSource) {
        try {
            if (Files.deleteIfExists(source.toPath())) {
                LOGGER.debug("Deleted {}", source.toString());
            } else {
                LOGGER.warn("Unable to delete {} after {} compression. File did not exist", source.toString(), name);
            }
        } catch (final Exception ex) {
            LOGGER.warn("Unable to delete {} after {} compression, {}", source.toString(), name, ex.getMessage());
        }
    }

    return true;
}
 

/**
 * Store the provided input stream to the desired file.
 *
 * @param in      the input stream. It is not closed by this method.
 * @param outFile the desired target file
 * @return the target file
 * @throws FileNotFoundException
 * @throws IOException
 */
public static File streamToFile (final InputStream in,
                                 final File outFile)
        throws FileNotFoundException, IOException
{
    final OutputStream out = new FileOutputStream(outFile);
    IOUtils.copy(in, out);
    out.close();

    return outFile;
}
 

public void executeWorkItem(WorkItem workItem,
                            WorkItemManager manager) {
    String archive = (String) workItem.getParameter("Archive");
    List<File> files = (List<File>) workItem.getParameter("Files");

    try {
        RequiredParameterValidator.validate(this.getClass(),
                                            workItem);

        OutputStream outputStream = new FileOutputStream(new File(archive));
        ArchiveOutputStream os = new ArchiveStreamFactory().createArchiveOutputStream("tar",
                                                                                      outputStream);

        if (files != null) {
            for (File file : files) {
                final TarArchiveEntry entry = new TarArchiveEntry("testdata/test1.xml");
                entry.setModTime(0);
                entry.setSize(file.length());
                entry.setUserId(0);
                entry.setGroupId(0);
                entry.setMode(0100000);
                os.putArchiveEntry(entry);
                IOUtils.copy(new FileInputStream(file),
                             os);
            }
        }
        os.closeArchiveEntry();
        os.close();
        manager.completeWorkItem(workItem.getId(),
                                 null);
    } catch (Exception e) {
        handleException(e);
        manager.abortWorkItem(workItem.getId());
    }
}
 

/**
 * Ungzip an input file into an output file.
 * <p>
 * The output file is created in the output folder, having the same name
 * as the input file, minus the '.gz' extension.
 *
 * @param inputFile     the input .gz file
 * @param outputDir     the output directory file.
 * @throws IOException
 * @throws FileNotFoundException
 *
 * @return  The {@File} with the ungzipped content.
 */
public static File unGzip(final File inputFile, final File outputDir) throws FileNotFoundException, IOException {

    logger.info(String.format("Ungzipping %s to dir %s.", inputFile.getAbsolutePath(), outputDir.getAbsolutePath()));

    final File outputFile = new File(outputDir, inputFile.getName().substring(0, inputFile.getName().length() - 3));

    final GZIPInputStream in = new GZIPInputStream(new FileInputStream(inputFile));
    final FileOutputStream out = new FileOutputStream(outputFile);

    IOUtils.copy(in, out);

    in.close();
    out.close();

    return outputFile;
}
 

/**
 * Unpacks a given zip file to {@link ApplicationLayout#getExtensionInstallationDir}. The 
 * file permissions in the original zip will be retained.
 * <p>
 * Note: This method uses the Apache zip files since they keep track of permissions info; 
 * the built-in java objects (ZipEntry et al.) do not.
 * 
 * @param zipFile the zip file to unpack
 * @param monitor the task monitor
 * @throws ExtensionException if any part of the unzipping fails, or if the target location is invalid
 * @throws CancelledException if the user cancels the unzip
 * @throws IOException if error unzipping zip file
 */
private static void unzipToInstallationFolder(File zipFile, TaskMonitor monitor)
		throws ExtensionException, CancelledException, IOException {

	ApplicationLayout layout = Application.getApplicationLayout();

	if (!isZip(zipFile)) {
		throw new ExtensionException(zipFile.getAbsolutePath() + " is not a valid zip file",
			ExtensionExceptionType.ZIP_ERROR);
	}

	if (layout.getExtensionInstallationDir() == null ||
		!layout.getExtensionInstallationDir().exists()) {
		throw new ExtensionException(
			"Extension installation directory is not valid: " +
				layout.getExtensionInstallationDir(),
			ExtensionExceptionType.INVALID_INSTALL_LOCATION);
	}

	try (ZipFile zFile = new ZipFile(zipFile)) {

		Enumeration<ZipArchiveEntry> entries = zFile.getEntries();
		while (entries.hasMoreElements()) {

			ZipArchiveEntry entry = entries.nextElement();

			String filePath =
				(layout.getExtensionInstallationDir() + File.separator + entry.getName());

			File file = new File(filePath);

			if (entry.isDirectory()) {
				file.mkdirs();
			}
			else {
				try (OutputStream outputStream =
					new BufferedOutputStream(new FileOutputStream(file))) {

					// Create the file at the new location...
					IOUtils.copy(zFile.getInputStream(entry), outputStream);

					// ...and update its permissions. But only continue if the zip
					// was created on a unix platform. If not we cannot use the posix
					// libraries to set permissions. 
					if (entry.getPlatform() == ZipArchiveEntry.PLATFORM_UNIX) {

						int mode = entry.getUnixMode();
						if (mode != 0) { // 0 indicates non-unix platform
							Set<PosixFilePermission> perms = getPermissions(mode);

							try {
								Files.setPosixFilePermissions(file.toPath(), perms);
							}
							catch (UnsupportedOperationException e) {
								// Need to catch this, as Windows does not support the
								// posix call. This is not an error, however, and should
								// just silently fail.
							}
						}
					}
				}
			}
		}
	}
}
 

@Test
public void testAppender() throws Exception {
    final Path path = Paths.get(DIR, "rollingtest.log");
    if (Files.exists(path) && createOnDemand) {
        Assert.fail(String.format("Unexpected file: %s (%s bytes)", path, Files.getAttribute(path, "size")));
    }
    for (int i = 0; i < 500; ++i) {
        logger.debug("This is test message number " + i);
    }
    try {
        Thread.sleep(100);
    } catch (final InterruptedException ie) {
        // Ignore the error.
    }

    final File dir = new File(DIR);
    assertTrue("Directory not created", dir.exists() && dir.listFiles().length > 0);
    final File[] files = dir.listFiles();
    assertNotNull(files);
    assertThat(files, hasItemInArray(that(hasName(that(endsWith(fileExtension))))));

    final FileExtension ext = FileExtension.lookup(fileExtension);
    if (ext == null || FileExtension.ZIP == ext || FileExtension.PACK200 == ext) {
        return; // Apache Commons Compress cannot deflate zip? TODO test decompressing these formats
    }
    // Stop the context to make sure all files are compressed and closed. Trying to remedy failures in CI builds.
    if (!loggerContextRule.getLoggerContext().stop(30, TimeUnit.SECONDS)) {
        System.err.println("Could not stop cleanly " + loggerContextRule + " for " + this);
    }
    for (final File file : files) {
        if (file.getName().endsWith(fileExtension)) {
            CompressorInputStream in = null;
            try (FileInputStream fis = new FileInputStream(file)) {
                try {
                    in = new CompressorStreamFactory().createCompressorInputStream(ext.name().toLowerCase(), fis);
                } catch (final CompressorException ce) {
                    ce.printStackTrace();
                    fail("Error creating input stream from " + file.toString() + ": " + ce.getMessage());
                }
                final ByteArrayOutputStream baos = new ByteArrayOutputStream();
                assertNotNull("No input stream for " + file.getName(), in);
                try {
                    IOUtils.copy(in, baos);
                } catch (final Exception ex) {
                    ex.printStackTrace();
                    fail("Unable to decompress " + file.getAbsolutePath());
                }
                final String text = new String(baos.toByteArray(), Charset.defaultCharset());
                final String[] lines = text.split("[\\r\\n]+");
                for (final String line : lines) {
                    assertTrue(line.contains(
                            "DEBUG o.a.l.l.c.a.r.RollingAppenderSizeTest [main] This is test message number"));
                }
            } finally {
                Closer.close(in);
            }
        }
    }
}