Java Code Examples for org.apache.commons.compress.archivers.tar.TarArchiveInputStream

@Override
protected void processInputStream(final InputStream stream, final FlowFile tarArchivedFlowFile, final Writer writer) throws IOException {
    try (final TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(stream))) {
        TarArchiveEntry tarEntry;
        while ((tarEntry = tarIn.getNextTarEntry()) != null) {
            if (tarEntry.isDirectory()) {
                continue;
            }
            final String key = tarEntry.getName();
            final long fileSize = tarEntry.getSize();
            final InputStreamWritable inStreamWritable = new InputStreamWritable(tarIn, (int) fileSize);
            writer.append(new Text(key), inStreamWritable);
            logger.debug("Appending FlowFile {} to Sequence File", new Object[]{key});
        }
    }
}
 

@Test
public void testFileWithEmptyDirectory() throws Exception {
  Path tempDir = Files.createTempDirectory("dockerDirectoryEmptySubdirectory");
  tempDir.toFile().deleteOnExit();
  assertThat(new File(tempDir.toFile(), "emptySubDir").mkdir(), is(true));

  try (CompressedDirectory dir = CompressedDirectory.create(tempDir);
       BufferedInputStream fileIn = new BufferedInputStream(Files.newInputStream(dir.file()));
       GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(fileIn);
       TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {

    final List<String> names = new ArrayList<>();
    TarArchiveEntry entry;
    while ((entry = tarIn.getNextTarEntry()) != null) {
      final String name = entry.getName();
      names.add(name);
    }
    assertThat(names, contains("emptySubDir/"));
  }
}
 

/**
 * Wraps the parent stream into TarInputStream 
 * and positions it to read the given entry (no wildcards are applicable).
 * 
 * If no entry is given, the stream is positioned to read the first file entry.
 * 
 * @param parentStream
 * @param entryName
 * @return
 * @throws IOException
 */
public static TarArchiveInputStream getTarInputStream(InputStream parentStream, String entryName) throws IOException {
    TarArchiveInputStream tis = new TarArchiveInputStream(parentStream) ;     
    TarArchiveEntry entry;

    // find a matching entry
    while ((entry = tis.getNextTarEntry()) != null) {
    	if (entry.isDirectory()) {
    		continue; // CLS-537: skip directories, we want to read the first file
    	}
    	// when url is given without anchor; first entry in tar file is used
        if (StringUtils.isEmpty(entryName) || entry.getName().equals(entryName)) {
           	return tis;
        }
    }
    
    //no entry found report
    throw new IOException("Wrong anchor (" + entryName + ") to tar file.");
}
 

private static void extract(final Path path, final String destDirectory) throws IOException {
  try (TarArchiveInputStream fin =
      new TarArchiveInputStream(
          new GzipCompressorInputStream(new FileInputStream(path.toAbsolutePath().toString())))) {
    TarArchiveEntry entry;
    while ((entry = fin.getNextTarEntry()) != null) {
      if (entry.isDirectory()) {
        continue;
      }
      final File curfile = new File(destDirectory, entry.getName());
      final File parent = curfile.getParentFile();
      if (!parent.exists()) {
        parent.mkdirs();
      }
      IOUtils.copy(fin, new FileOutputStream(curfile));
    }
  }
}
 

@Test
public void test() throws Exception {
    final String encoding = "utf-8";

    byte[] body;

    ExtractableResponse response = RestAssured.given() //
            .contentType(ContentType.TEXT + "; charset=" + encoding).body("Hello World").post("/tarfile/post") //
            .then().extract();

    body = response.body().asByteArray();
    Assertions.assertNotNull(body);

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    ByteArrayInputStream bis = new ByteArrayInputStream(body);
    TarArchiveInputStream tis = (TarArchiveInputStream) new ArchiveStreamFactory()
            .createArchiveInputStream(ArchiveStreamFactory.TAR, bis);

    TarArchiveEntry entry = tis.getNextTarEntry();
    if (entry != null) {
        IOHelper.copy(tis, bos);
    }

    String str = bos.toString(encoding);
    Assertions.assertEquals("Hello World", str);
}
 

/**
 * Extract a single file from a tar.gz file. Does not support directories.
 * NOTE: This should not be used for batch extraction of files, due to the need to iterate over the entries until the
 * specified entry is found. Use {@link #unzipFileTo(String, String)} for batch extraction instead
 *
 * @param tarGz       A tar.gz file
 * @param destination The destination file to extract to
 * @param pathInTarGz The path in the tar.gz file to extract
 */
public static void tarGzExtractSingleFile(File tarGz, File destination, String pathInTarGz) throws IOException {
    try(TarArchiveInputStream tin = new TarArchiveInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(tarGz))))) {
        ArchiveEntry entry;
        boolean extracted = false;
        while((entry = tin.getNextTarEntry()) != null){
            String name = entry.getName();
            if(pathInTarGz.equals(name)){
                try(OutputStream os = new BufferedOutputStream(new FileOutputStream(destination))){
                    IOUtils.copy(tin, os);
                }
                extracted = true;
            }
        }
        Preconditions.checkState(extracted, "No file was extracted. File not found? %s", pathInTarGz);
    }
}
 

/**
 * Extract a "tar.gz" file into a given folder.
 * @param file
 * @param folder 
 */
public static void extractTarArchive(File file, String folder) throws IOException {
    logger.info("Extracting archive {} into folder {}", file.getName(), folder);
    // @formatter:off
    try (FileInputStream fis = new FileInputStream(file);
        BufferedInputStream bis = new BufferedInputStream(fis);
        GzipCompressorInputStream gzip = new GzipCompressorInputStream(bis); 
        TarArchiveInputStream tar = new TarArchiveInputStream(gzip)) {
   // @formatter:on
        TarArchiveEntry entry;
        while ((entry = (TarArchiveEntry) tar.getNextEntry()) != null) {
            extractEntry(entry, tar, folder);
        }
    }
  logger.info("Archive extracted"); 
}
 

@Override
public void parse(File file) throws IOException {
    mEntries = new ArrayList<>();

    BufferedInputStream fis = new BufferedInputStream(new FileInputStream(file));
    TarArchiveInputStream is = new TarArchiveInputStream(fis);
    TarArchiveEntry entry = is.getNextTarEntry();
    while (entry != null) {
        if (entry.isDirectory()) {
            continue;
        }
        if (Utils.isImage(entry.getName())) {
            mEntries.add(new TarEntry(entry, Utils.toByteArray(is)));
        }
        entry = is.getNextTarEntry();
    }

    Collections.sort(mEntries, new NaturalOrderComparator() {
        @Override
        public String stringValue(Object o) {
            return ((TarEntry) o).entry.getName();
        }
    });
}
 

public Future<?> setDataPath(String dataFilePath) {
    try (TarArchiveInputStream tarInput = new TarArchiveInputStream(new GZIPInputStream(
            new FileInputStream(dataFilePath)))) {

        TarArchiveEntry currentEntry;
        boolean hasDatabaseFile = false;
        while ((currentEntry = tarInput.getNextTarEntry()) != null) {
            if (currentEntry.getName().contains(DATABASE_FILE_NAME)) {
                hasDatabaseFile = true;
                break;
            }
        }
        if (!hasDatabaseFile) {
            return Future.failedFuture(String.format("Database file %s not found in %s archive", DATABASE_FILE_NAME,
                    dataFilePath));
        }

        databaseReader = new DatabaseReader.Builder(tarInput).fileMode(Reader.FileMode.MEMORY).build();
        return Future.succeededFuture();
    } catch (IOException e) {
        return Future.failedFuture(
                String.format("IO Exception occurred while trying to read an archive/db file: %s", e.getMessage()));
    }
}
 

private static void extractTar(String dataIn, String dataOut) throws IOException {

        try (TarArchiveInputStream inStream = new TarArchiveInputStream(
                new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(dataIn))))) {
            TarArchiveEntry tarFile;
            while ((tarFile = (TarArchiveEntry) inStream.getNextEntry()) != null) {
                if (tarFile.isDirectory()) {
                    new File(dataOut + tarFile.getName()).mkdirs();
                } else {
                    int count;
                    byte data[] = new byte[BUFFER_SIZE];

                    FileOutputStream fileInStream = new FileOutputStream(dataOut + tarFile.getName());
                    BufferedOutputStream outStream=  new BufferedOutputStream(fileInStream, BUFFER_SIZE);
                    while ((count = inStream.read(data, 0, BUFFER_SIZE)) != -1) {
                        outStream.write(data, 0, count);
                    }
                }
            }
        }
    }
 

/**
 * Gets a Reader for a file in a gzipped tarball
 * @param tarPath   Path to the tarball
 * @param fileName   File within the tarball
 * @return   The file's reader
 */
public static BufferedReader getBufferedReaderTarGz(final String tarPath, final String fileName) {
    try {
        InputStream result = null;
        final TarArchiveInputStream tarStream = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tarPath)));
        TarArchiveEntry entry = tarStream.getNextTarEntry();
        while (entry != null) {
            if (entry.getName().equals(fileName)) {
                result = tarStream;
                break;
            }
            entry = tarStream.getNextTarEntry();
        }
        if (result == null) {
            throw new UserException.BadInput("Could not find file " + fileName + " in tarball " + tarPath);
        }
        return new BufferedReader(new InputStreamReader(result));
    } catch (final IOException e) {
        throw new UserException.BadInput("Could not open compressed tarball file " + fileName + " in " + tarPath, e);
    }
}
 

@Override
public void initialize(UimaContext context)
        throws ResourceInitializationException
{
    super.initialize(context);

    try {
        tarArchiveInputStream = new TarArchiveInputStream(
                new GZIPInputStream(new FileInputStream(sourceLocation)));

        fastForwardToNextValidEntry();
    }
    catch (IOException ex) {
        throw new ResourceInitializationException(ex);
    }
}
 

@Override
public boolean installLSP() {
    return lspInstallHelper("Would you like to proceed with downloading the Java Language Server by the Eclipse Foundation? This will take up about 94MB.", "https://github.com/eclipse/eclipse.jdt.ls", () -> {
        LOGGER.info("Installing Java LSP server...");
        lspPath.mkdirs();

        var tarGz = new File(lspPath, "jdt-language-server-latest.tar.gz");
        FileUtils.copyURLToFile(new URL("http://download.eclipse.org/jdtls/snapshots/jdt-language-server-latest.tar.gz"), tarGz);

        var destDir = new File(lspPath, "jdt-language-server-latest");

        try (var in = new TarArchiveInputStream(
                new GzipCompressorInputStream(
                        new BufferedInputStream(
                                new FileInputStream(tarGz))))) {
            ExtractUtils.untar(in, destDir);
        }

        tarGz.delete();

        LOGGER.info("Successfully downloaded the Java Language Server");
        return true;
    });
}
 

private void untar(Path archiveFile, Path project) throws IOException {
	try (TarArchiveInputStream input = new TarArchiveInputStream(
			new GzipCompressorInputStream(Files.newInputStream(archiveFile)))) {
		TarArchiveEntry entry = null;
		while ((entry = input.getNextTarEntry()) != null) {
			Path path = project.resolve(entry.getName());
			if (entry.isDirectory()) {
				Files.createDirectories(path);
			}
			else {
				Files.createDirectories(path.getParent());
				Files.write(path, StreamUtils.copyToByteArray(input));
			}
			applyPermissions(path, getPosixFilePermissions(entry.getMode()));
		}
	}
}
 

/**
 * 文件 解归档
 *
 * @param destFile
 *            目标文件
 * @param tais
 *            ZipInputStream
 * @throws Exception
 */
private static void dearchive(File destFile, TarArchiveInputStream tais)
        throws Exception {

    TarArchiveEntry entry = null;
    while ((entry = tais.getNextTarEntry()) != null) {

        // 文件
        String dir = destFile.getPath() + File.separator + entry.getName();

        File dirFile = new File(dir);

        // 文件检查
        fileProber(dirFile);

        if (entry.isDirectory()) {
            dirFile.mkdirs();
        } else {
            dearchiveFile(dirFile, tais);
        }

    }
}
 

private void downloadDatabase() throws IOException {
    // Avoid Socket leak with the parameters in case download url has proxy
    // https://rsl1122.github.io/mishaps/java_socket_leak_incident
    Properties properties = System.getProperties();
    properties.setProperty("sun.net.client.defaultConnectTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
    properties.setProperty("sun.net.client.defaultReadTimeout", Long.toString(TimeUnit.MINUTES.toMillis(1L)));
    properties.setProperty("sun.net.http.retryPost", Boolean.toString(false));

    String downloadFrom = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=DEyDUKfCwNbtc5eK&suffix=tar.gz";
    URL downloadSite = new URL(downloadFrom);
    try (
            InputStream in = downloadSite.openStream();
            GZIPInputStream gzipIn = new GZIPInputStream(in);
            TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn);
            FileOutputStream fos = new FileOutputStream(geolocationDB.getAbsoluteFile())
    ) {
        findAndCopyFromTar(tarIn, fos);
    }
}
 

private static void assertEntry(File file, String name) {
    boolean exist = false;

    try (TarArchiveInputStream tar = new TarArchiveInputStream(
            new GzipCompressorInputStream(
                            new FileInputStream(file)))) {


        ArchiveEntry entry;
        while ((entry = tar.getNextEntry()) != null) {
            if (entry.getName().equals(name)) {
                exist = true;
                break;
            }
        }

    } catch (IOException e) {
        e.printStackTrace();
        fail(e.getMessage());
    }

    if (!exist) {
        fail("zip entry " + name + " not exist!");
    }

}
 

public static void untar(
        InputStream tar,
        File parentDir
) throws IOException {
    TarArchiveInputStream tin = new TarArchiveInputStream(tar);
    ArchiveEntry e;
    while ((e = tin.getNextEntry()) != null) {
        File f = new File(parentDir, e.getName());
        f.setLastModified(e.getLastModifiedDate().getTime());
        f.getParentFile().mkdirs();
        if (e.isDirectory()) {
            f.mkdir();
            continue;
        }
        long size = e.getSize();
        checkFileSize(size);
        try (OutputStream out = new FileOutputStream(f)) {
            /* TarInputStream pretends each
               entry's EOF is the stream's EOF */
            IOUtils.copy(tin, out);
        }
    }
}
 

byte[] readArtifactContext(Path input) {
    try (InputStream in = Files.newInputStream(getDeflatedFile(input));
         TarArchiveInputStream tarIn = new TarArchiveInputStream(in)) {
        ArchiveEntry entry = tarIn.getNextEntry();
        while (entry != null) {
            if (ARTIFACT_CONTEXT.equals(entry.getName())) {
                byte[] data = new byte[(int) entry.getSize()];
                tarIn.read(data, 0, data.length);
                return data;
            }
            entry = tarIn.getNextEntry();
        }
    } catch (IOException e) {
        LOG.warn("[{} - {}]: Error extracting tar  {}.", request.getLogName(), request.getId(), e.getMessage());
        throw new IllegalStateException(e);
    }
    LOG.error("Missing artifact_context.xml in {}", input);
    throw new IllegalStateException("Missing artifact_context.xml in " + input.toString());
}
 

public JsonObject getMetaInfJson() throws IOException {

        try (TarArchiveInputStream tarInput = new TarArchiveInputStream(new GzipCompressorInputStream(new ByteArrayInputStream(pBytes)))) {

            TarArchiveEntry currentEntry = tarInput.getNextTarEntry();
            while (currentEntry != null) {
                if (currentEntry.getName().equals("metadata.json")) {
                    byte[] buf = new byte[(int) currentEntry.getSize()];
                    tarInput.read(buf, 0, (int) currentEntry.getSize());

                    try (InputStream stream = new ByteArrayInputStream(buf)) {
                        try (JsonReader reader = Json.createReader(stream)) {

                            return (JsonObject) reader.read();
                        }
                    }

                }
                currentEntry = tarInput.getNextTarEntry();

            }
        }

        return null;
    }
 

/**
 * Extract a single file from a tar.gz file. Does not support directories.
 * NOTE: This should not be used for batch extraction of files, due to the need to iterate over the entries until the
 * specified entry is found. Use {@link #unzipFileTo(String, String)} for batch extraction instead
 *
 * @param tarGz       A tar.gz file
 * @param destination The destination file to extract to
 * @param pathInTarGz The path in the tar.gz file to extract
 */
public static void tarGzExtractSingleFile(File tarGz, File destination, String pathInTarGz) throws IOException {
    try(TarArchiveInputStream tin = new TarArchiveInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(tarGz))))) {
        ArchiveEntry entry;
        boolean extracted = false;
        while((entry = tin.getNextTarEntry()) != null){
            String name = entry.getName();
            if(pathInTarGz.equals(name)){
                try(OutputStream os = new BufferedOutputStream(new FileOutputStream(destination))){
                    IOUtils.copy(tin, os);
                }
                extracted = true;
            }
        }
        Preconditions.checkState(extracted, "No file was extracted. File not found? %s", pathInTarGz);
    }
}
 

protected Map<String, String> getAttributes(final TarArchiveInputStream stream) throws IOException {

        final Properties props = new Properties();
        props.loadFromXML(new NonCloseableInputStream(stream));

        final Map<String, String> result = new HashMap<>();
        for (final Entry<Object, Object> entry : props.entrySet()) {
            final Object keyObject = entry.getKey();
            final Object valueObject = entry.getValue();
            if (!(keyObject instanceof String)) {
                throw new IOException("Flow file attributes object contains key of type "
                        + keyObject.getClass().getCanonicalName()
                        + " but expected java.lang.String");
            } else if (!(keyObject instanceof String)) {
                throw new IOException("Flow file attributes object contains value of type "
                        + keyObject.getClass().getCanonicalName()
                        + " but expected java.lang.String");
            }

            final String key = (String) keyObject;
            final String value = (String) valueObject;
            result.put(key, value);
        }

        return result;
    }
 

public static void dumpContainerLogDirToTarget(DockerClient docker, String containerId,
                                               String path) {
    final int READ_BLOCK_SIZE = 10000;

    try (InputStream dockerStream = docker.copyArchiveFromContainerCmd(containerId, path).exec();
         TarArchiveInputStream stream = new TarArchiveInputStream(dockerStream)) {
        TarArchiveEntry entry = stream.getNextTarEntry();
        while (entry != null) {
            if (entry.isFile()) {
                File output = new File(getTargetDirectory(containerId), entry.getName().replace("/", "-"));
                try (FileOutputStream os = new FileOutputStream(output)) {
                    byte[] block = new byte[READ_BLOCK_SIZE];
                    int read = stream.read(block, 0, READ_BLOCK_SIZE);
                    while (read > -1) {
                        os.write(block, 0, read);
                        read = stream.read(block, 0, READ_BLOCK_SIZE);
                    }
                }
            }
            entry = stream.getNextTarEntry();
        }
    } catch (RuntimeException|IOException e) {
        LOG.error("Error reading logs from container {}", containerId, e);
    }
}
 

private void extractDatabase(InputStream in, String database) throws IOException {
  GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(in);
  try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
    TarArchiveEntry entry;

    while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
      if (StringUtils.endsWith(entry.getName(), database)) {
        int count;
        byte[] data = new byte[4096];

        String outFile = Paths.get(entry.getName()).getFileName().toString();
        FileOutputStream fos =
            new FileOutputStream(FileUtil.appendPath(location, outFile), false);
        try (BufferedOutputStream dest = new BufferedOutputStream(fos, 4096)) {
          while ((count = tarIn.read(data, 0, 4096)) != -1) {
            dest.write(data, 0, count);
          }
        }
      }
    }
  }
}
 

private static List<String> unTgz(File tarFile, File directory) throws IOException {
  List<String> result = new ArrayList<>();
  try (TarArchiveInputStream in = new TarArchiveInputStream(
          new GzipCompressorInputStream(new FileInputStream(tarFile)))) {
    TarArchiveEntry entry = in.getNextTarEntry();
    while (entry != null) {
      if (entry.isDirectory()) {
        entry = in.getNextTarEntry();
        continue;
      }
      File curfile = new File(directory, entry.getName());
      File parent = curfile.getParentFile();
      if (!parent.exists()) {
        parent.mkdirs();
      }
      try (OutputStream out = new FileOutputStream(curfile)) {
        IOUtils.copy(in, out);
      }
      result.add(entry.getName());
      entry = in.getNextTarEntry();
    }
  }
  return result;
}
 

private byte[] getFileAsByteArray (File tarFile, Function<String, Boolean> filterFunc) {
    byte[] data = null;
    LOG.info("Looking in to file {} ", tarFile);
    try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(tarFile));
         TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(bis)) {
        TarArchiveEntry tarArchiveEntry = tarArchiveInputStream.getNextTarEntry();
        while (tarArchiveEntry != null) {
            if (filterFunc.apply(tarArchiveEntry.getName())) {
                data = IOUtils.toByteArray(tarArchiveInputStream);
                break;
            }
            tarArchiveEntry = tarArchiveInputStream.getNextTarEntry();
        }
    } catch (IOException e) {
        LOG.warn("Exception occured while looking in to tar file [] ", filterFunc, tarFile, e);
    }
    return data;
}
 

protected TarArchiveInputStream getTarFile() throws FileSystemException {
    if (tarFile == null && this.file.exists()) {
        recreateTarFile();
    }

    return tarFile;
}
 

@Override
public InputStream getInputStream(GFile file, TaskMonitor monitor)
		throws IOException, CancelledException {

	TarMetadata tmd = fsih.getMetadata(file);
	if (tmd == null) {
		throw new IOException("Unknown file " + file);
	}

	// Open a new instance of the tar file, seek to the requested embedded file,
	// and return the inputstream to the caller, who will close it when done.
	TarArchiveInputStream tarInput =
		new TarArchiveInputStream(new FileInputStream(containerFile));

	int fileNum = 0;
	TarArchiveEntry tarEntry;
	while ((tarEntry = tarInput.getNextTarEntry()) != null) {
		if (fileNum == tmd.fileNum) {
			if (!tmd.tarArchiveEntry.getName().equals(tarEntry.getName())) {
				throw new IOException("Mismatch between filenum and tarEntry for " + file);
			}
			return tarInput;
		}
		fileNum++;
	}
	throw new IOException("Could not find requested file " + file);
}
 

protected static List<String> tarGzListFiles(File file, boolean isTarGz) throws IOException {
    try(TarArchiveInputStream tin =
                isTarGz ? new TarArchiveInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)))) :
                        new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(file)))) {
        ArchiveEntry entry;
        List<String> out = new ArrayList<>();
        while((entry = tin.getNextTarEntry()) != null){
            String name = entry.getName();
            out.add(name);
        }
        return out;
    }
}
 

private void extractTarGzFile(Path file, Path toDirectory) throws IOException {
    Files.createDirectories(toDirectory);
    try (FileInputStream fin = new FileInputStream(file.toFile())) {
        try (BufferedInputStream bin = new BufferedInputStream(fin)) {
            try (GzipCompressorInputStream gzipArchive = new GzipCompressorInputStream(bin)) {
                try (TarArchiveInputStream tarArchive = new TarArchiveInputStream(gzipArchive)) {
                    extractTar(toDirectory, tarArchive);
                }
            }
        }
    }
}