Java Code Examples for org.apache.commons.compress.archivers.tar.TarArchiveInputStream#read()

public static void untar(TarArchiveInputStream tarIn, File targetDir) throws IOException {
    byte[] b = new byte[4096];
    TarArchiveEntry tarEntry;
    while ((tarEntry = tarIn.getNextTarEntry()) != null) {
        final File file = new File(targetDir, tarEntry.getName());
        if (tarEntry.isDirectory()) {
            if (!file.mkdirs()) {
                throw new IOException("Unable to create folder " + file.getAbsolutePath());
            }
        } else {
            final File parent = file.getParentFile();
            if (!parent.exists()) {
                if (!parent.mkdirs()) {
                    throw new IOException("Unable to create folder " + parent.getAbsolutePath());
                }
            }
            try (FileOutputStream fos = new FileOutputStream(file)) {
                int r;
                while ((r = tarIn.read(b)) != -1) {
                    fos.write(b, 0, r);
                }
            }
        }
    }
}
 

private static List<String> unTar(InputStream inputStream, File destDir) throws Exception {
    List<String> fileNames = new ArrayList<>();
    TarArchiveInputStream tarIn = new TarArchiveInputStream(inputStream, BUFFER_SIZE, CharsetUtil.GBK);
    TarArchiveEntry entry;
    try {
        while ((entry = tarIn.getNextTarEntry()) != null) {
            fileNames.add(entry.getName());
            if (entry.isDirectory()) {
                //是目录
                FileUtil.mkdir(new File(destDir, entry.getName()));
                //创建空目录
            } else {
                //是文件
                File tmpFile = new File(destDir, entry.getName());
                //创建输出目录
                FileUtil.mkParentDirs(destDir);
                OutputStream out = null;
                try {
                    out = new FileOutputStream(tmpFile);
                    int length;
                    byte[] b = new byte[2048];
                    while ((length = tarIn.read(b)) != -1) {
                        out.write(b, 0, length);
                    }
                } finally {
                    IOUtils.closeQuietly(out);
                }
            }
        }
    } finally {
        IOUtils.closeQuietly(tarIn);
    }
    return fileNames;
}
 

@Override
public Map<String, String> unpackageFlowFile(final InputStream in, final OutputStream out) throws IOException {
    flowFilesRead++;
    final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
    final TarArchiveEntry attribEntry = tarIn.getNextTarEntry();
    if (attribEntry == null) {
        return null;
    }

    final Map<String, String> attributes;
    if (attribEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
        attributes = getAttributes(tarIn);
    } else {
        throw new IOException("Expected two tar entries: "
                + FlowFilePackagerV1.FILENAME_CONTENT + " and "
                + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
    }

    final TarArchiveEntry contentEntry = tarIn.getNextTarEntry();

    if (contentEntry != null && contentEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
        final byte[] buffer = new byte[512 << 10];//512KB
        int bytesRead = 0;
        while ((bytesRead = tarIn.read(buffer)) != -1) { //still more data to read
            if (bytesRead > 0) {
                out.write(buffer, 0, bytesRead);
            }
        }
        out.flush();
    } else {
        throw new IOException("Expected two tar entries: "
                + FlowFilePackagerV1.FILENAME_CONTENT + " and "
                + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
    }

    return attributes;
}
 

private static void unpackEntries(TarArchiveInputStream tis,
    TarArchiveEntry entry, File outputDir) throws IOException {
  if (entry.isDirectory()) {
    File subDir = new File(outputDir, entry.getName());
    if (!subDir.mkdirs() && !subDir.isDirectory()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }

    for (TarArchiveEntry e : entry.getDirectoryEntries()) {
      unpackEntries(tis, e, subDir);
    }

    return;
  }

  File outputFile = new File(outputDir, entry.getName());
  if (!outputFile.getParentFile().exists()) {
    if (!outputFile.getParentFile().mkdirs()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }
  }

  int count;
  byte data[] = new byte[2048];
  BufferedOutputStream outputStream = new BufferedOutputStream(
      new FileOutputStream(outputFile));

  while ((count = tis.read(data)) != -1) {
    outputStream.write(data, 0, count);
  }

  outputStream.flush();
  outputStream.close();
}
 

/**
 * 文件解归档
 *
 * @param destFile
 *            目标文件
 * @param tais
 *            TarArchiveInputStream
 * @throws Exception
 */
private static void dearchiveFile(File destFile, TarArchiveInputStream tais)
        throws Exception {

    BufferedOutputStream bos = new BufferedOutputStream(
            new FileOutputStream(destFile));

    int count;
    byte data[] = new byte[BUFFER];
    while ((count = tais.read(data, 0, BUFFER)) != -1) {
        bos.write(data, 0, count);
    }

    bos.close();
}
 

private static void unpackEntries(TarArchiveInputStream tis,
    TarArchiveEntry entry, File outputDir) throws IOException {
  if (entry.isDirectory()) {
    File subDir = new File(outputDir, entry.getName());
    if (!subDir.mkdirs() && !subDir.isDirectory()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }

    for (TarArchiveEntry e : entry.getDirectoryEntries()) {
      unpackEntries(tis, e, subDir);
    }

    return;
  }

  File outputFile = new File(outputDir, entry.getName());
  if (!outputFile.getParentFile().exists()) {
    if (!outputFile.getParentFile().mkdirs()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }
  }

  int count;
  byte data[] = new byte[2048];
  BufferedOutputStream outputStream = new BufferedOutputStream(
      new FileOutputStream(outputFile));

  while ((count = tis.read(data)) != -1) {
    outputStream.write(data, 0, count);
  }

  outputStream.flush();
  outputStream.close();
}
 

public static void extractTarGz(InputStream inputTarGzStream, String outDir,
    boolean logging) {
  try {
    GzipCompressorInputStream gzIn = new GzipCompressorInputStream(
        inputTarGzStream);
    TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn);

    // read Tar entries
    TarArchiveEntry entry = null;
    while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
      if (logging) {
        LOG.info("Extracting: " + outDir + File.separator + entry.getName());
      }
      if (entry.isDirectory()) { // create directory
        File f = new File(outDir + File.separator + entry.getName());
        f.mkdirs();
      } else { // decompress file
        int count;
        byte data[] = new byte[EXTRACT_BUFFER_SIZE];

        FileOutputStream fos = new FileOutputStream(outDir + File.separator
            + entry.getName());
        BufferedOutputStream dest = new BufferedOutputStream(fos,
            EXTRACT_BUFFER_SIZE);
        while ((count = tarIn.read(data, 0, EXTRACT_BUFFER_SIZE)) != -1) {
          dest.write(data, 0, count);
        }
        dest.close();
      }
    }

    // close input stream
    tarIn.close();

  } catch (IOException e) {
    LOG.error("IOException: " + e.getMessage());
  }
}
 

/** @apiNote Caller should close `in` after calling this method. */
public static void untar(InputStream in, File targetDir) throws IOException {
  final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
  byte[] b = new byte[BUF_SIZE];
  TarArchiveEntry tarEntry;
  while ((tarEntry = tarIn.getNextTarEntry()) != null) {
    final File file = new File(targetDir, tarEntry.getName());
    if (tarEntry.isDirectory()) {
      if (!file.mkdirs()) {
        throw new IOException("Unable to create folder " + file.getAbsolutePath());
      }
    } else {
      final File parent = file.getParentFile();
      if (!parent.exists()) {
        if (!parent.mkdirs()) {
          throw new IOException("Unable to create folder " + parent.getAbsolutePath());
        }
      }
      try (FileOutputStream fos = new FileOutputStream(file)) {
        int r;
        while ((r = tarIn.read(b)) != -1) {
          fos.write(b, 0, r);
        }
      }
    }
  }
}
 

/**
 * Sets the agent version.
 */
private static void setAgentVersion() {
	// current agent version
	String agentBuildVersion = new String();
	try {
		// Resource base path.
		String basePath = AppStoreWrapper.getResourcePath();
		// Creating agent bundle path.
		String agentBundlePath = basePath + "scripts/agent/"
				+ AgentDeployer.AGENT_BUNDLE_NAME;

		FileInputStream fileInputStream = new FileInputStream(
				agentBundlePath);
		BufferedInputStream bufferedInputStream = new BufferedInputStream(
				fileInputStream);
		GzipCompressorInputStream gzInputStream = new GzipCompressorInputStream(
				bufferedInputStream);
		TarArchiveInputStream tarInputStream = new TarArchiveInputStream(
				gzInputStream);
		TarArchiveEntry entry = null;

		while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) {
			if (entry.getName()
					.equals(AgentDeployer.AGENT_VERSION_FILENAME)) {
				final int BUFFER = 10;
				byte data[] = new byte[BUFFER];
				tarInputStream.read(data, 0, BUFFER);
				String version = new String(data);
				agentBuildVersion = version.trim();
				// Set the agent version in the AppStore with key as
				// agentVersion
				AppStore.setObject(AppStoreWrapper.KEY_AGENT_VERISON,
						agentBuildVersion);
			}
		}
	} catch (Exception e) {
		// log error message.
		log.error(e.getMessage(), e);
	}
}
 

@Override
public Map<String, String> unpackageFlowFile(final InputStream in, final OutputStream out) throws IOException {
    flowFilesRead++;
    final TarArchiveInputStream tarIn = new TarArchiveInputStream(in);
    final TarArchiveEntry attribEntry = tarIn.getNextTarEntry();
    if (attribEntry == null) {
        return null;
    }

    final Map<String, String> attributes;
    if (attribEntry.getName().equals(FlowFilePackagerV1.FILENAME_ATTRIBUTES)) {
        attributes = getAttributes(tarIn);
    } else {
        throw new IOException("Expected two tar entries: "
                + FlowFilePackagerV1.FILENAME_CONTENT + " and "
                + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
    }

    final TarArchiveEntry contentEntry = tarIn.getNextTarEntry();

    if (contentEntry != null && contentEntry.getName().equals(FlowFilePackagerV1.FILENAME_CONTENT)) {
        final byte[] buffer = new byte[512 << 10];//512KB
        int bytesRead = 0;
        while ((bytesRead = tarIn.read(buffer)) != -1) { //still more data to read
            if (bytesRead > 0) {
                out.write(buffer, 0, bytesRead);
            }
        }
        out.flush();
    } else {
        throw new IOException("Expected two tar entries: "
                + FlowFilePackagerV1.FILENAME_CONTENT + " and "
                + FlowFilePackagerV1.FILENAME_ATTRIBUTES);
    }

    return attributes;
}
 

private static void unpackEntries(TarArchiveInputStream tis,
                                  TarArchiveEntry entry, File outputDir) throws IOException {
    if (entry.isDirectory()) {
        File subDir = new File(outputDir, entry.getName());
        if (!subDir.mkdirs() && !subDir.isDirectory()) {
            throw new IOException("Mkdirs failed to create tar internal dir "
                    + outputDir);
        }
        for (TarArchiveEntry e : entry.getDirectoryEntries()) {
            unpackEntries(tis, e, subDir);
        }
        return;
    }
    File outputFile = new File(outputDir, entry.getName());
    if (!outputFile.getParentFile().exists()) {
        if (!outputFile.getParentFile().mkdirs()) {
            throw new IOException("Mkdirs failed to create tar internal dir "
                    + outputDir);
        }
    }
    int count;
    byte data[] = new byte[2048];
    BufferedOutputStream outputStream = new BufferedOutputStream(
            new FileOutputStream(outputFile));

    while ((count = tis.read(data)) != -1) {
        outputStream.write(data, 0, count);
    }
    outputStream.flush();
    outputStream.close();
}
 

@Scheduled(fixedDelay = 86400 * 1000) //Every 24H
@PostConstruct
private void downloadIPLocationDatabase() throws Exception {
    if(config.getGeoIP().isEnabled()){
        logger.info("Downloading GEOIP Database from: "+config.getGeoIP().getUrl());

        File tmpGEOIPFileRound = File.createTempFile("geoIP", "mmdb");
        FileOutputStream fos = new FileOutputStream(tmpGEOIPFileRound);
        HttpURLConnection connection = getURLConnection(config.getGeoIP().getUrl());
        if (connection.getResponseCode() != 200) {
            return;
        }
        TarArchiveInputStream tarGzGeoIPStream = new TarArchiveInputStream(new GZIPInputStream(connection.getInputStream()));
        TarArchiveEntry entry= null;
        int offset;
        long pointer=0;
        while ((entry = tarGzGeoIPStream.getNextTarEntry()) != null) {
            pointer+=entry.getSize();
            if(entry.getName().endsWith("GeoLite2-City.mmdb")){
                byte[] content = new byte[(int) entry.getSize()];
                offset=0;
                //FileInputStream fis = new FileInputStream(entry.getFile());
                //IOUtils.copy(fis,fos);
                //tarGzGeoIPStream.skip(pointer);
                //tarGzGeoIPStream.read(content,offset,content.length-offset);
                //IOUtils.write(content,fos);
                int r;
                byte[] b = new byte[1024];
                while ((r = tarGzGeoIPStream.read(b)) != -1) {
                    fos.write(b, 0, r);
                }
                //fis.close();
                break;
            }
        }
        tarGzGeoIPStream.close();
        fos.flush();
        fos.close();
        connection.disconnect();
        geoIPDB = new DatabaseReader.Builder(tmpGEOIPFileRound).withCache(new CHMCache()).build();
        if (tmpGEOIPFile != null && tmpGEOIPFile.exists()) {
            tmpGEOIPFile.delete();
        }
        tmpGEOIPFile = tmpGEOIPFileRound;
    }
}
 

private static void unpackEntries(TarArchiveInputStream tis,
                                  TarArchiveEntry entry, File outputDir) throws IOException {
  String targetDirPath = outputDir.getCanonicalPath() + File.separator;
  File outputFile = new File(outputDir, entry.getName());
  if (!outputFile.getCanonicalPath().startsWith(targetDirPath)) {
    throw new IOException("expanding " + entry.getName()
        + " would create entry outside of " + outputDir);
  }

  if (entry.isDirectory()) {
    File subDir = new File(outputDir, entry.getName());
    if (!subDir.mkdirs() && !subDir.isDirectory()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }

    for (TarArchiveEntry e : entry.getDirectoryEntries()) {
      unpackEntries(tis, e, subDir);
    }

    return;
  }

  if (entry.isSymbolicLink()) {
    // Create symbolic link relative to tar parent dir
    Files.createSymbolicLink(FileSystems.getDefault()
            .getPath(outputDir.getPath(), entry.getName()),
        FileSystems.getDefault().getPath(entry.getLinkName()));
    return;
  }

  if (!outputFile.getParentFile().exists()) {
    if (!outputFile.getParentFile().mkdirs()) {
      throw new IOException("Mkdirs failed to create tar internal dir "
          + outputDir);
    }
  }

  if (entry.isLink()) {
    File src = new File(outputDir, entry.getLinkName());
    HardLink.createHardLink(src, outputFile);
    return;
  }

  int count;
  byte data[] = new byte[2048];
  try (BufferedOutputStream outputStream = new BufferedOutputStream(
      new FileOutputStream(outputFile));) {

    while ((count = tis.read(data)) != -1) {
      outputStream.write(data, 0, count);
    }

    outputStream.flush();
  }
}
 

/**
 * Extracts files to the specified destination
 *
 * @param file the file to extract to
 * @param dest the destination directory
 * @throws IOException
 */
public static void unzipFileTo(String file, String dest) throws IOException {
    File target = new File(file);
    if (!target.exists())
        throw new IllegalArgumentException("Archive doesnt exist");
    FileInputStream fin = new FileInputStream(target);
    int BUFFER = 2048;
    byte data[] = new byte[BUFFER];

    if (file.endsWith(".zip") || file.endsWith(".jar")) {
        try(ZipInputStream zis = new ZipInputStream(fin)) {
            //get the zipped file list entry
            ZipEntry ze = zis.getNextEntry();

            while (ze != null) {
                String fileName = ze.getName();
                File newFile = new File(dest + File.separator + fileName);

                if (ze.isDirectory()) {
                    newFile.mkdirs();
                    zis.closeEntry();
                    ze = zis.getNextEntry();
                    continue;
                }

                FileOutputStream fos = new FileOutputStream(newFile);

                int len;
                while ((len = zis.read(data)) > 0) {
                    fos.write(data, 0, len);
                }

                fos.close();
                ze = zis.getNextEntry();
                log.debug("File extracted: " + newFile.getAbsoluteFile());
            }

            zis.closeEntry();
        }
    } else if (file.endsWith(".tar.gz") || file.endsWith(".tgz")) {

        BufferedInputStream in = new BufferedInputStream(fin);
        GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in);
        TarArchiveInputStream tarIn = new TarArchiveInputStream(gzIn);

        TarArchiveEntry entry;
        /* Read the tar entries using the getNextEntry method **/
        while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
            log.info("Extracting: " + entry.getName());
            /* If the entry is a directory, create the directory. */

            if (entry.isDirectory()) {
                File f = new File(dest + File.separator + entry.getName());
                f.mkdirs();
            }
            /*
             * If the entry is a file,write the decompressed file to the disk
             * and close destination stream.
             */
            else {
                int count;
                try(FileOutputStream fos = new FileOutputStream(dest + File.separator + entry.getName());
                    BufferedOutputStream destStream = new BufferedOutputStream(fos, BUFFER);) {
                    while ((count = tarIn.read(data, 0, BUFFER)) != -1) {
                        destStream.write(data, 0, count);
                    }

                    destStream.flush();
                    IOUtils.closeQuietly(destStream);
                }
            }
        }

        // Close the input stream
        tarIn.close();
    } else if (file.endsWith(".gz")) {
        File extracted = new File(target.getParent(), target.getName().replace(".gz", ""));
        if (extracted.exists())
            extracted.delete();
        extracted.createNewFile();
        try(GZIPInputStream is2 = new GZIPInputStream(fin); OutputStream fos = FileUtils.openOutputStream(extracted)) {
            IOUtils.copyLarge(is2, fos);
            fos.flush();
        }
    } else {
        throw new IllegalStateException("Unable to infer file type (compression format) from source file name: " +
                file);
    }
    target.delete();
}