java.util.zip.GZIPOutputStream Java Examples

/**
 * Write a new inventory report to S3 and returns a locator which includes this inventory report's information
 * @return Locator which includes the information of this new report
 * @throws IOException thrown when GZIPOutputStream not created successfully or csvMapper.write() fails
 */
public InventoryManifest.Locator writeCsvFile(List<InventoryReportLine> inventoryReportLine) throws IOException{
    CsvMapper csvMapper = new CsvMapper();
    csvMapper.enable(JsonGenerator.Feature.IGNORE_UNKNOWN);
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream);
    csvMapper.writer(schema).writeValues(gzipOutputStream).writeAll(inventoryReportLine).close();
    byte[] zipByteArray = byteArrayOutputStream.toByteArray();

    InputStream zipInputStream = new ByteArrayInputStream(zipByteArray);
    ObjectMetadata metaData = new ObjectMetadata();
    metaData.setContentLength(zipByteArray.length);
    PutObjectRequest request = new PutObjectRequest(bucketName, outputInventoryReportKey, zipInputStream, metaData);
    s3Client.putObject(request);

    return this.buildLocator(zipByteArray);
}
 

@Override
public byte[] doEncode(String topic, Object obj) {
	if (!(obj instanceof byte[])) {
		throw new IllegalArgumentException(String.format("Can not encode input of type %s",
		      obj == null ? "null" : obj.getClass()));
	}

	ByteArrayInputStream input = new ByteArrayInputStream((byte[]) obj);
	ByteArrayOutputStream bout = new ByteArrayOutputStream();
	try {
		GZIPOutputStream gout = new GZIPOutputStream(bout);
		IO.INSTANCE.copy(input, gout, AutoClose.INPUT_OUTPUT);
	} catch (IOException e) {
		throw new RuntimeException(String.format("Unexpected exception when encode %s of topic %s", obj, topic), e);
	}

	return bout.toByteArray();
}
 

void close() throws IOException {
    steCache = null;
    GZIPOutputStream stream = new GZIPOutputStream(outStream, 64 * 1024);
    ObjectOutputStream out = new ObjectOutputStream(stream);
    int size = samples.size();
    out.writeInt(size);
    out.writeLong(getSample(size-1).getTime());
    openProgress();
    for (int i=0; i<size;i++) {
        Sample s = getSample(i);
        removeSample(i);
        if (i > 0 && i % RESET_THRESHOLD == 0) {
            out.reset();
        }
        s.writeToStream(out);
        if ((i+40) % 50 == 0) step((STEPS*i)/size);
    }
    step(STEPS); // set progress at 100%
    out.close();
    closeProgress();
}
 

private static void processFile(File file)
{
    String outPath = String.format("%s.0-3-1.txt.gz", file);
    try (PDDocument doc = PDDocument.load(file);
        GZIPOutputStream gzip = new GZIPOutputStream(new FileOutputStream(outPath));
        Writer w = new BufferedWriter(new OutputStreamWriter(gzip, "UTF-8"));
    ) {
        for (int i = 0; i < doc.getNumberOfPages(); i++) {
            PDFExtractor ext = new PDFExtractor(doc.getPage(i), i + 1, w);
            ext.processPage(doc.getPage(i));
            ext.write();
        }
    }
    catch (Exception e) {
        System.out.println(e.getMessage());
    }
}
 

@Override
  protected byte[] perform(byte[] input) throws Exception {
  	ByteArrayOutputStream out = new ByteArrayOutputStream();
  	GZIPOutputStream gzos = new GZIPOutputStream(out);     
  	ByteArrayInputStream in = new ByteArrayInputStream(input);
      
      byte[] buffer = new byte[1024];   
      int len;
      while ((len = in.read(buffer)) > 0) {
      	gzos.write(buffer, 0, len);
      }
      
in.close();
gzos.close();
out.close();
      return out.toByteArray();
  }
 

public static int  writeCompressedByteArray(DataOutput out, 
                                            byte[] bytes) throws IOException {
  if (bytes != null) {
    ByteArrayOutputStream bos =  new ByteArrayOutputStream();
    GZIPOutputStream gzout = new GZIPOutputStream(bos);
    try {
      gzout.write(bytes, 0, bytes.length);
      gzout.close();
      gzout = null;
    } finally {
      IOUtils.closeStream(gzout);
    }
    byte[] buffer = bos.toByteArray();
    int len = buffer.length;
    out.writeInt(len);
    out.write(buffer, 0, len);
    /* debug only! Once we have confidence, can lose this. */
    return ((bytes.length != 0) ? (100*buffer.length)/bytes.length : 0);
  } else {
    out.writeInt(-1);
    return -1;
  }
}
 

private BurstEncryptedMessage encryptPlainMessage(byte[] message, boolean isText, byte[] myPrivateKey, byte[] theirPublicKey) {
    if (message.length == 0) {
        return new BurstEncryptedMessage(new byte[0], new byte[0], isText);
    }
    try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
        GZIPOutputStream gzip = new GZIPOutputStream(bos);
        gzip.write(message);
        gzip.flush();
        gzip.close();
        byte[] compressedPlaintext = bos.toByteArray();
        byte[] nonce = new byte[32];
        secureRandom.nextBytes(nonce);
        byte[] data = aesSharedEncrypt(compressedPlaintext, myPrivateKey, theirPublicKey, nonce);
        return new BurstEncryptedMessage(data, nonce, isText);
    } catch (IOException e) {
        throw new RuntimeException(e.getMessage(), e);
    }
}
 

private static String c(String str) {
    String str2 = null;
    try {
        byte[] bytes = str.getBytes(z[5]);
        OutputStream byteArrayOutputStream = new ByteArrayOutputStream();
        OutputStream gZIPOutputStream = new GZIPOutputStream(byteArrayOutputStream);
        gZIPOutputStream.write(bytes);
        gZIPOutputStream.close();
        bytes = byteArrayOutputStream.toByteArray();
        byteArrayOutputStream.close();
        str2 = a.a(bytes);
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    } catch (IOException e2) {
        e2.printStackTrace();
    }
    return str2;
}
 

/**
 * Ensures that Content-Encoding headers are removed when JavaScript is uncompressed.
 * (was causing java.io.IOException: Not in GZIP format).
 * @throws Exception if the test fails
 */
@Test
public void gzip() throws Exception {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(baos)) {
        IOUtils.write("alert(1)", gzipOutputStream, UTF_8);
    }

    final MockWebConnection mockConnection = new MockWebConnection();
    final List<NameValuePair> responseHeaders = Arrays.asList(
        new NameValuePair("Content-Encoding", "gzip"));
    mockConnection.setResponse(URL_FIRST, baos.toByteArray(), 200, "OK", MimeType.APPLICATION_JAVASCRIPT,
        responseHeaders);

    final String dirName = "test-" + getClass().getSimpleName();
    try (DebuggingWebConnection dwc = new DebuggingWebConnection(mockConnection, dirName)) {

        final WebRequest request = new WebRequest(URL_FIRST);
        final WebResponse response = dwc.getResponse(request); // was throwing here
        assertNull(response.getResponseHeaderValue("Content-Encoding"));

        FileUtils.deleteDirectory(dwc.getReportFolder());
    }
}
 

@Before
public void setUp() {
  if (useGzipInflatingBuffer) {
    deframer.setFullStreamDecompressor(new GzipInflatingBuffer() {
      @Override
      public void addGzippedBytes(ReadableBuffer buffer) {
        try {
          ByteArrayOutputStream gzippedOutputStream = new ByteArrayOutputStream();
          OutputStream gzipCompressingStream = new GZIPOutputStream(
                  gzippedOutputStream);
          buffer.readBytes(gzipCompressingStream, buffer.readableBytes());
          gzipCompressingStream.close();
          super.addGzippedBytes(ReadableBuffers.wrap(gzippedOutputStream.toByteArray()));
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }
    });
  }
}
 

public static String compressOption(final String data, final Charset characterSet) throws IOException {
    final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
    final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream);
    final DataOutputStream dataOut = new DataOutputStream(gzipStream);
    
    byte[] arr = data.getBytes(characterSet);
    final int length = arr.length;
    
    dataOut.writeInt(length);
    dataOut.write(arr);
    
    dataOut.close();
    byteStream.close();
    
    return new String(Base64.encodeBase64(byteStream.toByteArray()));
}
 

static public void gzipRandom(String filenameOut, boolean buffer) throws IOException {
  FileOutputStream fout = new FileOutputStream(filenameOut);
  OutputStream out = new GZIPOutputStream(fout);
  if (buffer)
    out = new BufferedOutputStream(out, 1000); // 3X performance by having this !!
  DataOutputStream dout = new DataOutputStream(out);

  Random r = new Random();
  long start = System.currentTimeMillis();
  for (int i = 0; i < (1000 * 1000); i++) {
    dout.writeFloat(r.nextFloat());
  }
  fout.flush();
  fout.close();
  double took = .001 * (System.currentTimeMillis() - start);
  File f = new File(filenameOut);
  long len = f.length();
  double ratio = 4 * 1000.0 * 1000.0 / len;
  System.out.println(" gzipRandom took = " + took + " sec; compress = " + ratio);
}
 

private void prepareFile() throws HopFileException {
  try {
    String filename = environmentSubstitute( meta.getFilename() );
    if ( meta.isAddToResultFiles() ) {
      // Add this to the result file names...
      ResultFile resultFile =
        new ResultFile(
          ResultFile.FILE_TYPE_GENERAL, HopVfs.getFileObject( filename ), getPipelineMeta()
          .getName(), getTransformName() );
      resultFile.setComment( "This file was created with a cube file output transform" );
      addResultFile( resultFile );
    }

    data.fos = HopVfs.getOutputStream( filename, false );
    data.zip = new GZIPOutputStream( data.fos );
    data.dos = new DataOutputStream( data.zip );
  } catch ( Exception e ) {
    throw new HopFileException( e );
  }
}
 

/**
 * {@inheritDoc}
 */
@Override
protected void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException {
    final byte[] bytes = RESPONSE.getBytes(UTF_8);
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final GZIPOutputStream gout = new GZIPOutputStream(bos);
    gout.write(bytes);
    gout.finish();

    final byte[] encoded = bos.toByteArray();

    response.setContentType(MimeType.TEXT_XML);
    response.setCharacterEncoding(UTF_8.name());
    response.setStatus(200);
    response.setContentLength(encoded.length);
    response.setHeader("Content-Encoding", "gzip");

    final OutputStream rout = response.getOutputStream();
    rout.write(encoded);
}
 

public static byte[] gzipData(String content) throws IOException {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    PrintWriter gzip = new PrintWriter(new GZIPOutputStream(bos));
    gzip.print(content);
    gzip.flush();
    gzip.close();
    return bos.toByteArray();
}
 

public static void internalWrite(
    TableMetadata metadata, OutputFile outputFile, boolean overwrite) {
  boolean isGzip = Codec.fromFileName(outputFile.location()) == Codec.GZIP;
  OutputStream stream = overwrite ? outputFile.createOrOverwrite() : outputFile.create();
  try (OutputStream ou = isGzip ? new GZIPOutputStream(stream) : stream;
       OutputStreamWriter writer = new OutputStreamWriter(ou, StandardCharsets.UTF_8)) {
    JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
    generator.useDefaultPrettyPrinter();
    toJson(metadata, generator);
    generator.flush();
  } catch (IOException e) {
    throw new RuntimeIOException(e, "Failed to write json to file: %s", outputFile);
  }
}
 

private String compressAndEncodeInput(InputStream is) throws IOException {
   ByteArrayOutputStream baos = new ByteArrayOutputStream();
   try (GZIPOutputStream gos = new GZIPOutputStream(baos)) {
      gos.write(IOUtils.toByteArray(is));
   } catch (Exception ex) {
      throw new IOException("could not compress data", ex);
   }

   return Base64.encodeBase64String(baos.toByteArray());
}
 

public static byte[] compress(byte[] input) throws IOException {
   long size = (long)input.length;
   ByteArrayOutputStream outstream = new ByteArrayOutputStream();
   GZIPOutputStream out = new GZIPOutputStream(outstream);
   out.write(input);
   out.flush();
   outstream.flush();
   out.close();
   outstream.close();
   byte[] ret = outstream.toByteArray();
   LOG.info("Compression of data from " + size + " bytes to " + ret.length + " bytes");
   return ret;
}
 

@Override public void aroundWriteTo(WriterInterceptorContext context) throws IOException, WebApplicationException {

    @SuppressWarnings("unchecked") List<String> acceptEncodings = (List<String>) context.getProperty(ACCEPT_ENCODING);

    logger.debug("write: " + acceptEncodings);

    // Only gzip the response if asked to do so in the request headers.
    if (acceptEncodings != null && (acceptEncodings.contains("gzip") || acceptEncodings.contains(APPLICATION_GZIP))) {
      context.setOutputStream(new GZIPOutputStream(context.getOutputStream()));
      context.getHeaders().add("Content-Encoding", "gzip");
    }

    context.proceed();
  }
 

protected InputStream createGZIPInputStream() throws IOException {
  // Create an in-memory GZIP output stream for use by the input stream (to avoid exceptions)
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  GZIPOutputStream gos = new GZIPOutputStream( baos );
  byte[] testBytes = "Test".getBytes();
  gos.write( testBytes );
  ByteArrayInputStream in = new ByteArrayInputStream( baos.toByteArray() );
  return in;
}
 

private byte[] compress(byte[] content) throws IOException {
   ByteArrayOutputStream baos = new ByteArrayOutputStream();
   try (GZIPOutputStream os = new GZIPOutputStream(baos)) {
      os.write(content);
   }

   return baos.toByteArray();
}
 

@Test
public void testGzipCodecRead() throws IOException {
  // Create a gzipped file and try to read it back, using a decompressor
  // from the CodecPool.

  // Don't use native libs for this test.
  Configuration conf = new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
  assertFalse("ZlibFactory is using native libs against request",
      ZlibFactory.isNativeZlibLoaded(conf));

  // Ensure that the CodecPool has a BuiltInZlibInflater in it.
  Decompressor zlibDecompressor = ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!", zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",
      zlibDecompressor instanceof BuiltInZlibInflater);
  CodecPool.returnDecompressor(zlibDecompressor);

  // Now create a GZip text file.
  String tmpDir = System.getProperty("test.build.data", "/tmp/");
  Path f = new Path(new Path(tmpDir), "testGzipCodecRead.txt.gz");
  BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
    new GZIPOutputStream(new FileOutputStream(f.toString()))));
  final String msg = "This is the message in the file!";
  bw.write(msg);
  bw.close();

  // Now read it back, using the CodecPool to establish the
  // decompressor to use.
  CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
  CompressionCodec codec = ccf.getCodec(f);
  Decompressor decompressor = CodecPool.getDecompressor(codec);
  FileSystem fs = FileSystem.getLocal(conf);
  InputStream is = fs.open(f);
  is = codec.createInputStream(is, decompressor);
  BufferedReader br = new BufferedReader(new InputStreamReader(is));
  String line = br.readLine();
  assertEquals("Didn't get the same message back!", msg, line);
  br.close();
}
 

public void write(OutputStream pStream) throws XmlRpcException, IOException, SAXException {
    try {
        GZIPOutputStream gStream = new GZIPOutputStream(pStream);
        reqWriter.write(gStream);
        pStream.close();
        pStream = null;
    } catch (IOException e) {
        throw new XmlRpcException("Failed to write request: " + e.getMessage(), e);
    } finally {
        if (pStream != null) { try { pStream.close(); } catch (Throwable ignore) {} }
    }
}
 

public <T> String compress(final String modelInput) throws IOException {
    final Map<String, String> map = new HashMap<>();
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        try (GZIPOutputStream gzip = new GZIPOutputStream(byteArrayOutputStream)) {
            gzip.write(modelInput.getBytes(StandardCharsets.UTF_8));
        }
        map.put(COMPRESSED, Base64.encodeBase64String(byteArrayOutputStream.toByteArray()));
        map.put(COMPRESSION_METHOD, COMPRESSION_GZIP_BASE64);
    }
    return OBJECT_MAPPER.writeValueAsString(map);
}
 

public static byte[] gzipData(String content) throws IOException {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    PrintWriter gzip = new PrintWriter(new GZIPOutputStream(bos));
    gzip.print(content);
    gzip.flush();
    gzip.close();
    return bos.toByteArray();
}
 

@Override
protected void cacheToDisk() throws IOException {
  File cacheFile = getCacheFile();
  DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(cacheFile))));
  out.writeInt(data_.size());
  for (TIntObjectIterator<EntityType> itr = data_.iterator(); itr.hasNext(); ) {
    itr.advance();
    out.writeInt(itr.key());
    out.writeInt(itr.value().getDBId());
  }
  out.flush();
  out.close();
}
 

@SuppressWarnings("resource")
public Backup(File file, OperationMode operationMode, EnumSet<BackupManager.Flag> flags) throws IOException {
    this.operationMode = operationMode;
    if (!file.exists()) {
        if (operationMode == OperationMode.READ) {
            throw new UnsupportedOperationException("The backup could not be read");
        } else {
            file.createNewFile();
        }
    }

    // Set some base data if we're writing
    if (operationMode == OperationMode.WRITE) {
        revision = CURRENT_REVISION;
        created = System.currentTimeMillis() / 1000;
    }

    // Are we using compression?
    boolean compression = flags.contains(BackupManager.Flag.COMPRESSION);

    // create the stream we need
    if (operationMode == OperationMode.READ) {
        FileInputStream fis = new FileInputStream(file);
        inputStream = new DataInputStream(compression ? new GZIPInputStream(fis) : fis);
    } else if (operationMode == OperationMode.WRITE) {
        FileOutputStream fos = new FileOutputStream(file);
        outputStream = new DataOutputStream(compression ? new GZIPOutputStream(fos) : fos);
    }
}
 

public static String encodeBinaryData( byte[] val ) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  GZIPOutputStream gzos = new GZIPOutputStream( baos );
  BufferedOutputStream bos = new BufferedOutputStream( gzos );
  bos.write( val );
  bos.flush();
  bos.close();

  return new String( Base64.encodeBase64( baos.toByteArray() ) );
}
 

/**
 * GZIP 压缩字符串
 * 
 * @param String str
 * @param String charsetName
 * @return byte[]
 * @throws IOException
 */
public static byte[] compressString2byte(String str, String charsetName) throws IOException {
    if (str == null || str.trim().length() == 0) {
        return null;
    }
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    GZIPOutputStream gzip = new GZIPOutputStream(out);
    gzip.write(str.getBytes(charsetName));
    gzip.close();
    return out.toByteArray();
}
 

/**
 * Gzips the given String.
 *
 * @param str The string to gzip.
 * @return The gzipped String.
 * @throws IOException If the compression failed.
 */
private static byte[] compress(final String str) throws IOException {
    if (str == null) {
        return null;
    }
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    try (GZIPOutputStream gzip = new GZIPOutputStream(outputStream)) {
        gzip.write(str.getBytes(StandardCharsets.UTF_8));
    }
    return outputStream.toByteArray();
}