Java Code Examples for org.apache.hadoop.io.IOUtils#copyBytes()

The following examples show how to use org.apache.hadoop.io.IOUtils#copyBytes() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: recsys-offline   File: HdfsHelper.java    License: Apache License 2.0 6 votes vote down vote up
public static String read(String path) {
	String content = null;
	Path dst = new Path(path);
	try {
		FSDataInputStream in = hdfs.open(dst);
		OutputStream os = new ByteArrayOutputStream();
		IOUtils.copyBytes(in, os, 4096, true);
		content = os.toString();
		os.close();
		in.close();
		log.info("read hdfs content from " + path + " successed. ");
	} catch (Exception e) {
		log.error("read hdfs content from " + path + " failed. ", e);
	}
	return content;
}
 
Example 2
Source Project: big-c   File: CopyCommands.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void processArguments(LinkedList<PathData> items)
throws IOException {
  super.processArguments(items);
  if (exitCode != 0) { // check for error collecting paths
    return;
  }
  FSDataOutputStream out = dst.fs.create(dst.path);
  try {
    for (PathData src : srcs) {
      FSDataInputStream in = src.fs.open(src.path);
      try {
        IOUtils.copyBytes(in, out, getConf(), false);
        if (delimiter != null) {
          out.write(delimiter.getBytes("UTF-8"));
        }
      } finally {
        in.close();
      }
    }
  } finally {
    out.close();
  }      
}
 
Example 3
Source Project: big-c   File: FileUtil.java    License: Apache License 2.0 6 votes vote down vote up
/** Copy FileSystem files to local files. */
private static boolean copy(FileSystem srcFS, FileStatus srcStatus,
                            File dst, boolean deleteSource,
                            Configuration conf) throws IOException {
  Path src = srcStatus.getPath();
  if (srcStatus.isDirectory()) {
    if (!dst.mkdirs()) {
      return false;
    }
    FileStatus contents[] = srcFS.listStatus(src);
    for (int i = 0; i < contents.length; i++) {
      copy(srcFS, contents[i],
           new File(dst, contents[i].getPath().getName()),
           deleteSource, conf);
    }
  } else {
    InputStream in = srcFS.open(src);
    IOUtils.copyBytes(in, new FileOutputStream(dst), conf);
  }
  if (deleteSource) {
    return srcFS.delete(src, true);
  } else {
    return true;
  }
}
 
Example 4
Source Project: kite   File: TestDatasetDescriptor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSchemaFromHdfs() throws IOException {
  MiniDFSTest.setupFS();
  FileSystem fs = MiniDFSTest.getDFS();

  // copy a schema to HDFS
  Path schemaPath = fs.makeQualified(new Path("schema.avsc"));
  FSDataOutputStream out = fs.create(schemaPath);
  IOUtils.copyBytes(DatasetTestUtilities.USER_SCHEMA_URL.toURL().openStream(),
      out, fs.getConf());
  out.close();

  // build a schema using the HDFS path and check it's the same
  Schema schema = new DatasetDescriptor.Builder().schemaUri(schemaPath.toUri()).build()
      .getSchema();

  Assert.assertEquals(DatasetTestUtilities.USER_SCHEMA, schema);
  MiniDFSTest.teardownFS();
}
 
Example 5
Source Project: incubator-gobblin   File: HadoopUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copy a file from a srcFs {@link FileSystem} to a dstFs {@link FileSystem}. The src {@link Path} must be a file,
 * that is {@link FileSystem#isFile(Path)} must return true for src.
 *
 * <p>
 *   If overwrite is specified to true, this method may delete the dst directory even if the copy from src to dst fails.
 * </p>
 *
 * @param srcFs the src {@link FileSystem} to copy the file from
 * @param src the src {@link Path} to copy
 * @param dstFs the destination {@link FileSystem} to write to
 * @param dst the destination {@link Path} to write to
 * @param overwrite true if the dst {@link Path} should be overwritten, false otherwise
 */
public static void copyFile(FileSystem srcFs, Path src, FileSystem dstFs, Path dst, boolean overwrite,
    Configuration conf) throws IOException {

  Preconditions.checkArgument(srcFs.isFile(src),
      String.format("Cannot copy from %s to %s because src is not a file", src, dst));
  Preconditions.checkArgument(overwrite || !dstFs.exists(dst),
      String.format("Cannot copy from %s to %s because dst exists", src, dst));

  try (InputStream in = srcFs.open(src); OutputStream out = dstFs.create(dst, overwrite)) {
    IOUtils.copyBytes(in, out, conf, false);
  } catch (Throwable t1) {
    try {
      deleteIfExists(dstFs, dst, true);
    } catch (Throwable t2) {
      // Do nothing
    }
    throw t1;
  }
}
 
Example 6
Source Project: tez   File: HttpConnection.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Cleanup the error stream if any, for keepAlive connections
 *
 * @param errorStream
 */
private void readErrorStream(InputStream errorStream) {
  if (errorStream == null) {
    return;
  }
  try {
    DataOutputBuffer errorBuffer = new DataOutputBuffer();
    IOUtils.copyBytes(errorStream, errorBuffer, 4096);
    IOUtils.closeStream(errorBuffer);
    IOUtils.closeStream(errorStream);
  } catch (IOException ioe) {
    // ignore
  }
}
 
Example 7
Source Project: hadoop   File: FileUtil.java    License: Apache License 2.0 5 votes vote down vote up
/** Copy files between FileSystems. */
public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
                           FileSystem dstFS, Path dst,
                           boolean deleteSource,
                           boolean overwrite,
                           Configuration conf) throws IOException {
  Path src = srcStatus.getPath();
  dst = checkDest(src.getName(), dstFS, dst, overwrite);
  if (srcStatus.isDirectory()) {
    checkDependencies(srcFS, src, dstFS, dst);
    if (!dstFS.mkdirs(dst)) {
      return false;
    }
    FileStatus contents[] = srcFS.listStatus(src);
    for (int i = 0; i < contents.length; i++) {
      copy(srcFS, contents[i], dstFS,
           new Path(dst, contents[i].getPath().getName()),
           deleteSource, overwrite, conf);
    }
  } else {
    InputStream in=null;
    OutputStream out = null;
    try {
      in = srcFS.open(src);
      out = dstFS.create(dst, overwrite);
      IOUtils.copyBytes(in, out, conf, true);
    } catch (IOException e) {
      IOUtils.closeStream(out);
      IOUtils.closeStream(in);
      throw e;
    }
  }
  if (deleteSource) {
    return srcFS.delete(src, true);
  } else {
    return true;
  }

}
 
Example 8
Source Project: lucene-solr   File: RawLocalFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void concat(final Path trg, final Path [] psrcs) throws IOException {
  final int bufferSize = 4096;
  try(FSDataOutputStream out = create(trg)) {
    for (Path src : psrcs) {
      try(FSDataInputStream in = open(src)) {
        IOUtils.copyBytes(in, out, bufferSize, false);
      }
    }
  }
}
 
Example 9
Source Project: big-c   File: DFSTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return URL contents as a byte array
 */
public static byte[] urlGetBytes(URL url) throws IOException {
  URLConnection conn = url.openConnection();
  HttpURLConnection hc = (HttpURLConnection)conn;
  
  assertEquals(HttpURLConnection.HTTP_OK, hc.getResponseCode());
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
  return out.toByteArray();
}
 
Example 10
Source Project: hadoop   File: TestCodec.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGzipCompatibility() throws IOException {
  Random r = new Random();
  long seed = r.nextLong();
  r.setSeed(seed);
  LOG.info("seed: " + seed);

  DataOutputBuffer dflbuf = new DataOutputBuffer();
  GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
  byte[] b = new byte[r.nextInt(128 * 1024 + 1)];
  r.nextBytes(b);
  gzout.write(b);
  gzout.close();

  DataInputBuffer gzbuf = new DataInputBuffer();
  gzbuf.reset(dflbuf.getData(), dflbuf.getLength());

  Configuration conf = new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, false);
  CompressionCodec codec = ReflectionUtils.newInstance(GzipCodec.class, conf);
  Decompressor decom = codec.createDecompressor();
  assertNotNull(decom);
  assertEquals(BuiltInGzipDecompressor.class, decom.getClass());
  InputStream gzin = codec.createInputStream(gzbuf, decom);

  dflbuf.reset();
  IOUtils.copyBytes(gzin, dflbuf, 4096);
  final byte[] dflchk = Arrays.copyOf(dflbuf.getData(), dflbuf.getLength());
  assertArrayEquals(b, dflchk);
}
 
Example 11
Source Project: hadoop   File: InputStreamEntity.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(OutputStream os) throws IOException {
  IOUtils.skipFully(is, offset);
  if (len == -1) {
    IOUtils.copyBytes(is, os, 4096, true);
  } else {
    IOUtils.copyBytes(is, os, len, true);
  }
}
 
Example 12
Source Project: hadoop   File: CommandWithDestination.java    License: Apache License 2.0 5 votes vote down vote up
void writeStreamToFile(InputStream in, PathData target,
                       boolean lazyPersist) throws IOException {
  FSDataOutputStream out = null;
  try {
    out = create(target, lazyPersist);
    IOUtils.copyBytes(in, out, getConf(), true);
  } finally {
    IOUtils.closeStream(out); // just in case copyBytes didn't
  }
}
 
Example 13
Source Project: RDFS   File: FsShell.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compress a file.
 */
private int compress(String argv[], Configuration conf) throws IOException {
  int i = 0;
  String cmd = argv[i++];
  String srcf = argv[i++];
  String dstf = argv[i++];

  Path srcPath = new Path(srcf);
  FileSystem srcFs = srcPath.getFileSystem(getConf());
  Path dstPath = new Path(dstf);
  FileSystem dstFs = dstPath.getFileSystem(getConf());

  // Create codec
  CompressionCodecFactory factory = new CompressionCodecFactory(conf);
  CompressionCodec codec = factory.getCodec(dstPath);
  if (codec == null) {
    System.err.println(cmd.substring(1) + ": cannot find compression codec for "
        + dstf);
    return 1;
  }

  // open input stream
  InputStream in = srcFs.open(srcPath);

  // Create compression stream
  OutputStream out = dstFs.create(dstPath);
  out = codec.createOutputStream(out);

  IOUtils.copyBytes(in, out, conf, true);

  return 0;
}
 
Example 14
Source Project: big-c   File: RunJar.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Unpack matching files from a jar. Entries inside the jar that do
 * not match the given pattern will be skipped.
 *
 * @param jarFile the .jar file to unpack
 * @param toDir the destination directory into which to unpack the jar
 * @param unpackRegex the pattern to match jar entries against
 */
public static void unJar(File jarFile, File toDir, Pattern unpackRegex)
  throws IOException {
  JarFile jar = new JarFile(jarFile);
  try {
    Enumeration<JarEntry> entries = jar.entries();
    while (entries.hasMoreElements()) {
      final JarEntry entry = entries.nextElement();
      if (!entry.isDirectory() &&
          unpackRegex.matcher(entry.getName()).matches()) {
        InputStream in = jar.getInputStream(entry);
        try {
          File file = new File(toDir, entry.getName());
          ensureDirectory(file.getParentFile());
          OutputStream out = new FileOutputStream(file);
          try {
            IOUtils.copyBytes(in, out, 8192);
          } finally {
            out.close();
          }
        } finally {
          in.close();
        }
      }
    }
  } finally {
    jar.close();
  }
}
 
Example 15
Source Project: big-c   File: CommandWithDestination.java    License: Apache License 2.0 5 votes vote down vote up
void writeStreamToFile(InputStream in, PathData target,
                       boolean lazyPersist) throws IOException {
  FSDataOutputStream out = null;
  try {
    out = create(target, lazyPersist);
    IOUtils.copyBytes(in, out, getConf(), true);
  } finally {
    IOUtils.closeStream(out); // just in case copyBytes didn't
  }
}
 
Example 16
Source Project: hadoop-ozone   File: CopyKeyHandler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void execute(OzoneClient client, OzoneAddress address)
    throws IOException, OzoneClientException {

  String volumeName = address.getVolumeName();
  String bucketName = address.getBucketName();

  OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
  OzoneBucket bucket = vol.getBucket(bucketName);

  if (replicationFactor == null) {
    replicationFactor = ReplicationFactor.valueOf(
        getConf().getInt(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT));
  }

  if (replicationType == null) {
    replicationType = ReplicationType.valueOf(
        getConf().get(OZONE_REPLICATION_TYPE,
            OZONE_REPLICATION_TYPE_DEFAULT));
  }

  OzoneKeyDetails keyDetail = bucket.getKey(fromKey);
  Map<String, String> keyMetadata = new HashMap<>(keyDetail.getMetadata());
  keyMetadata.remove(OzoneConsts.GDPR_SECRET);
  keyMetadata.remove(OzoneConsts.GDPR_ALGORITHM);

  String gdprEnabled = bucket.getMetadata().get(OzoneConsts.GDPR_FLAG);
  if (Boolean.parseBoolean(gdprEnabled)) {
    keyMetadata.put(OzoneConsts.GDPR_FLAG, Boolean.TRUE.toString());
  }

  int chunkSize = (int) getConf().getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
      OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
  try (InputStream input = bucket.readKey(fromKey);
       OutputStream output = bucket.createKey(toKey, input.available(),
           replicationType, replicationFactor, keyMetadata)) {
    IOUtils.copyBytes(input, output, chunkSize);
  }

  if (isVerbose()) {
    out().printf("Copy Key : %s to %s%n", fromKey, toKey);
  }
}
 
Example 17
Source Project: hadoop   File: WebAppProxyServlet.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Download link and have it be the response.
 * @param req the http request
 * @param resp the http response
 * @param link the link to download
 * @param c the cookie to set if any
 * @throws IOException on any error.
 */
private static void proxyLink(HttpServletRequest req, 
    HttpServletResponse resp, URI link, Cookie c, String proxyHost)
    throws IOException {
  DefaultHttpClient client = new DefaultHttpClient();
  client
      .getParams()
      .setParameter(ClientPNames.COOKIE_POLICY,
          CookiePolicy.BROWSER_COMPATIBILITY)
      .setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);
  // Make sure we send the request from the proxy address in the config
  // since that is what the AM filter checks against. IP aliasing or
  // similar could cause issues otherwise.
  InetAddress localAddress = InetAddress.getByName(proxyHost);
  if (LOG.isDebugEnabled()) {
    LOG.debug("local InetAddress for proxy host: {}", localAddress);
  }
  client.getParams()
      .setParameter(ConnRoutePNames.LOCAL_ADDRESS, localAddress);
  HttpGet httpGet = new HttpGet(link);
  @SuppressWarnings("unchecked")
  Enumeration<String> names = req.getHeaderNames();
  while(names.hasMoreElements()) {
    String name = names.nextElement();
    if(passThroughHeaders.contains(name)) {
      String value = req.getHeader(name);
      if (LOG.isDebugEnabled()) {
        LOG.debug("REQ HEADER: {} : {}", name, value);
      }
      httpGet.setHeader(name, value);
    }
  }

  String user = req.getRemoteUser();
  if (user != null && !user.isEmpty()) {
    httpGet.setHeader("Cookie",
        PROXY_USER_COOKIE_NAME + "=" + URLEncoder.encode(user, "ASCII"));
  }
  OutputStream out = resp.getOutputStream();
  try {
    HttpResponse httpResp = client.execute(httpGet);
    resp.setStatus(httpResp.getStatusLine().getStatusCode());
    for (Header header : httpResp.getAllHeaders()) {
      resp.setHeader(header.getName(), header.getValue());
    }
    if (c != null) {
      resp.addCookie(c);
    }
    InputStream in = httpResp.getEntity().getContent();
    if (in != null) {
      IOUtils.copyBytes(in, out, 4096, true);
    }
  } finally {
    httpGet.releaseConnection();
  }
}
 
Example 18
public void testDatanodeBlockScanner() throws IOException {
  
  long startTime = System.currentTimeMillis();
  
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  
  FileSystem fs = cluster.getFileSystem();
  Path file1 = new Path("/tmp/testBlockVerification/file1");
  Path file2 = new Path("/tmp/testBlockVerification/file2");
  
  /*
   * Write the first file and restart the cluster.
   */
  DFSTestUtil.createFile(fs, file1, 10, (short)1, 0);
  cluster.shutdown();
  cluster = new MiniDFSCluster(conf, 1, false, null);
  cluster.waitActive();
  
  DFSClient dfsClient =  new DFSClient(new InetSocketAddress("localhost", 
                                       cluster.getNameNodePort()), conf);
  fs = cluster.getFileSystem();
  DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
  
  /*
   * The cluster restarted. The block should be verified by now.
   */
  assertTrue(waitForVerification(dn, fs, file1) > startTime);
  
  /*
   * Create a new file and read the block. The block should be marked 
   * verified since the client reads the block and verifies checksum. 
   */
  DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
  IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
                    conf, true); 
  assertTrue(waitForVerification(dn, fs, file2) > startTime);
  
  cluster.shutdown();
}
 
Example 19
Source Project: hadoop   File: TestFuseDFS.java    License: Apache License 2.0 4 votes vote down vote up
/** Create and write the given file */
private static void createFile(File f, String s) throws IOException {
  InputStream is = new ByteArrayInputStream(s.getBytes());
  FileOutputStream fos = new FileOutputStream(f);
  IOUtils.copyBytes(is, fos, s.length(), true);
}
 
Example 20
Source Project: RDFS   File: TestDatanodeBlockScanner.java    License: Apache License 2.0 4 votes vote down vote up
public void testDatanodeBlockScanner() throws IOException {
  
  long startTime = System.currentTimeMillis();
  
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
  cluster.waitActive();
  
  FileSystem fs = cluster.getFileSystem();
  Path file1 = new Path("/tmp/testBlockVerification/file1");
  Path file2 = new Path("/tmp/testBlockVerification/file2");
  
  /*
   * Write the first file and restart the cluster.
   */
  DFSTestUtil.createFile(fs, file1, 10, (short)1, 0);
  cluster.shutdown();
  cluster = new MiniDFSCluster(conf, 1, false, null);
  cluster.waitActive();
  
  DFSClient dfsClient =  new DFSClient(new InetSocketAddress("localhost", 
                                       cluster.getNameNodePort()), conf);
  fs = cluster.getFileSystem();
  DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
  
  /*
   * The cluster restarted. The block should be verified by now.
   */
  assertTrue(waitForVerification(dn, fs, file1) > startTime);
  
  /*
   * Create a new file and read the block. The block should be marked 
   * verified since the client reads the block and verifies checksum. 
   */
  DFSTestUtil.createFile(fs, file2, 10, (short)1, 0);
  IOUtils.copyBytes(fs.open(file2), new IOUtils.NullOutputStream(), 
                    conf, true); 
  assertTrue(waitForVerification(dn, fs, file2) > startTime);
  
  cluster.shutdown();
}