Java Code Examples for java.net.HttpURLConnection#getContentLengthLong()

/**
 * Checks to see if an update to the region daily dump, and if so, downloads the update.
 */
private void updateRegionsDump() {
	InputStream stream = null;
	try {
		HttpURLConnection conn = (HttpURLConnection)(new URL(REGIONS_URL)).openConnection();
		conn.setRequestProperty("User-Agent", userAgent);
		conn.connect();
		final long contentLength = conn.getContentLengthLong();
		final long time = conn.getHeaderFieldDate("Last-Modified", -1);
		final DateTime serverModified = new DateTime(time, DateTimeZone.forOffsetHours(0)); //set to UTC time
		Logger.info("Checking region dump for {}, length: {}, lastModified: {}", serverModified, contentLength, serverModified);
		
		File regionsDump = new File(regionsDir, serverModified.toString(FILE_DATE) + "-regions.xml.gz");
		latestRegionDump.set(regionsDump);
		
		stream = conn.getInputStream();
		if (!regionsDump.exists() || regionsDump.length() != contentLength) {
			Logger.info("Saving regions dump to " + regionsDump.getAbsolutePath());
			try (FileOutputStream fos = new FileOutputStream(regionsDump)) {
				IOUtils.copy(stream, fos);
				Logger.info("Saved regions dump, size: {}", regionsDump.length());
			}
		} else {
			Logger.debug("Regions dump is up to date");
		}
	} catch (IOException e) {
		Logger.error("Unable to process regions dump", e);
	} finally {
		IOUtils.closeQuietly(stream);
	}
}
 

/**
 * Checks to see if an update to the nation daily dump, and if so, downloads the update.
 * 
 * Will create a DumpUpdateTask thread after the update is downloaded.
 */
private void updateNationsDump() {
	InputStream stream = null;
	try {
		HttpURLConnection conn = (HttpURLConnection)(new URL(NATIONS_URL)).openConnection();
		conn.setRequestProperty("User-Agent", userAgent);
		conn.connect();
		final long contentLength = conn.getContentLengthLong();
		final long time = conn.getHeaderFieldDate("Last-Modified", -1);
		final DateTime serverModified = new DateTime(time, DateTimeZone.forOffsetHours(0)); //set to UTC
		
		Logger.info("Checking nations dump, length: {}, lastModified: {}", contentLength, serverModified);
		File nationsDump = new File(nationsDir, serverModified.toString(FILE_DATE) + "-nations.xml.gz");
		latestNationDump.set(nationsDump);
		
		stream = conn.getInputStream();
		if (!nationsDump.exists() || nationsDump.length() != contentLength) {
			Logger.info("Saving nations dump to " + nationsDump.getAbsolutePath());
			try (FileOutputStream fos = new FileOutputStream(nationsDump)) {
				IOUtils.copy(stream, fos);
				Logger.info("Saved nations dump successfully, size: {}", nationsDump.length());
			}
			(new Thread(new DumpUpdateTask(access, getMostRecentRegionDump(), nationsDump), "Daily Dump Update Thread")).start();
		} else {
			Logger.debug("Nations dump is up to date");
		}
	} catch (IOException e) {
		Logger.error("Unable to process nations dump", e);
	} finally {
		IOUtils.closeQuietly(stream);
	}
}
 

@VisibleForTesting
PhotoResult importSinglePhoto(
    UUID jobId,
    TokensAndUrlAuthData authData,
    PhotoModel inputPhoto,
    IdempotentImportExecutor idempotentImportExecutor)
    throws IOException, CopyExceptionWithFailureReason {
  /*
  TODO: resumable uploads https://developers.google.com/photos/library/guides/resumable-uploads
  Resumable uploads would allow the upload of larger media that don't fit in memory.  To do this,
  however, seems to require knowledge of the total file size.
  */
  // Upload photo
  InputStream inputStream;
  Long bytes;
  if (inputPhoto.isInTempStore()) {
    final InputStreamWrapper streamWrapper =
        jobStore.getStream(jobId, inputPhoto.getFetchableUrl());
    bytes = streamWrapper.getBytes();
    inputStream = streamWrapper.getStream();
  } else {
    HttpURLConnection conn = imageStreamProvider.getConnection(inputPhoto.getFetchableUrl());
    final long contentLengthLong = conn.getContentLengthLong();
    bytes = contentLengthLong != -1 ? contentLengthLong : 0;
    inputStream = conn.getInputStream();
  }

  String uploadToken =
      getOrCreatePhotosInterface(jobId, authData).uploadPhotoContent(inputStream);

  String description = getPhotoDescription(inputPhoto);
  NewMediaItem newMediaItem = new NewMediaItem(description, uploadToken);

  String albumId;
  if (Strings.isNullOrEmpty(inputPhoto.getAlbumId())) {
    // This is ok, since NewMediaItemUpload will ignore all null values and it's possible to
    // upload a NewMediaItem without a corresponding album id.
    albumId = null;
  } else {
    // Note this will throw if creating the album failed, which is what we want
    // because that will also mark this photo as being failed.
    albumId = idempotentImportExecutor.getCachedValue(inputPhoto.getAlbumId());
  }

  NewMediaItemUpload uploadItem =
      new NewMediaItemUpload(albumId, Collections.singletonList(newMediaItem));
  try {
    return new PhotoResult(
        getOrCreatePhotosInterface(jobId, authData)
            .createPhoto(uploadItem)
            .getResults()[0]
            .getMediaItem()
            .getId(),
        bytes);
  } catch (IOException e) {
    if (e.getMessage() != null
        && e.getMessage().contains("The remaining storage in the user's account is not enough")) {
      throw new DestinationMemoryFullException("Google destination storage full", e);
    } else {
      throw e;
    }
  }
}
 

private static long getContentLengthLongCompat(HttpURLConnection conn) {
  return Build.VERSION.SDK_INT < 24
      ? (long) conn.getContentLength()
      : conn.getContentLengthLong();
}
 

@Override
public boolean fetch(
    BuckEventBus eventBus, URI uri, Optional<PasswordAuthentication> authentication, Path output)
    throws IOException {
  if (!("https".equals(uri.getScheme()) || "http".equals(uri.getScheme()))) {
    return false;
  }

  Started started = DownloadEvent.started(uri);
  eventBus.post(started);

  try {
    HttpURLConnection connection = createConnection(uri);

    if (authentication.isPresent()) {
      if ("https".equals(uri.getScheme()) && connection instanceof HttpsURLConnection) {
        PasswordAuthentication p = authentication.get();
        String authStr = p.getUserName() + ":" + new String(p.getPassword());
        String authEncoded =
            BaseEncoding.base64().encode(authStr.getBytes(StandardCharsets.UTF_8));
        connection.addRequestProperty("Authorization", "Basic " + authEncoded);
      } else {
        LOG.info("Refusing to send basic authentication over plain http.");
        return false;
      }
    }

    if (HttpURLConnection.HTTP_OK != connection.getResponseCode()) {
      LOG.info(
          "Unable to download url: '%s', response code: %d, response message: '%s'",
          uri, connection.getResponseCode(), connection.getResponseMessage());
      return false;
    }
    long contentLength = connection.getContentLengthLong();
    try (InputStream is = new BufferedInputStream(connection.getInputStream());
        OutputStream os = new BufferedOutputStream(Files.newOutputStream(output))) {
      long read = 0;

      while (true) {
        int r = is.read();
        read++;
        if (r == -1) {
          break;
        }
        if (read % PROGRESS_REPORT_EVERY_N_BYTES == 0) {
          eventBus.post(new DownloadProgressEvent(uri, contentLength, read));
        }
        os.write(r);
      }
    }

    return true;
  } finally {
    eventBus.post(DownloadEvent.finished(started));
  }
}