// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.reporting.icann;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.request.Action.Method.POST;
import static google.registry.schema.cursor.CursorDao.loadAndCompareAll;
import static javax.servlet.http.HttpServletResponse.SC_OK;

import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteStreams;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.Cursor;
import google.registry.model.common.Cursor.CursorType;
import google.registry.model.registry.Registries;
import google.registry.model.registry.Registry;
import google.registry.model.registry.Registry.TldType;
import google.registry.request.Action;
import google.registry.request.HttpException.ServiceUnavailableException;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.request.lock.LockHandler;
import google.registry.schema.cursor.CursorDao;
import google.registry.util.Clock;
import google.registry.util.EmailMessage;
import google.registry.util.Retrier;
import google.registry.util.SendEmailService;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.mail.internet.InternetAddress;
import org.joda.time.DateTime;
import org.joda.time.Duration;

/**
 * Action that uploads the monthly activity/transactions reports from GCS to ICANN via an HTTP PUT.
 *
 * <p>This should be run after {@link IcannReportingStagingAction}, which writes out the month's
 * reports and a MANIFEST.txt file. This action checks each ICANN_UPLOAD_TX and
 * ICANN_UPLOAD_ACTIVITY cursor and uploads the corresponding report if the cursor time is before
 * now.
 *
 * <p>Parameters:
 *
 * <p>subdir: the subdirectory of gs://[project-id]-reporting/ to retrieve reports from. For
 * example: "manual/dir" means reports will be stored under gs://[project-id]-reporting/manual/dir.
 * Defaults to "icann/monthly/[last month in yyyy-MM format]".
 */
@Action(
    service = Action.Service.BACKEND,
    path = IcannReportingUploadAction.PATH,
    method = POST,
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
public final class IcannReportingUploadAction implements Runnable {

  static final String PATH = "/_dr/task/icannReportingUpload";

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

  @Inject
  @Config("reportingBucket")
  String reportingBucket;

  @Inject GcsUtils gcsUtils;
  @Inject IcannHttpReporter icannReporter;
  @Inject Retrier retrier;
  @Inject Response response;
  @Inject @Config("gSuiteOutgoingEmailAddress") InternetAddress sender;
  @Inject @Config("alertRecipientEmailAddress") InternetAddress recipient;
  @Inject SendEmailService emailService;
  @Inject Clock clock;
  @Inject LockHandler lockHandler;

  @Inject
  IcannReportingUploadAction() {}

  @Override
  public void run() {
    Callable<Void> lockRunner =
        () -> {
          ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();

          ImmutableMap<Cursor, String> cursors = loadCursors();

          // If cursor time is before now, upload the corresponding report
          cursors.entrySet().stream()
              .filter(entry -> getCursorTimeOrStartOfTime(entry.getKey()).isBefore(clock.nowUtc()))
              .forEach(
                  entry -> {
                    DateTime cursorTime = getCursorTimeOrStartOfTime(entry.getKey());
                    uploadReport(
                        cursorTime,
                        entry.getKey().getType(),
                        entry.getValue(),
                        reportSummaryBuilder);
                  });
          // Send email of which reports were uploaded
          emailUploadResults(reportSummaryBuilder.build());
          response.setStatus(SC_OK);
          response.setContentType(PLAIN_TEXT_UTF_8);
          return null;
        };

    String lockname = "IcannReportingUploadAction";
    if (!lockHandler.executeWithLocks(lockRunner, null, Duration.standardHours(2), lockname)) {
      throw new ServiceUnavailableException("Lock for IcannReportingUploadAction already in use");
    }
  }

  /** Uploads the report and rolls forward the cursor for that report. */
  private void uploadReport(
      DateTime cursorTime,
      CursorType cursorType,
      String tldStr,
      ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
    DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
    String reportSubdir =
        String.format(
            "icann/monthly/%d-%02d",
            cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear());
    String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir);
    String filename = getFileName(cursorType, cursorTime, tldStr);
    final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename);
    logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
    // Check that the report exists
    try {
      verifyFileExists(gcsFilename);
    } catch (IllegalArgumentException e) {
      String logMessage =
          String.format(
              "Could not upload %s report for %s because file %s did not exist.",
              cursorType, tldStr, filename);
      if (clock.nowUtc().dayOfMonth().get() == 1) {
        logger.atInfo().withCause(e).log(logMessage + " This report may not have been staged yet.");
      } else {
        logger.atSevere().withCause(e).log(logMessage);
      }
      reportSummaryBuilder.put(filename, false);
      return;
    }

    // Upload the report
    boolean success = false;
    try {
      success =
          retrier.callWithRetry(
              () -> {
                final byte[] payload = readBytesFromGcs(gcsFilename);
                return icannReporter.send(payload, filename);
              },
              IcannReportingUploadAction::isUploadFailureRetryable);
    } catch (RuntimeException e) {
      logger.atWarning().withCause(e).log("Upload to %s failed", gcsFilename);
    }
    reportSummaryBuilder.put(filename, success);

    // Set cursor to first day of next month if the upload succeeded
    if (success) {
      Cursor newCursor =
          Cursor.create(
              cursorType,
              cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1),
              Registry.get(tldStr));
      CursorDao.saveCursor(
          newCursor,
          Optional.ofNullable(tldStr).orElse(google.registry.schema.cursor.Cursor.GLOBAL));
    }
  }

  private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) {
    DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
    return String.format(
        "%s%s%d%02d.csv",
        tld,
        (cursorType.equals(CursorType.ICANN_UPLOAD_ACTIVITY) ? "-activity-" : "-transactions-"),
        cursorTimeMinusMonth.year().get(),
        cursorTimeMinusMonth.monthOfYear().get());
  }

  /** Returns a map of each cursor to the tld. */
  private ImmutableMap<Cursor, String> loadCursors() {

    ImmutableSet<Registry> registries = Registries.getTldEntitiesOfType(TldType.REAL);

    ImmutableMap<Key<Cursor>, Registry> activityKeyMap =
        loadKeyMap(registries, CursorType.ICANN_UPLOAD_ACTIVITY);
    ImmutableMap<Key<Cursor>, Registry> transactionKeyMap =
        loadKeyMap(registries, CursorType.ICANN_UPLOAD_TX);

    ImmutableSet.Builder<Key<Cursor>> keys = new ImmutableSet.Builder<>();
    keys.addAll(activityKeyMap.keySet());
    keys.addAll(transactionKeyMap.keySet());

    Map<Key<Cursor>, Cursor> cursorMap = ofy().load().keys(keys.build());
    ImmutableMap.Builder<Cursor, String> cursors = new ImmutableMap.Builder<>();
    cursors.putAll(
        defaultNullCursorsToNextMonthAndAddToMap(
            activityKeyMap, CursorType.ICANN_UPLOAD_ACTIVITY, cursorMap));
    cursors.putAll(
        defaultNullCursorsToNextMonthAndAddToMap(
            transactionKeyMap, CursorType.ICANN_UPLOAD_TX, cursorMap));
    return cursors.build();
  }

  private ImmutableMap<Key<Cursor>, Registry> loadKeyMap(
      ImmutableSet<Registry> registries, CursorType type) {
    return Maps.uniqueIndex(registries, r -> Cursor.createKey(type, r));
  }

  /**
   * Return a map with the Cursor and scope for each key in the keyMap. If the key from the keyMap
   * does not have an existing cursor, create a new cursor with a default cursorTime of the first of
   * next month.
   */
  private ImmutableMap<Cursor, String> defaultNullCursorsToNextMonthAndAddToMap(
      Map<Key<Cursor>, Registry> keyMap, CursorType type, Map<Key<Cursor>, Cursor> cursorMap) {
    ImmutableMap.Builder<Cursor, String> cursors = new ImmutableMap.Builder<>();
    keyMap.forEach(
        (key, registry) -> {
          // Cursor time is defaulted to the first of next month since a new tld will not yet have a
          // report staged for upload.
          Cursor cursor =
              cursorMap.getOrDefault(
                  key,
                  Cursor.create(
                      type,
                      clock.nowUtc().withDayOfMonth(1).withTimeAtStartOfDay().plusMonths(1),
                      registry));
          if (!cursorMap.containsValue(cursor)) {
            tm().transact(() -> ofy().save().entity(cursor));
          }
          cursors.put(cursor, registry.getTldStr());
        });
    loadAndCompareAll(cursors.build(), type);
    return cursors.build();
  }

  /** Don't retry when reports are already uploaded or can't be uploaded. */
  private static final String ICANN_UPLOAD_PERMANENT_ERROR_MESSAGE =
      "A report for that month already exists, the cut-off date already passed";

  /** Don't retry when the IP address isn't allow-listed, as retries go through the same IP. */
  private static final Pattern ICANN_UPLOAD_ALLOW_LIST_ERROR =
      Pattern.compile("Your IP address .+ is not allowed to connect");

  /** Predicate to retry uploads on IOException, so long as they aren't non-retryable errors. */
  private static boolean isUploadFailureRetryable(Throwable e) {
    return (e instanceof IOException)
        && !e.getMessage().contains(ICANN_UPLOAD_PERMANENT_ERROR_MESSAGE)
        && !ICANN_UPLOAD_ALLOW_LIST_ERROR.matcher(e.getMessage()).matches();
  }

  private void emailUploadResults(ImmutableMap<String, Boolean> reportSummary) {
    String subject = String.format(
        "ICANN Monthly report upload summary: %d/%d succeeded",
        reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size());
    String body =
        String.format(
            "Report Filename - Upload status:\n%s",
            reportSummary.entrySet().stream()
                .map(
                    (e) ->
                        String.format("%s - %s", e.getKey(), e.getValue() ? "SUCCESS" : "FAILURE"))
                .collect(Collectors.joining("\n")));
    emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender));
  }

  private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
    try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
      return ByteStreams.toByteArray(gcsInput);
    }
  }

  private void verifyFileExists(GcsFilename gcsFilename) {
    checkArgument(
        gcsUtils.existsAndNotEmpty(gcsFilename),
        "Object %s in bucket %s not found",
        gcsFilename.getObjectName(),
        gcsFilename.getBucketName());
  }

}