package datawave.query.metrics;

import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;

import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;

import datawave.configuration.DatawaveEmbeddedProjectStageHolder;
import datawave.data.hash.UID;
import datawave.data.hash.UIDBuilder;
import datawave.ingest.data.RawRecordContainer;
import datawave.ingest.config.RawRecordContainerImpl;
import datawave.ingest.data.Type;
import datawave.ingest.data.TypeRegistry;
import datawave.ingest.data.config.NormalizedContentInterface;
import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler;
import datawave.ingest.mapreduce.job.BulkIngestKey;
import datawave.ingest.mapreduce.job.writer.LiveContextWriter;
import datawave.ingest.table.config.TableConfigHelper;
import datawave.query.iterator.QueryOptions;
import datawave.query.map.SimpleQueryGeometryHandler;
import datawave.security.authorization.DatawavePrincipal;
import datawave.security.system.CallerPrincipal;
import datawave.security.util.AuthorizationsUtil;
import datawave.webservice.common.connection.AccumuloConnectionFactory;
import datawave.webservice.common.connection.AccumuloConnectionFactory.Priority;
import datawave.webservice.common.logging.ThreadConfigurableLogger;
import datawave.webservice.common.logging.ThreadLocalLogLevel;
import datawave.webservice.query.Query;
import datawave.webservice.query.QueryImpl;
import datawave.webservice.query.QueryImpl.Parameter;
import datawave.webservice.query.cache.QueryMetricFactory;
import datawave.webservice.query.cache.ResultsPage;
import datawave.webservice.query.exception.QueryException;
import datawave.webservice.query.exception.QueryExceptionType;
import datawave.webservice.query.logic.QueryLogic;
import datawave.webservice.query.logic.QueryLogicFactory;
import datawave.webservice.query.metric.BaseQueryMetric;
import datawave.webservice.query.metric.BaseQueryMetric.PageMetric;
import datawave.webservice.query.metric.BaseQueryMetric.Lifecycle;
import datawave.webservice.query.metric.BaseQueryMetricListResponse;
import datawave.webservice.query.metric.QueryMetric;
import datawave.webservice.query.metric.QueryMetricListResponse;
import datawave.webservice.query.metric.QueryMetricsDetailListResponse;
import datawave.webservice.query.metric.QueryMetricsSummaryHtmlResponse;
import datawave.webservice.query.metric.QueryMetricsSummaryResponse;
import datawave.webservice.query.result.event.EventBase;
import datawave.webservice.query.result.event.FieldBase;
import datawave.webservice.query.runner.RunningQuery;
import datawave.webservice.query.util.QueryUtil;
import datawave.webservice.result.BaseQueryResponse;
import datawave.webservice.result.BaseResponse;
import datawave.webservice.result.EventQueryResponseBase;

import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections4.map.LRUMap;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.deltaspike.core.api.config.ConfigProperty;
import org.apache.deltaspike.core.api.exclude.Exclude;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MapContext;
import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.MapContextImpl;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;

@ApplicationScoped
@Exclude(ifProjectStage = DatawaveEmbeddedProjectStageHolder.DatawaveEmbedded.class)
@SuppressWarnings("unused")
public class ShardTableQueryMetricHandler extends BaseQueryMetricHandler<QueryMetric> {
    private static final Logger log = ThreadConfigurableLogger.getLogger(ShardTableQueryMetricHandler.class);
    
    private static final String QUERY_METRICS_LOGIC_NAME = "QueryMetricsQuery";
    protected static final String DEFAULT_SECURITY_MARKING = "PUBLIC";
    
    @Inject
    private AccumuloConnectionFactory connectionFactory;
    
    @Inject
    private QueryLogicFactory queryLogicFactory;
    
    @Inject
    @CallerPrincipal
    private DatawavePrincipal callerPrincipal;
    
    @Inject
    @ConfigProperty(name = "dw.query.metrics.marking")
    protected String markingString;
    
    @Inject
    @ConfigProperty(name = "dw.query.metrics.visibility")
    protected String visibilityString;
    
    @Inject
    private QueryMetricFactory metricFactory;
    
    private Collection<String> connectorAuthorizationCollection = null;
    private String connectorAuthorizations = null;
    
    @SuppressWarnings("FieldCanBeLocal")
    private final String JOB_ID = "job_201109071404_1";
    @SuppressWarnings("FieldCanBeLocal")
    private static final String NULL_BYTE = "\0";
    public static final String CONTEXT_WRITER_MAX_CACHE_SIZE = "context.writer.max.cache.size";
    
    // static to share the cache across instances of this class held by QueryExecutorBean, CachedResultsBean, QueryMetricsEnrichmentInterceptor, etc
    @SuppressWarnings("unchecked")
    private static Map metricsCache = Collections.synchronizedMap(new LRUMap(5000));
    
    private final Configuration conf = new Configuration();
    private final StatusReporter reporter = new MockStatusReporter();
    private final AtomicBoolean tablesChecked = new AtomicBoolean(false);
    private AccumuloRecordWriter recordWriter = null;
    
    private UIDBuilder<UID> uidBuilder = UID.builder();
    
    public ShardTableQueryMetricHandler() {
        URL queryMetricsUrl = Thread.currentThread().getContextClassLoader().getResource("datawave/query/QueryMetrics.xml");
        Preconditions.checkNotNull(queryMetricsUrl);
        conf.addResource(queryMetricsUrl);
        
        // encode the password because that's how the AccumuloRecordWriter
        String accumuloPassword = conf.get("AccumuloRecordWriter.password");
        byte[] encodedAccumuloPassword = Base64.encodeBase64(accumuloPassword.getBytes());
        conf.set("AccumuloRecordWriter.password", new String(encodedAccumuloPassword));
    }
    
    @PostConstruct
    private void initialize() {
        Connector connector = null;
        
        try {
            connector = connectionFactory.getConnection(Priority.ADMIN, new HashMap<>());
            connectorAuthorizations = connector.securityOperations().getUserAuthorizations(connector.whoami()).toString();
            connectorAuthorizationCollection = Lists.newArrayList(StringUtils.split(connectorAuthorizations, ","));
            reload();
            
            if (tablesChecked.compareAndSet(false, true))
                verifyTables();
        } catch (Exception e) {
            log.error("Error setting connection factory", e);
        } finally {
            if (connector != null) {
                try {
                    connectionFactory.returnConnection(connector);
                } catch (Exception e) {
                    log.error("Error returning connection to connection factory", e);
                }
            }
        }
    }
    
    @Override
    protected void finalize() throws Throwable {
        super.finalize();
        this.recordWriter.close(null);
    }
    
    @Override
    public void flush() throws Exception {
        this.recordWriter.flush();
    }
    
    private void verifyTables() {
        Connector connector = null;
        
        try {
            connector = this.connectionFactory.getConnection(Priority.ADMIN, new HashMap<>());
            AbstractColumnBasedHandler<Key> handler = new ContentQueryMetricsHandler<>();
            createAndConfigureTablesIfNecessary(handler.getTableNames(conf), connector.tableOperations(), conf);
        } catch (Exception e) {
            log.error("Error verifying table configuration", e);
        } finally {
            if (null != connector) {
                try {
                    this.connectionFactory.returnConnection(connector);
                } catch (Exception e) {
                    log.error("Error returning connection to connection factory");
                }
            }
        }
    }
    
    private void writeMetrics(QueryMetric updatedQueryMetric, List<QueryMetric> storedQueryMetrics, Date lastUpdated, boolean delete) throws Exception {
        LiveContextWriter contextWriter = null;
        
        MapContext<Text,RawRecordContainer,Text,Mutation> context = null;
        
        try {
            contextWriter = new LiveContextWriter();
            contextWriter.setup(conf, false);
            
            TaskAttemptID taskId = new TaskAttemptID(new TaskID(new JobID(JOB_ID, 1), TaskType.MAP, 1), 1);
            context = new MapContextImpl<>(conf, taskId, null, recordWriter, null, reporter, null);
            
            for (QueryMetric storedQueryMetric : storedQueryMetrics) {
                AbstractColumnBasedHandler<Key> handler = new ContentQueryMetricsHandler<>();
                handler.setup(context);
                
                Multimap<BulkIngestKey,Value> r = getEntries(handler, updatedQueryMetric, storedQueryMetric, lastUpdated, delete);
                
                try {
                    if (r != null) {
                        contextWriter.write(r, context);
                    }
                    
                    if (handler.getMetadata() != null) {
                        contextWriter.write(handler.getMetadata().getBulkMetadata(), context);
                    }
                } finally {
                    contextWriter.commit(context);
                }
            }
        } finally {
            if (contextWriter != null && context != null) {
                contextWriter.cleanup(context);
            }
        }
    }
    
    public Map<String,String> getEventFields(BaseQueryMetric queryMetric) {
        // ignore duplicates as none are expected
        Map<String,String> eventFields = new HashMap<>();
        ContentQueryMetricsIngestHelper ingestHelper = new ContentQueryMetricsIngestHelper(false);
        ingestHelper.setup(conf);
        Multimap<String,NormalizedContentInterface> fieldsToWrite = ingestHelper.getEventFieldsToWrite(queryMetric);
        for (Entry<String,NormalizedContentInterface> entry : fieldsToWrite.entries()) {
            eventFields.put(entry.getKey(), entry.getValue().getEventFieldValue());
        }
        return eventFields;
    }
    
    private Multimap<BulkIngestKey,Value> getEntries(AbstractColumnBasedHandler<Key> handler, QueryMetric updatedQueryMetric, QueryMetric storedQueryMetric,
                    Date lastUpdated, boolean delete) {
        Type type = TypeRegistry.getType("querymetrics");
        ContentQueryMetricsIngestHelper ingestHelper = new ContentQueryMetricsIngestHelper(delete);
        
        ingestHelper.setup(conf);
        
        RawRecordContainerImpl event = new RawRecordContainerImpl();
        event.setConf(this.conf);
        event.setDataType(type);
        event.setDate(storedQueryMetric.getCreateDate().getTime());
        // get security marking set in the config, otherwise default to PUBLIC
        if (visibilityString != null) {
            event.setVisibility(new ColumnVisibility(visibilityString));
        } else {
            event.setVisibility(new ColumnVisibility(DEFAULT_SECURITY_MARKING));
        }
        event.setAuxData(storedQueryMetric);
        event.setRawRecordNumber(1000L);
        event.addAltId(storedQueryMetric.getQueryId());
        
        event.setId(uidBuilder.newId(storedQueryMetric.getQueryId().getBytes(), (Date) null));
        
        final Multimap<String,NormalizedContentInterface> fields;
        
        if (delete) {
            fields = ingestHelper.getEventFieldsToDelete(updatedQueryMetric, storedQueryMetric);
        } else {
            fields = ingestHelper.getEventFieldsToWrite(updatedQueryMetric);
        }
        
        Key key = new Key();
        
        if (handler.getMetadata() != null) {
            handler.getMetadata().addEventWithoutLoadDates(ingestHelper, event, fields);
        }
        
        String eventTable = handler.getShardTableName().toString();
        String indexTable = handler.getShardIndexTableName().toString();
        String reverseIndexTable = handler.getShardReverseIndexTableName().toString();
        int fieldSizeThreshold = ingestHelper.getFieldSizeThreshold();
        Multimap<BulkIngestKey,Value> r = handler.processBulk(key, event, fields, reporter);
        List<BulkIngestKey> keysToRemove = new ArrayList<>();
        Map<String,BulkIngestKey> tfFields = new HashMap<>();
        
        // if an event has more than two entries for a given field, only keep the longest
        for (Entry<BulkIngestKey,Collection<Value>> entry : r.asMap().entrySet()) {
            String table = entry.getKey().getTableName().toString();
            BulkIngestKey bulkIngestKey = entry.getKey();
            Key currentKey = bulkIngestKey.getKey();
            
            if (table.equals(indexTable) || table.equals(reverseIndexTable)) {
                String value = currentKey.getRow().toString();
                if (value.length() > fieldSizeThreshold) {
                    keysToRemove.add(bulkIngestKey);
                }
            }
        }
        
        // remove any keys from the index or reverseIndex where the value size exceeds the fieldSizeThreshold
        for (BulkIngestKey b : keysToRemove) {
            r.removeAll(b);
        }
        
        // replace the longest of the keys from fields that get parsed as content
        for (Entry<String,BulkIngestKey> l : tfFields.entrySet()) {
            r.put(l.getValue(), new Value(new byte[0]));
        }
        
        for (Entry<BulkIngestKey,Collection<Value>> entry : r.asMap().entrySet()) {
            if (delete) {
                entry.getKey().getKey().setTimestamp(lastUpdated.getTime());
            } else {
                // this will ensure that the QueryMetrics can be found within second precision in most cases
                entry.getKey().getKey().setTimestamp(storedQueryMetric.getCreateDate().getTime() + storedQueryMetric.getNumUpdates());
            }
            entry.getKey().getKey().setDeleted(delete);
        }
        
        return r;
    }
    
    @SuppressWarnings("unchecked")
    @Override
    public void updateMetric(QueryMetric updatedQueryMetric, DatawavePrincipal datawavePrincipal) throws Exception {
        Date lastUpdated = updatedQueryMetric.getLastUpdated();
        
        try {
            enableLogs(false);
            String sid = updatedQueryMetric.getUser();
            if (sid == null) {
                sid = datawavePrincipal.getShortName();
            }
            
            // find and remove previous entries
            BaseQueryMetricListResponse response = new QueryMetricListResponse();
            Date end = new Date();
            Date begin = DateUtils.setYears(end, 2000);
            
            // user's DatawavePrincipal must have the Administrator role to use the Metrics query logic
            QueryMetric cachedQueryMetric;
            QueryMetric newCachedQueryMetric;
            synchronized (ShardTableQueryMetricHandler.class) {
                cachedQueryMetric = (QueryMetric) metricsCache.get(updatedQueryMetric.getQueryId());
                // duplicate updatedQueryMetric because we're counting on the cache to be a snapshot of the QueryMetric
                // so that we can retrieve it next update call to create the delete Mutations for the values written to Accumulo
                Map<Long,PageMetric> storedPageMetricMap = new TreeMap<>();
                if (cachedQueryMetric != null) {
                    List<PageMetric> cachedPageMetrics = cachedQueryMetric.getPageTimes();
                    if (cachedPageMetrics != null) {
                        for (PageMetric p : cachedPageMetrics) {
                            storedPageMetricMap.put(p.getPageNumber(), p);
                        }
                    }
                }
                // combine all of the page metrics from the cached metric and the updated metric
                for (PageMetric p : updatedQueryMetric.getPageTimes()) {
                    storedPageMetricMap.put(p.getPageNumber(), p);
                }
                newCachedQueryMetric = (QueryMetric) updatedQueryMetric.duplicate();
                ArrayList<PageMetric> newPageMetrics = new ArrayList<>();
                newPageMetrics.addAll(storedPageMetricMap.values());
                newCachedQueryMetric.setPageTimes(newPageMetrics);
                metricsCache.put(updatedQueryMetric.getQueryId(), newCachedQueryMetric);
            }
            
            List<QueryMetric> queryMetrics = new ArrayList<>();
            
            if (cachedQueryMetric == null) {
                // if numPages > 0 or Lifecycle > DEFINED, then we should have a metric cached already
                // if we don't, then query for the current stored metric
                if (updatedQueryMetric.getNumPages() > 0 || updatedQueryMetric.getLifecycle().compareTo(Lifecycle.DEFINED) > 0) {
                    QueryImpl query = new QueryImpl();
                    query.setBeginDate(begin);
                    query.setEndDate(end);
                    query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
                    query.setQuery("QUERY_ID == '" + updatedQueryMetric.getQueryId() + "'");
                    query.setQueryName(QUERY_METRICS_LOGIC_NAME);
                    query.setColumnVisibility(visibilityString);
                    query.setQueryAuthorizations(connectorAuthorizations);
                    query.setUserDN(sid);
                    query.setExpirationDate(DateUtils.addDays(new Date(), 1));
                    query.setPagesize(1000);
                    query.setId(UUID.randomUUID());
                    query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
                    queryMetrics = getQueryMetrics(response, query, callerPrincipal);
                }
            } else {
                queryMetrics = Collections.singletonList(cachedQueryMetric);
            }
            
            if (!queryMetrics.isEmpty()) {
                writeMetrics(updatedQueryMetric, queryMetrics, lastUpdated, true);
            }
            
            long nextUpdateNumber = 0;
            
            for (BaseQueryMetric m : queryMetrics) {
                if ((m.getNumUpdates() + 1) > nextUpdateNumber) {
                    nextUpdateNumber = m.getNumUpdates() + 1;
                }
            }
            
            updatedQueryMetric.setNumUpdates(nextUpdateNumber);
            
            synchronized (ShardTableQueryMetricHandler.class) {
                newCachedQueryMetric.setNumUpdates(nextUpdateNumber);
                metricsCache.put(updatedQueryMetric.getQueryId(), newCachedQueryMetric);
            }
            
            // write new entry
            writeMetrics(updatedQueryMetric, Collections.singletonList(updatedQueryMetric), lastUpdated, false);
        } finally {
            enableLogs(true);
        }
    }
    
    private List<QueryMetric> getQueryMetrics(BaseResponse response, Query query, DatawavePrincipal datawavePrincipal) {
        List<QueryMetric> queryMetrics = new ArrayList<>();
        RunningQuery runningQuery = null;
        Connector connector = null;
        
        try {
            Map<String,String> trackingMap = this.connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
            connector = this.connectionFactory.getConnection(Priority.ADMIN, trackingMap);
            QueryLogic<?> queryLogic = queryLogicFactory.getQueryLogic(query.getQueryLogicName(), datawavePrincipal);
            if (queryLogic instanceof QueryMetricQueryLogic) {
                ((QueryMetricQueryLogic) queryLogic).setRolesSets(datawavePrincipal.getPrimaryUser().getRoles());
            }
            runningQuery = new RunningQuery(null, connector, Priority.ADMIN, queryLogic, query, query.getQueryAuthorizations(), datawavePrincipal,
                            metricFactory);
            
            boolean done = false;
            List<Object> objectList = new ArrayList<>();
            
            while (!done) {
                ResultsPage resultsPage = runningQuery.next();
                
                if (!resultsPage.getResults().isEmpty()) {
                    objectList.addAll(resultsPage.getResults());
                } else {
                    done = true;
                }
            }
            
            BaseQueryResponse queryResponse = queryLogic.getTransformer(query).createResponse(new ResultsPage(objectList));
            List<QueryExceptionType> exceptions = queryResponse.getExceptions();
            
            if (queryResponse.getExceptions() != null && !queryResponse.getExceptions().isEmpty()) {
                if (response != null) {
                    response.setExceptions(new LinkedList<>(exceptions));
                    response.setHasResults(false);
                }
            }
            
            if (!(queryResponse instanceof EventQueryResponseBase)) {
                if (response != null) {
                    response.addException(new QueryException("incompatible response")); // TODO: Should this be an IllegalStateException?
                    response.setHasResults(false);
                }
            }
            
            EventQueryResponseBase eventQueryResponse = (EventQueryResponseBase) queryResponse;
            List<EventBase> eventList = eventQueryResponse.getEvents();
            
            for (EventBase<?,?> event : eventList) {
                QueryMetric metric = toMetric(event);
                queryMetrics.add(metric);
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
            if (response != null) {
                response.addExceptions(new QueryException(e).getQueryExceptionsInStack());
            }
        } finally {
            if (null != this.connectionFactory) {
                if (null != runningQuery && null != connector) {
                    try {
                        runningQuery.closeConnection(this.connectionFactory);
                    } catch (Exception e) {
                        log.warn("Could not return connector to factory", e);
                    }
                } else if (null != connector) {
                    try {
                        this.connectionFactory.returnConnection(connector);
                    } catch (Exception e) {
                        log.warn("Could not return connector to factory", e);
                    }
                }
            }
        }
        
        return queryMetrics;
    }
    
    @Override
    public QueryMetricListResponse query(String user, String queryId, DatawavePrincipal datawavePrincipal) {
        QueryMetricsDetailListResponse response = new QueryMetricsDetailListResponse();
        
        try {
            enableLogs(false);
            
            Collection<? extends Collection<String>> authorizations = datawavePrincipal.getAuthorizations();
            Date end = new Date();
            Date begin = DateUtils.setYears(end, 2000);
            
            QueryImpl query = new QueryImpl();
            query.setBeginDate(begin);
            query.setEndDate(end);
            query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
            // QueryMetricQueryLogic now enforces that you must be a QueryMetricsAdministrator to query metrics that do not belong to you
            query.setQuery("QUERY_ID == '" + queryId + "'");
            query.setQueryName(QUERY_METRICS_LOGIC_NAME);
            query.setColumnVisibility(visibilityString);
            query.setQueryAuthorizations(AuthorizationsUtil.buildAuthorizationString(authorizations));
            query.setExpirationDate(DateUtils.addDays(new Date(), 1));
            query.setPagesize(1000);
            query.setUserDN(datawavePrincipal.getShortName());
            query.setOwner(datawavePrincipal.getShortName());
            query.setId(UUID.randomUUID());
            query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
            List<QueryMetric> queryMetrics = getQueryMetrics(response, query, datawavePrincipal);
            
            response.setResult(queryMetrics);
            
            response.setGeoQuery(queryMetrics.stream().anyMatch(SimpleQueryGeometryHandler::isGeoQuery));
        } finally {
            enableLogs(true);
        }
        
        return response;
    }
    
    @Override
    public QueryMetricsSummaryResponse getTotalQueriesSummaryCounts(Date begin, Date end, DatawavePrincipal datawavePrincipal) {
        QueryMetricsSummaryResponse response = new QueryMetricsSummaryResponse();
        
        try {
            enableLogs(false);
            // this method is open to any user
            datawavePrincipal = callerPrincipal;
            
            Collection<? extends Collection<String>> authorizations = datawavePrincipal.getAuthorizations();
            QueryImpl query = new QueryImpl();
            query.setBeginDate(begin);
            query.setEndDate(end);
            query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
            query.setQuery("USER > 'A' && USER < 'ZZZZZZZ'");
            query.setQueryName(QUERY_METRICS_LOGIC_NAME);
            query.setColumnVisibility(visibilityString);
            query.setQueryAuthorizations(AuthorizationsUtil.buildAuthorizationString(authorizations));
            query.setExpirationDate(DateUtils.addDays(new Date(), 1));
            query.setPagesize(1000);
            query.setUserDN(datawavePrincipal.getShortName());
            query.setId(UUID.randomUUID());
            query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
            
            List<QueryMetric> queryMetrics = getQueryMetrics(response, query, datawavePrincipal);
            response = processQueryMetricsSummary(queryMetrics);
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            enableLogs(true);
        }
        
        return response;
    }
    
    @Override
    public QueryMetricsSummaryHtmlResponse getUserQueriesSummary(Date begin, Date end, DatawavePrincipal datawavePrincipal) {
        QueryMetricsSummaryHtmlResponse response = new QueryMetricsSummaryHtmlResponse();
        
        try {
            String user = datawavePrincipal.getShortName();
            enableLogs(false);
            // this method is open to any user
            datawavePrincipal = callerPrincipal;
            
            Collection<? extends Collection<String>> authorizations = datawavePrincipal.getAuthorizations();
            QueryImpl query = new QueryImpl();
            query.setBeginDate(begin);
            query.setEndDate(end);
            query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
            query.setQuery("USER == '" + user + "'");
            query.setQueryName(QUERY_METRICS_LOGIC_NAME);
            query.setColumnVisibility(visibilityString);
            query.setQueryAuthorizations(AuthorizationsUtil.buildAuthorizationString(authorizations));
            query.setExpirationDate(DateUtils.addDays(new Date(), 1));
            query.setPagesize(1000);
            query.setUserDN(datawavePrincipal.getShortName());
            query.setId(UUID.randomUUID());
            query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
            
            List<QueryMetric> queryMetrics = getQueryMetrics(response, query, datawavePrincipal);
            response = processQueryMetricsHtmlSummary(queryMetrics);
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            enableLogs(true);
        }
        
        return response;
    }
    
    public QueryMetric toMetric(datawave.webservice.query.result.event.EventBase event) {
        SimpleDateFormat sdf_date_time1 = new SimpleDateFormat("yyyyMMdd HHmmss");
        SimpleDateFormat sdf_date_time2 = new SimpleDateFormat("yyyyMMdd HHmmss");
        
        try {
            QueryMetric m = new QueryMetric();
            List<FieldBase> field = event.getFields();
            
            TreeMap<Long,PageMetric> pageMetrics = Maps.newTreeMap();
            
            for (FieldBase f : field) {
                String fieldName = f.getName();
                String fieldValue = f.getValueString();
                
                if (fieldName.equals("USER")) {
                    m.setUser(fieldValue);
                } else if (fieldName.equals("USER_DN")) {
                    m.setUserDN(fieldValue);
                } else if (fieldName.equals("QUERY_ID")) {
                    m.setQueryId(fieldValue);
                } else if (fieldName.equals("CREATE_DATE")) {
                    try {
                        Date d = sdf_date_time2.parse(fieldValue);
                        m.setCreateDate(d);
                    } catch (Exception e) {
                        log.error(e.getMessage());
                    }
                } else if (fieldName.equals("QUERY")) {
                    m.setQuery(fieldValue);
                } else if (fieldName.equals("PLAN")) {
                    m.setPlan(fieldValue);
                } else if (fieldName.equals("QUERY_LOGIC")) {
                    m.setQueryLogic(fieldValue);
                } else if (fieldName.equals("QUERY_ID")) {
                    m.setQueryId(fieldValue);
                } else if (fieldName.equals("BEGIN_DATE")) {
                    try {
                        Date d = sdf_date_time1.parse(fieldValue);
                        m.setBeginDate(d);
                    } catch (Exception e) {
                        log.error(e.getMessage());
                    }
                } else if (fieldName.equals("END_DATE")) {
                    try {
                        Date d = sdf_date_time1.parse(fieldValue);
                        m.setEndDate(d);
                    } catch (Exception e) {
                        log.error(e.getMessage());
                    }
                } else if (fieldName.equals("HOST")) {
                    m.setHost(fieldValue);
                } else if (fieldName.equals("PROXY_SERVERS")) {
                    m.setProxyServers(Arrays.asList(StringUtils.split(fieldValue, ",")));
                } else if (fieldName.equals("AUTHORIZATIONS")) {
                    m.setQueryAuthorizations(fieldValue);
                } else if (fieldName.equals("QUERY_TYPE")) {
                    m.setQueryType(fieldValue);
                } else if (fieldName.equals("LIFECYCLE")) {
                    m.setLifecycle(Lifecycle.valueOf(fieldValue));
                } else if (fieldName.equals("ERROR_CODE")) {
                    m.setErrorCode(fieldValue);
                } else if (fieldName.equals("ERROR_MESSAGE")) {
                    m.setErrorMessage(fieldValue);
                } else if (fieldName.equals("SETUP_TIME")) {
                    m.setSetupTime(Long.parseLong(fieldValue));
                } else if (fieldName.equals("CREATE_CALL_TIME")) {
                    m.setCreateCallTime(Long.parseLong(fieldValue));
                } else if (fieldName.startsWith("PAGE_METRICS")) {
                    int index = fieldName.indexOf(".");
                    if (-1 == index) {
                        log.error("Could not parse field name to extract repetition count: " + fieldName);
                    } else {
                        Long repetition = Long.parseLong(fieldName.substring(index + 1));
                        
                        String[] parts = StringUtils.split(fieldValue, "/");
                        PageMetric pageMetric = null;
                        if (parts.length == 8) {
                            pageMetric = new PageMetric(Long.parseLong(parts[0]), Long.parseLong(parts[1]), Long.parseLong(parts[2]), Long.parseLong(parts[3]),
                                            Long.parseLong(parts[4]), Long.parseLong(parts[5]), Long.parseLong(parts[6]), Long.parseLong(parts[7]));
                        } else if (parts.length == 7) {
                            pageMetric = new PageMetric(Long.parseLong(parts[0]), Long.parseLong(parts[1]), Long.parseLong(parts[2]), Long.parseLong(parts[3]),
                                            Long.parseLong(parts[4]), Long.parseLong(parts[5]), Long.parseLong(parts[6]));
                        } else if (parts.length == 5) {
                            pageMetric = new PageMetric(Long.parseLong(parts[0]), Long.parseLong(parts[1]), Long.parseLong(parts[2]), Long.parseLong(parts[3]),
                                            Long.parseLong(parts[4]), 0l, 0l);
                        } else if (parts.length == 2) {
                            pageMetric = new PageMetric(Long.parseLong(parts[0]), Long.parseLong(parts[1]), 0l, 0l);
                        }
                        
                        if (pageMetric != null)
                            pageMetrics.put(repetition, pageMetric);
                    }
                } else if (fieldName.equals("POSITIVE_SELECTORS")) {
                    List<String> positiveSelectors = m.getPositiveSelectors();
                    if (positiveSelectors == null) {
                        positiveSelectors = new ArrayList<>();
                    }
                    positiveSelectors.add(fieldValue);
                    m.setPositiveSelectors(positiveSelectors);
                } else if (fieldName.equals("NEGATIVE_SELECTORS")) {
                    List<String> negativeSelectors = m.getNegativeSelectors();
                    if (negativeSelectors == null) {
                        negativeSelectors = new ArrayList<>();
                    }
                    negativeSelectors.add(fieldValue);
                    m.setNegativeSelectors(negativeSelectors);
                } else if (fieldName.equals("LAST_UPDATED")) {
                    try {
                        Date d = sdf_date_time2.parse(fieldValue);
                        m.setLastUpdated(d);
                    } catch (Exception e) {
                        log.error(e.getMessage());
                    }
                } else if (fieldName.equals("NUM_UPDATES")) {
                    try {
                        long numUpdates = Long.parseLong(fieldValue);
                        m.setNumUpdates(numUpdates);
                    } catch (Exception e) {
                        log.error(e.getMessage());
                    }
                } else if (fieldName.equals("QUERY_NAME")) {
                    m.setQueryName(fieldValue);
                } else if (fieldName.equals("PARAMETERS")) {
                    if (fieldValue != null) {
                        try {
                            Set<Parameter> parameters = QueryUtil.parseParameters(fieldValue);
                            m.setParameters(parameters);
                            
                        } catch (Exception e) {
                            log.debug(e.getMessage());
                        }
                    }
                }
                
                else if (fieldName.equals("SOURCE_COUNT")) {
                    m.setSourceCount(Long.parseLong(fieldValue));
                }
                
                else if (fieldName.equals("NEXT_COUNT")) {
                    m.setNextCount(Long.parseLong(fieldValue));
                }
                
                else if (fieldName.equals("SEEK_COUNT")) {
                    m.setSeekCount(Long.parseLong(fieldValue));
                }
                
                else if (fieldName.equals("YIELD_COUNT")) {
                    m.setYieldCount(Long.parseLong(fieldValue));
                }
                
                else if (fieldName.equals("DOC_RANGES")) {
                    m.setDocRanges(Long.parseLong(fieldValue));
                }
                
                else if (fieldName.equals("FI_RANGES")) {
                    m.setFiRanges(Long.parseLong(fieldValue));
                } else {
                    log.error("encountered unanticipated field name: " + fieldName);
                }
            }
            
            for (final Entry<Long,PageMetric> entry : pageMetrics.entrySet())
                m.addPageMetric(entry.getValue());
            
            return m;
        } catch (Exception e) {
            log.warn("Unexpected error creating query metric. Returning null", e);
            return null;
        }
    }
    
    protected void createAndConfigureTablesIfNecessary(String[] tableNames, TableOperations tops, Configuration conf) throws AccumuloSecurityException,
                    AccumuloException, TableNotFoundException {
        for (String table : tableNames) {
            // If the tables don't exist, then create them.
            try {
                if (!tops.exists(table)) {
                    tops.create(table);
                    Map<String,TableConfigHelper> tableConfigs = getTableConfigs(log, conf, tableNames);
                    
                    TableConfigHelper tableHelper = tableConfigs.get(table);
                    
                    if (tableHelper != null) {
                        tableHelper.configure(tops);
                    } else {
                        log.info("No configuration supplied for table: " + table);
                    }
                }
            } catch (TableExistsException te) {
                // in this case, somebody else must have created the table after our existence check
                log.debug("Tried to create " + table + " but somebody beat us to the punch");
            }
        }
    }
    
    @SuppressWarnings("unchecked")
    private Map<String,TableConfigHelper> getTableConfigs(Logger log, Configuration conf, String[] tableNames) {
        Map<String,TableConfigHelper> helperMap = new HashMap<>(tableNames.length);
        
        for (String table : tableNames) {
            String prop = table + TableConfigHelper.TABLE_CONFIG_CLASS_SUFFIX;
            String className = conf.get(prop, null);
            TableConfigHelper tableHelper = null;
            
            if (className != null) {
                try {
                    Class<? extends TableConfigHelper> tableHelperClass = (Class<? extends TableConfigHelper>) Class.forName(className.trim());
                    tableHelper = tableHelperClass.newInstance();
                    
                    if (tableHelper != null)
                        tableHelper.setup(table, conf, log);
                } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
                    throw new IllegalArgumentException(e);
                }
            }
            
            helperMap.put(table, tableHelper);
        }
        
        return helperMap;
    }
    
    private void enableLogs(boolean enable) {
        if (enable) {
            ThreadLocalLogLevel.clear();
        } else {
            // All loggers that are encountered in the call chain during metrics calls should be included here.
            // If you need to add a logger name here, you also need to change the Logger declaration where that Logger is instantiated
            // Change:
            // Logger log = Logger.getLogger(MyClass.class);
            // to
            // Logger log = ThreadConfigurableLogger.getLogger(MyClass.class);
            
            ThreadLocalLogLevel.setLevel("datawave.query.index.lookup.RangeStream", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.metrics.ShardTableQueryMetricHandler", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.planner.DefaultQueryPlanner", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.planner.ThreadedRangeBundlerIterator", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.scheduler.SequentialScheduler", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.tables.ShardQueryLogic", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.metrics.ShardTableQueryMetricHandler", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.jexl.visitors.QueryModelVisitor", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.jexl.visitors.ExpandMultiNormalizedTerms", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.jexl.lookups.LookupBoundedRangeForTerms", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.query.jexl.visitors.RangeConjunctionRebuildingVisitor", Level.ERROR);
            
            ThreadLocalLogLevel.setLevel("datawave.ingest.data.TypeRegistry", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.ingest.data.config.ingest.BaseIngestHelper", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.ingest.util.RegionTimer", Level.ERROR);
            ThreadLocalLogLevel.setLevel("datawave.ingest.data.Event", Level.OFF);
        }
    }
    
    @Override
    public void reload() {
        try {
            if (this.recordWriter != null) {
                // don't try to flush the mtbw (close). If recordWriter != null then this method is being called
                // because of an Exception and the metrics have been saved off to be added to the new recordWriter.
                this.recordWriter.returnConnector();
            }
            recordWriter = new AccumuloRecordWriter(this.connectionFactory, conf);
        } catch (AccumuloException | AccumuloSecurityException | IOException e) {
            log.error(e.getMessage(), e);
        }
    }
    
    @Override
    public QueryMetricsSummaryHtmlResponse getTotalQueriesSummary(Date begin, Date end, DatawavePrincipal datawavePrincipal) {
        QueryMetricsSummaryHtmlResponse response = new QueryMetricsSummaryHtmlResponse();
        
        try {
            enableLogs(false);
            enableLogs(true);
            // this method is open to any user
            datawavePrincipal = callerPrincipal;
            
            Collection<? extends Collection<String>> authorizations = datawavePrincipal.getAuthorizations();
            QueryImpl query = new QueryImpl();
            query.setBeginDate(begin);
            query.setEndDate(end);
            query.setQueryLogicName(QUERY_METRICS_LOGIC_NAME);
            query.setQuery("USER > 'A' && USER < 'ZZZZZZZ'");
            query.setQueryName(QUERY_METRICS_LOGIC_NAME);
            query.setColumnVisibility(visibilityString);
            query.setQueryAuthorizations(AuthorizationsUtil.buildAuthorizationString(authorizations));
            query.setExpirationDate(DateUtils.addDays(new Date(), 1));
            query.setPagesize(1000);
            query.setUserDN(datawavePrincipal.getShortName());
            query.setId(UUID.randomUUID());
            query.setParameters(ImmutableMap.of(QueryOptions.INCLUDE_GROUPING_CONTEXT, "true"));
            
            List<QueryMetric> queryMetrics = getQueryMetrics(response, query, datawavePrincipal);
            response = processQueryMetricsHtmlSummary(queryMetrics);
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            enableLogs(true);
        }
        
        return response;
    }
}