/*
 * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co.,
 * Inc., Kenilworth, NJ, USA.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.msd.gin.halyard.sail;

import com.msd.gin.halyard.vocab.HALYARD;
import com.msd.gin.halyard.common.HalyardTableUtils;
import com.msd.gin.halyard.optimizers.HalyardEvaluationStatistics;
import com.msd.gin.halyard.strategy.HalyardEvaluationStrategy;
import com.msd.gin.halyard.strategy.HalyardEvaluationStrategy.ServiceRoot;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.eclipse.rdf4j.IsolationLevel;
import org.eclipse.rdf4j.IsolationLevels;
import org.eclipse.rdf4j.common.iteration.CloseableIteration;
import org.eclipse.rdf4j.common.iteration.CloseableIteratorIteration;
import org.eclipse.rdf4j.common.iteration.ExceptionConvertingIteration;
import org.eclipse.rdf4j.common.iteration.TimeLimitIteration;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import org.eclipse.rdf4j.model.Namespace;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.Value;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.SimpleNamespace;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.model.vocabulary.SD;
import org.eclipse.rdf4j.model.vocabulary.VOID;
import org.eclipse.rdf4j.query.BindingSet;
import org.eclipse.rdf4j.query.Dataset;
import org.eclipse.rdf4j.query.QueryEvaluationException;
import org.eclipse.rdf4j.query.algebra.QueryRoot;
import org.eclipse.rdf4j.query.algebra.Service;
import org.eclipse.rdf4j.query.algebra.TupleExpr;
import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy;
import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource;
import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedService;
import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolver;
import org.eclipse.rdf4j.query.algebra.evaluation.impl.StrictEvaluationStrategy;
import org.eclipse.rdf4j.query.impl.EmptyBindingSet;
import org.eclipse.rdf4j.repository.sparql.query.InsertBindingSetCursor;
import org.eclipse.rdf4j.sail.Sail;
import org.eclipse.rdf4j.sail.SailConnection;
import org.eclipse.rdf4j.sail.SailException;
import org.eclipse.rdf4j.sail.UnknownSailTransactionStateException;
import org.eclipse.rdf4j.sail.UpdateContext;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

/**
 * HBaseSail is the RDF Storage And Inference Layer (SAIL) implementation on top of Apache HBase.
 * It implements the interfaces - {@code Sail, SailConnection} and {@code FederatedServiceResolver}. Currently federated queries are
 * only supported for queries across multiple graphs in one Halyard database.
 * @author Adam Sotona (MSD)
 */
public class HBaseSail implements Sail, SailConnection, FederatedServiceResolver, FederatedService {

    /**
     * Ticker is a simple service interface that is notified when some data are processed.
     * It's purpose is to notify a caller (for example MapReduce task) that the execution is still alive.
     */
    public interface Ticker {

        /**
         * This method is called whenever a new Statement is populated from HBase.
         */
        public void tick();
    }

    private static final Logger LOG = Logger.getLogger(HBaseSail.class.getName());
    private static final long STATUS_CACHING_TIMEOUT = 60000l;
    private static final int ELASTIC_RESULT_SIZE = 10000;

    private final Configuration config; //the configuration of the HBase database
    final String tableName;
    final boolean create;
    final boolean pushStrategy;
    final int splitBits;
    protected final HalyardEvaluationStatistics statistics;
    final int evaluationTimeout;
    private boolean readOnly = false;
    private long readOnlyTimestamp = -1;
    final String elasticIndexURL;
    private final Ticker ticker;

    HTable table = null;

    private final Map<String, Namespace> namespaces = new HashMap<>();
    private final Map<String, HBaseSail> federatedServices = new HashMap<>();

    /**
     * Construct HBaseSail object with given arguments.
     * @param config Hadoop Configuration to access HBase
     * @param tableName HBase table name used to store data
     * @param create boolean option to create the table if it does not exist
     * @param splitBits int number of bits used for the calculation of HTable region pre-splits (applies for new tables only)
     * @param pushStrategy boolean option to use {@link com.msd.gin.halyard.strategy.HalyardEvaluationStrategy} instead of {@link org.eclipse.rdf4j.query.algebra.evaluation.impl.StrictEvaluationStrategy}
     * @param evaluationTimeout int timeout in seconds for each query evaluation, negative values mean no timeout
     * @param elasticIndexURL String optional ElasticSearch index URL
     * @param ticker optional Ticker callback for keep-alive notifications
     */
    public HBaseSail(Configuration config, String tableName, boolean create, int splitBits, boolean pushStrategy, int evaluationTimeout, String elasticIndexURL, Ticker ticker) {
        this.config = config;
        this.tableName = tableName;
        this.create = create;
        this.splitBits = splitBits;
        this.pushStrategy = pushStrategy;
        this.statistics = new HalyardEvaluationStatistics(new HalyardStatsBasedStatementPatternCardinalityCalculator(this), (String service) -> {
            FederatedService servSail = getService(service);
            return servSail != null ? ((HBaseSail)servSail).statistics : null;
        });
        this.evaluationTimeout = evaluationTimeout;
        this.elasticIndexURL = elasticIndexURL;
        this.ticker = ticker;
    }

    /**
     * Not used in Halyard
     */
    @Override
    public void setDataDir(File dataDir) {
    }

    /**
     * Not used in Halyard
     */
    @Override
    public File getDataDir() {
        throw new UnsupportedOperationException();
    }

    @Override
    public void initialize() throws SailException { //initialize the SAIL
        try {
        	    //get or create and get the HBase table
            table = HalyardTableUtils.getTable(config, tableName, create, splitBits);

            //Iterate over statements relating to namespaces and add them to the namespace map.
            try (CloseableIteration<? extends Statement, SailException> nsIter = getStatements(null, HALYARD.NAMESPACE_PREFIX_PROPERTY, null, true)) {
                while (nsIter.hasNext()) {
                    Statement st = nsIter.next();
                    if (st.getObject() instanceof Literal) {
                        String prefix = st.getObject().stringValue();
                        String name = st.getSubject().stringValue();
                        namespaces.put(prefix, new SimpleNamespace(prefix, name));
                    }
                }
            }
        } catch (IOException ex) {
            throw new SailException(ex);
        }
    }

    @Override
    public FederatedService getService(String serviceUrl) throws QueryEvaluationException {
        //provide a service to query over Halyard graphs. Remote service queries are not supported.
        if (serviceUrl.startsWith(HALYARD.NAMESPACE)) {
            String federatedTable = serviceUrl.substring(HALYARD.NAMESPACE.length());
            HBaseSail sail = federatedServices.get(federatedTable);
            if (sail == null) {
                sail = new HBaseSail(config, federatedTable, false, 0, true, evaluationTimeout, null, ticker);
                federatedServices.put(federatedTable, sail);
                sail.initialize();
            }
            return sail;
        } else {
            throw new QueryEvaluationException("Unsupported service URL: " + serviceUrl);
        }
    }

    @Override
    public boolean ask(Service service, BindingSet bindings, String baseUri) throws QueryEvaluationException {
        try (CloseableIteration<BindingSet, QueryEvaluationException> res = evaluate(new ServiceRoot(service.getArg()), null, bindings, true)) {
            return res.hasNext();
        }
    }

    @Override
    public CloseableIteration<BindingSet, QueryEvaluationException> select(Service service, Set<String> projectionVars, BindingSet bindings, String baseUri) throws QueryEvaluationException {
        return new InsertBindingSetCursor(evaluate(new ServiceRoot(service.getArg()), null, bindings, true), bindings);
    }

    @Override
    public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(Service service, CloseableIteration<BindingSet, QueryEvaluationException> bindings, String baseUri) throws QueryEvaluationException {
        throw new UnsupportedOperationException();
    }

    @Override
    public boolean isInitialized() {
        return table != null;
    }

    @Override
    public void shutdown() throws QueryEvaluationException {
        shutDown();
    }

    @Override
    public void shutDown() throws SailException { //release resources
        try {
            if (table != null) table.close(); //close the HTable
            table = null;
        } catch (IOException ex) {
            throw new SailException(ex);
        }
        for (HBaseSail s : federatedServices.values()) { //shutdown all federated services
            s.shutdown();
        }
        federatedServices.clear(); // release the references to the services
    }

    @Override
    public boolean isWritable() throws SailException {
        if (readOnlyTimestamp + STATUS_CACHING_TIMEOUT < System.currentTimeMillis()) try {
            readOnly = table.getTableDescriptor().isReadOnly();
            readOnlyTimestamp = System.currentTimeMillis();
        } catch (IOException ex) {
            throw new SailException(ex);
        }
        return !readOnly;
    }

    @Override
    public SailConnection getConnection() throws SailException {
        return this;
    }

    @Override
    public ValueFactory getValueFactory() {
        return SimpleValueFactory.getInstance();
    }

    @Override
    public List<IsolationLevel> getSupportedIsolationLevels() {
        return Collections.singletonList((IsolationLevel) IsolationLevels.NONE); //limited by HBase's capabilities
    }

    @Override
    public IsolationLevel getDefaultIsolationLevel() {
        return IsolationLevels.NONE;
    }

    @Override
    public boolean isOpen() throws SailException {
        return table != null;  //if the table exists the table is open
    }

    @Override
    public void close() throws SailException {
    }

    //generates a Resource[] from 0 or more Resources
    protected static Resource[] normalizeContexts(Resource... contexts) {
        if (contexts == null || contexts.length == 0) {
            return new Resource[] {null};
        } else {
            return contexts;
        }
    }

    //evaluate queries/ subqueries
    @Override
    public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings, final boolean includeInferred) throws SailException {
        LOG.log(Level.FINE, "Evaluated TupleExpr before optimizers:\n{0}", tupleExpr);
        tupleExpr = tupleExpr.clone();
        if (!(tupleExpr instanceof QueryRoot)) {
            // Add a dummy root node to the tuple expressions to allow the
            // optimizers to modify the actual root node
            tupleExpr = new QueryRoot(tupleExpr);
        }

        final long startTime = System.currentTimeMillis();

        TripleSource source = new TripleSource() {
            @Override
            public CloseableIteration<? extends Statement, QueryEvaluationException> getStatements(Resource subj, IRI pred, Value obj, Resource... contexts) throws QueryEvaluationException {
                try {
                    return new ExceptionConvertingIteration<Statement, QueryEvaluationException>(createScanner(startTime, subj, pred, obj, contexts)) {
                        @Override
                        protected QueryEvaluationException convert(Exception e) {
                            return new QueryEvaluationException(e);
                        }

                    };
                } catch (SailException ex) {
                    throw new QueryEvaluationException(ex);
                }
            }

            @Override
            public ValueFactory getValueFactory() {
                return HBaseSail.this.getValueFactory();
            }
        };

        EvaluationStrategy strategy = pushStrategy ? new HalyardEvaluationStrategy(source, dataset, this, statistics, evaluationTimeout) : new StrictEvaluationStrategy(source, dataset, this);
        strategy.optimize(tupleExpr, statistics, bindings);
        LOG.log(Level.FINE, "Evaluated TupleExpr after optimization:\n{0}", tupleExpr);
        try {
        		//evaluate the expression against the TripleSource according to the EvaluationStrategy.
            CloseableIteration<BindingSet, QueryEvaluationException> iter = evaluateInternal(strategy, tupleExpr);
            return evaluationTimeout <= 0 ? iter : new TimeLimitIteration<BindingSet, QueryEvaluationException>(iter, 1000l * evaluationTimeout) {
                @Override
                protected void throwInterruptedException() throws QueryEvaluationException {
                    throw new QueryEvaluationException("Query evaluation exceeded specified timeout " + evaluationTimeout + "s");
                }
            };
        } catch (QueryEvaluationException ex) {
            throw new SailException(ex);
        }
    }

    protected CloseableIteration<BindingSet, QueryEvaluationException> evaluateInternal(EvaluationStrategy strategy, TupleExpr tupleExpr) {
        return strategy.evaluate(tupleExpr, EmptyBindingSet.getInstance());
    }

    @Override
    public CloseableIteration<? extends Resource, SailException> getContextIDs() throws SailException {

        //generate an iterator over the identifiers of the contexts available in Halyard.
        final CloseableIteration<? extends Statement, SailException> scanner = getStatements(HALYARD.STATS_ROOT_NODE, SD.NAMED_GRAPH_PROPERTY, null, true, HALYARD.STATS_GRAPH_CONTEXT);
        return new CloseableIteration<Resource, SailException>() {
            @Override
            public void close() throws SailException {
                scanner.close();
            }

            @Override
            public boolean hasNext() throws SailException {
                return scanner.hasNext();
            }

            @Override
            public Resource next() throws SailException {
                return (IRI)scanner.next().getObject();
            }

            @Override
            public void remove() throws SailException {
                throw new UnsupportedOperationException();
            }

        };
    }

    @Override
    public CloseableIteration<? extends Statement, SailException> getStatements(Resource subj, IRI pred, Value obj, boolean includeInferred, Resource... contexts) throws SailException {
        return createScanner(System.currentTimeMillis(), subj, pred, obj, contexts);
    }

    private StatementScanner createScanner(long startTime, Resource subj, IRI pred, Value obj, Resource...contexts) throws SailException {
        if ((obj instanceof Literal) && (HALYARD.SEARCH_TYPE.equals(((Literal)obj).getDatatype()))) {
            return new LiteralSearchStatementScanner(startTime, subj, pred, obj.stringValue(), contexts);
        } else {
            return new StatementScanner(startTime, subj, pred, obj, contexts);
        }
    }

    @Override
    public synchronized long size(Resource... contexts) throws SailException {
        long size = 0;
        if (contexts != null && contexts.length > 0 && contexts[0] != null) {
            for (Resource ctx : contexts) {
            		//examine the VOID statistics for the count of triples in this context
                try (CloseableIteration<? extends Statement, SailException> scanner = getStatements(ctx, VOID.TRIPLES, null, true, HALYARD.STATS_GRAPH_CONTEXT)) {
                    if (scanner.hasNext()) {
                        size += ((Literal)scanner.next().getObject()).longValue();
                    }
                    if (scanner.hasNext()) {
                        throw new SailException("Multiple different values exist in VOID statistics for context: "+ctx.stringValue()+". Considering removing and recomputing statistics");
                    }
                }
            }
        } else {
            try (CloseableIteration<? extends Statement, SailException> scanner = getStatements(HALYARD.STATS_ROOT_NODE, VOID.TRIPLES, null, true, HALYARD.STATS_GRAPH_CONTEXT)) {
                if (scanner.hasNext()) {
                    size += ((Literal)scanner.next().getObject()).longValue();
                }
                if (scanner.hasNext()) {
                    throw new SailException("Multiple different values exist in VOID statistics. Considering removing and recomputing statistics");
                }
            }
        }
        // try to count it manually if there are no stats and there is a specific timeout
        if (size == 0 && evaluationTimeout > 0) try (CloseableIteration<? extends Statement, SailException> scanner = getStatements(null, null, null, true, contexts)) {
            while (scanner.hasNext()) {
                scanner.next();
                size++;
            }
        }
        return size;
    }

    @Override
    public void begin() throws SailException { //transactions are not supported
    }

    @Override
    public void begin(IsolationLevel level) throws UnknownSailTransactionStateException, SailException {
        if (level != null && level != IsolationLevels.NONE) {
            throw new UnknownSailTransactionStateException("Isolation level " + level + " is not compatible with this HBaseSail");
        }
    }

    @Override
    public void flush() throws SailException {
    }

    @Override
    public void prepare() throws SailException {
    }

    @Override
    public void commit() throws SailException {
        try {
            table.flushCommits(); //execute all buffered puts in HBase
        } catch (IOException ex) {
            throw new SailException(ex);
        }
    }

    @Override
    public void rollback() throws SailException {
    }

    @Override
    public boolean isActive() throws UnknownSailTransactionStateException {
        return true;
    }

    @Override
    public void addStatement(UpdateContext op, Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException {
        addStatement(subj, pred, obj, contexts);
    }

    protected long getDefaultTimeStamp() {
        return System.currentTimeMillis();
    }

    @Override
    public void addStatement(Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException {
        long timestamp = getDefaultTimeStamp();
        for (Resource ctx : normalizeContexts(contexts)) {
            addStatementInternal(subj, pred, obj, ctx, timestamp);
        }
    }

    protected void addStatementInternal(Resource subj, IRI pred, Value obj, Resource context, long timestamp) throws SailException {
        if (!isWritable()) throw new SailException(tableName + " is read only");
        try {
            for (KeyValue kv : HalyardTableUtils.toKeyValues(subj, pred, obj, context, false, timestamp)) { //serialize the key value pairs relating to the statement in HBase
                put(kv);
            }
        } catch (IOException e) {
            throw new SailException(e);
        }
    }

    protected synchronized void put(KeyValue kv) throws IOException {
        table.put(new Put(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), kv.getTimestamp()).add(kv));
    }

    @Override
    public void removeStatement(UpdateContext op, Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException {
    	    //should we be defensive about nulls for subj, pre and obj? removeStatements is where you would use nulls, not here.

        if (!isWritable()) throw new SailException(tableName + " is read only");
        long timestamp = getDefaultTimeStamp();
        try {
            for (Resource ctx : normalizeContexts(contexts)) {
                for (KeyValue kv : HalyardTableUtils.toKeyValues(subj, pred, obj, ctx, true, timestamp)) { //calculate the kv's corresponding to the quad (or triple)
                    delete(kv);
                }
            }
        } catch (IOException e) {
            throw new SailException(e);
        }
    }

    protected void delete(KeyValue kv) throws IOException {
        table.delete(new Delete(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()).addDeleteMarker(kv));
    }

    @Override
    public void removeStatements(Resource subj, IRI pred, Value obj, Resource... contexts) throws SailException {
        if (!isWritable()) throw new SailException(tableName + " is read only");
        contexts = normalizeContexts(contexts);
        if (subj == null && pred == null && obj == null && contexts[0] == null) {
            clearAll();
        } else {
            try (CloseableIteration<? extends Statement, SailException> iter = getStatements(subj, pred, obj, true, contexts)) {
                while (iter.hasNext()) {
                    Statement st = iter.next();
                    removeStatement(null, st.getSubject(), st.getPredicate(), st.getObject(), st.getContext());
                }
            }
        }
    }

    @Override
    public boolean pendingRemovals() {
        return false;
    }

    @Override
    public void startUpdate(UpdateContext op) throws SailException {
    }

    @Override
    public void endUpdate(UpdateContext op) throws SailException {
    }

    @Override
    public void clear(Resource... contexts) throws SailException {
        removeStatements(null, null, null, contexts); //remove all statements in the contexts.
    }

    private void clearAll() throws SailException {
        if (!isWritable()) throw new SailException(tableName + " is read only");
        try {
            table = HalyardTableUtils.truncateTable(table); //delete all triples, the whole DB but retains splits!
        } catch (IOException ex) {
            throw new SailException(ex);
        }
    }

    @Override
    public String getNamespace(String prefix) throws SailException {
        Namespace namespace = namespaces.get(prefix);
        return (namespace == null) ? null : namespace.getName();
    }

    @Override
    public CloseableIteration<? extends Namespace, SailException> getNamespaces() {
        return new CloseableIteratorIteration<>(namespaces.values().iterator());
    }

    @Override
    public void setNamespace(String prefix, String name) throws SailException {
        namespaces.put(prefix, new SimpleNamespace(prefix, name));
        ValueFactory vf = getValueFactory();
        try {
            removeStatements(null, HALYARD.NAMESPACE_PREFIX_PROPERTY, vf.createLiteral(prefix));
            addStatementInternal(vf.createIRI(name), HALYARD.NAMESPACE_PREFIX_PROPERTY, vf.createLiteral(prefix), HALYARD.SYSTEM_GRAPH_CONTEXT, getDefaultTimeStamp());
        } catch (SailException e) {
            LOG.log(Level.WARNING, "Namespace prefix could not be presisted due to an exception", e);
        }
    }

    @Override
    public void removeNamespace(String prefix) throws SailException {
        ValueFactory vf = getValueFactory();
        namespaces.remove(prefix);
        try {
            removeStatements(null, HALYARD.NAMESPACE_PREFIX_PROPERTY, vf.createLiteral(prefix));
        } catch (SailException e) {
            LOG.log(Level.WARNING, "Namespace prefix could not be removed due to an exception", e);
        }
    }

    @Override
    public void clearNamespaces() throws SailException {
        try {
            removeStatements(null, HALYARD.NAMESPACE_PREFIX_PROPERTY, null);
        } catch (SailException e) {
            LOG.log(Level.WARNING, "Namespaces could not be cleared due to an exception", e);
        }
        namespaces.clear();
    }

    private static final Map <String, List<byte[]>> SEARCH_CACHE = new WeakHashMap<>();

    //Scans the Halyard table for statements that match the specified pattern
    private class LiteralSearchStatementScanner extends StatementScanner {

        Iterator<byte[]> objectHashes = null;
        private final String literalSearchQuery;

        public LiteralSearchStatementScanner(long startTime, Resource subj, IRI pred, String literalSearchQuery, Resource... contexts) throws SailException {
            super(startTime, subj, pred, null, contexts);
            if (elasticIndexURL == null || elasticIndexURL.length() == 0) {
                throw new SailException("ElasticSearch Index URL is not properly configured.");
            }
            this.literalSearchQuery = literalSearchQuery;
        }

        @Override
        protected Result nextResult() throws IOException {
            while (true) {
                if (objHash == null) {
                    if (objectHashes == null) { //perform ES query and parse results
                        synchronized (SEARCH_CACHE) {
                            List<byte[]> objectHashesList = SEARCH_CACHE.get(literalSearchQuery);
                            if (objectHashesList == null) {
                                objectHashesList = new ArrayList<>();
                                HttpURLConnection http = (HttpURLConnection)(new URL(elasticIndexURL + "/_search").openConnection());
                                try {
                                    http.setRequestMethod("POST");
                                    http.setDoOutput(true);
                                    http.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
                                    http.connect();
                                    try (PrintStream out = new PrintStream(http.getOutputStream(), true, "UTF-8")) {
                                        out.print("{\"query\":{\"query_string\":{\"query\":" + JSONObject.quote(literalSearchQuery) + "}},\"_source\":false,\"stored_fields\":\"_id\",\"size\":" + ELASTIC_RESULT_SIZE + "}");
                                    }
                                    int response = http.getResponseCode();
                                    String msg = http.getResponseMessage();
                                    if (response != 200) {
                                        LOG.info("ElasticSearch error response: " + msg + " on query: " + literalSearchQuery);
                                    } else {
                                        try (InputStreamReader isr = new InputStreamReader(http.getInputStream(), "UTF-8")) {
                                            JSONArray hits = new JSONObject(new JSONTokener(isr)).getJSONObject("hits").getJSONArray("hits");
                                            for (int i=0; i<hits.length(); i++) {
                                                objectHashesList.add(Hex.decodeHex(hits.getJSONObject(i).getString("_id").toCharArray()));
                                            }
                                        }
                                        SEARCH_CACHE.put(new String(literalSearchQuery), objectHashesList);
                                    }
                                } catch (JSONException | DecoderException ex) {
                                    throw new IOException(ex);
                                } finally {
                                    http.disconnect();
                                }
                            }
                            objectHashes = objectHashesList.iterator();
                        }
                    }
                    if (objectHashes.hasNext()) {
                        objHash = objectHashes.next();
                    } else {
                        return null;
                    }
                    contexts = contextsList.iterator(); //reset iterator over contexts
                }
                Result res = super.nextResult();
                if (res == null) {
                    objHash = null;
                } else {
                    return res;
                }
            }
        }
    }

    private class StatementScanner implements CloseableIteration<Statement, SailException> {

        private final Resource subj;
        private final IRI pred;
        private final Value obj;
        private final byte[] subjHash, predHash;
        protected final List<Resource> contextsList;
        protected Iterator<Resource> contexts;
        protected byte[] objHash;
        private ResultScanner rs = null;
        private final long endTime;
        private Statement next = null;
        private Iterator<Statement> iter = null;

        public StatementScanner(long startTime, Resource subj, IRI pred, Value obj, Resource...contexts) throws SailException {
            this.subj = subj;
            this.pred = pred;
            this.obj = obj;
            this.subjHash = HalyardTableUtils.hashKey(subj);
            this.predHash = HalyardTableUtils.hashKey(pred);
            this.objHash = HalyardTableUtils.hashKey(obj);
            this.contextsList = Arrays.asList(normalizeContexts(contexts));
            this.contexts = contextsList.iterator();
            this.endTime = startTime + (1000l * evaluationTimeout);
            LOG.log(Level.FINEST, "New StatementScanner {0} {1} {2} {3}", new Object[]{subj, pred, obj, contextsList});
        }

        protected Result nextResult() throws IOException { //gets the next result to consider from the HBase Scan
            while (true) {
                if (rs == null) {
                    if (contexts.hasNext()) {

                        //build a ResultScanner from an HBase Scan that finds potential matches
                        rs = table.getScanner(HalyardTableUtils.scan(subjHash, predHash, objHash, HalyardTableUtils.hashKey(contexts.next())));
                    } else {
                        return null;
                    }
                }
                Result res = rs.next();
                if (ticker != null) ticker.tick(); //sends a tick for keep alive purposes
                if (res == null) { // no more results from this ResultScanner, close and clean up.
                    rs.close();
                    rs = null;
                } else {
                    return res;
                }
            }
        }

        @Override
        public void close() throws SailException {
            if (rs != null) {
                rs.close();
            }
        }

        @Override
        public synchronized boolean hasNext() throws SailException {
            if (evaluationTimeout > 0 && System.currentTimeMillis() > endTime) {
                throw new SailException("Statements scanning exceeded specified timeout " + evaluationTimeout + "s");
            }
            if (next == null) try { //try and find the next result
                while (true) {
                    if (iter == null) {
                        Result res = nextResult();
                        if (res == null) {
                            return false; //no more Results
                        } else {
                            iter = HalyardTableUtils.parseStatements(res, getValueFactory()).iterator();
                        }
                    }
                    while (iter.hasNext()) {
                        Statement s = iter.next();
                        if ((subj == null || subj.equals(s.getSubject())) && (pred == null || pred.equals(s.getPredicate())) && (obj == null || obj.equals(s.getObject()))) {
                            next = s;  //cache the next statement which will be returned with a call to next().
                            return true; //there is another statement
                        }
                    }
                    iter = null;
                }
            } catch (IOException e) {
                throw new SailException(e);
            } else {
                return true;
            }
        }

        @Override
        public synchronized Statement next() throws SailException {
            if (hasNext()) { //return the next statement and set next to null so it can be refilled by the next call to hasNext()
                Statement st = next;
                next = null;
                return st;
            } else {
                throw new NoSuchElementException();
            }
        }

        @Override
        public void remove() throws SailException {
            throw new UnsupportedOperationException();
        }
    }
}