* Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * See the License for the specific language governing permissions and
 * limitations under the License.

package org.apache.hadoop.hbase.regionserver.wal;

import static org.junit.Assert.assertEquals;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.io.util.LRUDictionary;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.IndexTestingUtils;
import org.apache.phoenix.hbase.index.wal.IndexedKeyValue;
import org.junit.BeforeClass;
import org.junit.Test;

 * Simple test to read/write simple files via our custom {@link WALCellCodec} to ensure properly
 * encoding/decoding without going through a cluster.
public class ReadWriteKeyValuesWithCodecIT {

  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
  private static final byte[] ROW = Bytes.toBytes("row");
  private static final byte[] FAMILY = Bytes.toBytes("family");

  public static void setupCodec() {
    Configuration conf = UTIL.getConfiguration();
    conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, IndexedWALEditCodec.class.getName());

  public void testWithoutCompression() throws Exception {
    // get the FS ready to read/write the edits
    Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withoutCompression");
    Path testFile = new Path(testDir, "testfile");
    FileSystem fs = UTIL.getTestFileSystem();

    List<WALEdit> edits = getEdits();
    writeReadAndVerify(null, fs, edits, testFile);

  public void testWithCompression() throws Exception {
    // get the FS ready to read/write the edit
    Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withCompression");
    Path testFile = new Path(testDir, "testfile");
    FileSystem fs = UTIL.getTestFileSystem();

    List<WALEdit> edits = getEdits();
    CompressionContext compression = new CompressionContext(LRUDictionary.class, false, false);
    writeReadAndVerify(compression, fs, edits, testFile);

   * @return a bunch of {@link WALEdit}s that test a range of serialization possibilities.
  private List<WALEdit> getEdits() {
    // Build up a couple of edits
    List<WALEdit> edits = new ArrayList<WALEdit>();
    Put p = new Put(ROW);
    p.add(FAMILY, null, Bytes.toBytes("v1"));

    WALEdit withPut = new WALEdit();
    addMutation(withPut, p, FAMILY);

    Delete d = new Delete(ROW);
    d.deleteColumn(FAMILY, null);
    WALEdit withDelete = new WALEdit();
    addMutation(withDelete, d, FAMILY);
    WALEdit withPutsAndDeletes = new WALEdit();
    addMutation(withPutsAndDeletes, d, FAMILY);
    addMutation(withPutsAndDeletes, p, FAMILY);
    WALEdit justIndexUpdates = new WALEdit();
    byte[] table = Bytes.toBytes("targetTable");
    IndexedKeyValue ikv = new IndexedKeyValue(table, p);

    WALEdit mixed = new WALEdit();
    addMutation(mixed, d, FAMILY);
    addMutation(mixed, p, FAMILY);

    return edits;

   * Add all the {@link KeyValue}s in the {@link Mutation}, for the pass family, to the given
   * {@link WALEdit}.
  private void addMutation(WALEdit edit, Mutation m, byte[] family) {
    List<Cell> kvs = m.getFamilyCellMap().get(FAMILY);
    for (Cell kv : kvs) {

  private void writeWALEdit(WALCellCodec codec, List<Cell> kvs, FSDataOutputStream out) throws IOException {
    Codec.Encoder cellEncoder = codec.getEncoder(out);
    // We interleave the two lists for code simplicity
    for (Cell kv : kvs) {
   * Write the edits to the specified path on the {@link FileSystem} using the given codec and then
   * read them back in and ensure that we read the same thing we wrote.
  private void writeReadAndVerify(final CompressionContext compressionContext, FileSystem fs, List<WALEdit> edits,
      Path testFile) throws IOException {
	WALCellCodec codec = WALCellCodec.create(UTIL.getConfiguration(), compressionContext);  
    // write the edits out
    FSDataOutputStream out = fs.create(testFile);
    for (WALEdit edit : edits) {
      writeWALEdit(codec, edit.getCells(), out);

    // read in the edits
    FSDataInputStream in = fs.open(testFile);
    List<WALEdit> read = new ArrayList<WALEdit>();
    for (int i = 0; i < edits.size(); i++) {
      WALEdit edit = new WALEdit();
      int numEdits = in.readInt();
      edit.readFromCells(codec.getDecoder(in), numEdits);

    // make sure the read edits match the written
    for(int i=0; i< edits.size(); i++){
      WALEdit expected = edits.get(i);
      WALEdit found = read.get(i);
      for(int j=0; j< expected.getCells().size(); j++){
        Cell fkv = found.getCells().get(j);
        Cell ekv = expected.getCells().get(j);
        assertEquals("KV mismatch for edit! Expected: "+expected+", but found: "+found, ekv, fkv);