/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package org.pentaho.di.starmodeler.generator;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.gui.Point;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.job.JobHopMeta;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.job.entries.sql.JobEntrySQL;
import org.pentaho.di.job.entry.JobEntryCopy;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.repository.RepositoryDirectoryInterface;
import org.pentaho.di.starmodeler.AttributeType;
import org.pentaho.di.starmodeler.ConceptUtil;
import org.pentaho.di.starmodeler.DefaultIDs;
import org.pentaho.di.starmodeler.DimensionType;
import org.pentaho.di.starmodeler.StarDomain;
import org.pentaho.di.trans.TransHopMeta;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.combinationlookup.CombinationLookupMeta;
import org.pentaho.di.trans.steps.dimensionlookup.DimensionLookupMeta;
import org.pentaho.di.trans.steps.tableinput.TableInputMeta;
import org.pentaho.di.trans.steps.tableoutput.TableOutputMeta;
import org.pentaho.di.ui.spoon.Spoon;
import org.pentaho.metadata.model.Domain;
import org.pentaho.metadata.model.LogicalColumn;
import org.pentaho.metadata.model.LogicalModel;
import org.pentaho.metadata.model.LogicalTable;
import org.pentaho.metadata.model.concept.types.DataType;
import org.pentaho.metadata.model.concept.types.TableType;

/**
 * The job generator creates a template job based on a star domain.
 * It creates one job to create all possible target dimensions, fact table, lookup indexes and so forth.
 *
 * @author matt
 *
 */
public class JobGenerator {
  private static Class<?> PKG = JobGenerator.class; // for i18n


  protected StarDomain starDomain;
  protected Repository repository;
  protected RepositoryDirectoryInterface targetDirectory;
  protected String locale;
  protected Domain domain;
  protected List<DatabaseMeta> databases;

  private static int GRAPH_MARGIN = 250;
  private static int GRAPH_TOP = 100;
  private static int GRAPH_LEFT   = 100;
  private static int GRAPH_MAX_WIDTH  = 1000;

  /**
   * @param starDomain
   * @param repository
   * @param targetDirectory
   * @param databases the list of shared database connections to reference for source and target databases.
   * @param locale
   */
  public JobGenerator(StarDomain starDomain, Repository repository, RepositoryDirectoryInterface targetDirectory, List<DatabaseMeta> databases, String locale) {
    this.starDomain = starDomain;
    this.repository = repository;
    this.targetDirectory = targetDirectory;
    this.databases = databases;
    this.locale = locale;

    this.domain = starDomain.getDomain();
  }

  public JobMeta generateSqlJob() throws KettleException {
    DatabaseMeta databaseMeta = findTargetDatabaseMeta();
    Database db = new Database(Spoon.loggingObject, databaseMeta);

    try {
      db.connect();

      JobMeta jobMeta = new JobMeta();

      jobMeta.setName("Create tables for '"+ConceptUtil.getName(domain, locale)+"'");
      jobMeta.setDescription(ConceptUtil.getDescription(domain, locale));

      // Let's not forget to add the database connection
      //
      jobMeta.addDatabase(databaseMeta);

      Point location = new Point(GRAPH_LEFT, GRAPH_TOP);

      // Create a job entry
      //
      JobEntryCopy startEntry = JobMeta.createStartEntry();
      startEntry.setLocation(location.x, location.y);
      startEntry.setDrawn();
      jobMeta.addJobEntry(startEntry);
      JobEntryCopy lastEntry = startEntry;
      nextLocation(location);

      // Create one SQL entry for all the physically unique dimensions and facts
      // We need to get a list of all known dimensions with physical table name.
      //
      List<LogicalTable> tables = getUniqueLogicalTables();
      for (LogicalTable logicalTable : tables) {
        String phTable = ConceptUtil.getString(logicalTable, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
        String tableName = ConceptUtil.getName(logicalTable, locale);
        String tableDescription = ConceptUtil.getDescription(logicalTable, locale);
        TableType tableType = ConceptUtil.getTableType(logicalTable);
        DimensionType dimensionType = ConceptUtil.getDimensionType(logicalTable);
        boolean isFact = tableType==TableType.FACT;
        boolean isDimension = tableType==TableType.DIMENSION;
        boolean isJunk = isDimension && dimensionType==DimensionType.JUNK_DIMENSION;

        JobEntrySQL sqlEntry = new JobEntrySQL(phTable);
        sqlEntry.setDatabase(databaseMeta);

        // Get the SQL for this table...
        //
        String schemaTable = databaseMeta.getQuotedSchemaTableCombination(null, phTable);
        String phKeyField = null;

        // The technical key is the first KEY field...
        //
        LogicalColumn keyColumn = null;
        if (isDimension) {
          keyColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.TECHNICAL_KEY);
        }
        if (keyColumn!=null) {
          phKeyField = ConceptUtil.getString(keyColumn, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
        }

        // Get all the fields for the logical table...
        //
        RowMetaInterface fields = getRowForLogicalTable(databaseMeta, logicalTable);

        // Generate the required SQL to make this happen
        //
        String sql = db.getCreateTableStatement(schemaTable, fields, phKeyField, databaseMeta.supportsAutoinc() && !isFact, null, true);

        // Also generate an index for the technical key field
        //
        if (keyColumn!=null) {
          ValueMetaInterface keyValueMeta = getValueForLogicalColumn(databaseMeta, keyColumn);
          String indexName = databaseMeta.quoteField( "IDX_" + phTable.replace(" ", "_").toUpperCase() + "_" + phKeyField.toUpperCase() );
          String indexSql = db.getCreateIndexStatement(schemaTable, indexName, new String[] { keyValueMeta.getName(), }, true, false, true, true);
          sql+=Const.CR+indexSql;
        }

        // In case it's a fact table generate an index for each TK column
        //
        if (isFact) {
          List<LogicalColumn> fks = ConceptUtil.findLogicalColumns(logicalTable, AttributeType.TECHNICAL_KEY);
          for (LogicalColumn fk : fks) {
            ValueMetaInterface keyValueMeta = getValueForLogicalColumn(databaseMeta, fk);
            String phColumn = ConceptUtil.getString(fk, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
            if (!Utils.isEmpty(phColumn)) {
              String indexName = databaseMeta.quoteField( "IDX_" + phTable.replace(" ", "_").toUpperCase() + "_" + phColumn.toUpperCase() );
              String indexSql = db.getCreateIndexStatement(schemaTable, indexName, new String[] { keyValueMeta.getName(), }, true, false, true, true);
              sql+=Const.CR+indexSql;
            }
          }
        }

        // Put an index on all natural keys too...
        //
        if (isDimension) {
          List<LogicalColumn> naturalKeys = ConceptUtil.findLogicalColumns(logicalTable, AttributeType.NATURAL_KEY);
          if (!naturalKeys.isEmpty()) {
            String indexName = databaseMeta.quoteField( "IDX_" + phTable.replace(" ", "_").toUpperCase() + "_LOOKUP" );
            String[] fieldNames = new String[naturalKeys.size()];
            for (int i=0;i<fieldNames.length;i++) {
              ValueMetaInterface keyValueMeta = getValueForLogicalColumn(databaseMeta, naturalKeys.get(i));
              fieldNames[i] = keyValueMeta.getName();
            }
            String indexSql = db.getCreateIndexStatement(schemaTable, indexName, fieldNames, false, false, false, true);
            sql+=Const.CR+indexSql;
          }
        }
        if (isJunk) {
          List<LogicalColumn> attributes = ConceptUtil.findLogicalColumns(logicalTable, AttributeType.ATTRIBUTE);
          if (!attributes.isEmpty()) {
            String indexName = databaseMeta.quoteField( "IDX_" + phTable.replace(" ", "_").toUpperCase() + "_LOOKUP" );
            String[] fieldNames = new String[attributes.size()];
            for (int i=0;i<fieldNames.length;i++) {
              ValueMetaInterface attrValueMeta = getValueForLogicalColumn(databaseMeta, attributes.get(i));
              fieldNames[i] = attrValueMeta.getName();
            }
            String indexSql = db.getCreateIndexStatement(schemaTable, indexName, fieldNames, false, false, false, true);
            sql+=Const.CR+indexSql;
          }
        }


        // If it's

        sqlEntry.setSQL(sql);

        sqlEntry.setDescription("Generated based on logical table '"+tableName+"'"+Const.CR+Const.CR+Const.NVL(tableDescription, ""));

        JobEntryCopy sqlCopy = new JobEntryCopy(sqlEntry);
        sqlCopy.setLocation(location.x, location.y);
        sqlCopy.setDrawn();
        nextLocation(location);
        jobMeta.addJobEntry(sqlCopy);

        // Hook up with the previous job entry too...
        //
        JobHopMeta jobHop = new JobHopMeta(lastEntry, sqlCopy);
        jobHop.setEnabled();
        jobHop.setConditional();
        jobHop.setEvaluation(true);
        if (lastEntry.isStart()) {
          jobHop.setUnconditional();
        }

        jobMeta.addJobHop(jobHop);
        lastEntry = sqlCopy;
      }

      return jobMeta;
    } catch(Exception e) {
      throw new KettleException("There was an error during the generation of the SQL job", e);
    } finally {
      if (db!=null) {
        db.disconnect();
      }
    }
  }

  protected DatabaseMeta findTargetDatabaseMeta() throws KettleException {

    String targetDbName = ConceptUtil.getString(starDomain.getDomain(), DefaultIDs.DOMAIN_TARGET_DATABASE);
    if (Utils.isEmpty(targetDbName)) {
      throw new KettleException(BaseMessages.getString(PKG, "LogicalModelerPerspective.MessageBox.NoTargetDBSpecified.Message"));
    }
    DatabaseMeta databaseMeta = DatabaseMeta.findDatabase(databases, targetDbName);
    if (databaseMeta==null) {
      throw new KettleException(BaseMessages.getString(PKG, "LogicalModelerPerspective.MessageBox.TargetDBNotFound.Message", targetDbName));
    }
    return databaseMeta;
  }

  protected DatabaseMeta findSourceDatabaseMeta(String databaseName) throws KettleException {

    DatabaseMeta databaseMeta = DatabaseMeta.findDatabase(databases, databaseName);
    if (databaseMeta==null) {
      throw new KettleException(BaseMessages.getString(PKG, "LogicalModelerPerspective.MessageBox.SourceDBNotFound.Message", databaseName));
    }
    return databaseMeta;
  }

  private RowMetaInterface getRowForLogicalTable(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
    RowMetaInterface fields = new RowMeta();
    for (LogicalColumn column : logicalTable.getLogicalColumns()) {
      ValueMetaInterface valueMeta = getValueForLogicalColumn(databaseMeta, column);
      fields.addValueMeta(valueMeta);
    }
    return fields;
  }

  private ValueMetaInterface getValueForLogicalColumn(DatabaseMeta databaseMeta, LogicalColumn column) {
    String columnName = ConceptUtil.getName(column, locale);
    String phColumnName = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
    DataType columnType = column.getDataType();
    String lengthString = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_LENGTH);
    int length = Const.toInt(lengthString, -1);
    String precisionString = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_PRECISION);
    int precision = Const.toInt(precisionString, -1);

    int type=ValueMetaInterface.TYPE_STRING;
    switch(columnType) {
    case UNKNOWN:
    case URL:
    case STRING: precision=-1; break;
    case IMAGE:
    case BINARY: type = ValueMetaInterface.TYPE_BINARY; precision=-1; break;
    case BOOLEAN: type = ValueMetaInterface.TYPE_BOOLEAN; length=-1; precision=-1; break;
    case DATE: type = ValueMetaInterface.TYPE_DATE; length=-1; precision=-1; break;
    case NUMERIC:
      if (precision<=0 && length<15) {
        type = ValueMetaInterface.TYPE_INTEGER;
      } else {
        if (length>=15) {
          type = ValueMetaInterface.TYPE_BIGNUMBER;
        } else {
          type = ValueMetaInterface.TYPE_NUMBER;
        }
      }
      break;
      default:
        break;
    }
    ValueMetaInterface value = new ValueMeta(databaseMeta.quoteField(Const.NVL(phColumnName, columnName)), type);
    value.setLength(length, precision);
    return value;
  }

  /**
   * Get a list of all unique physical table names wrapped in their logical tables
   * @return
   */
  protected List<LogicalTable> getUniqueLogicalTables() {
    List<LogicalTable> tables = new ArrayList<LogicalTable>();
    List<String> phTabs = new ArrayList<String>();
    for (LogicalModel model : domain.getLogicalModels()) {
      for (LogicalTable table : model.getLogicalTables()) {
        String phTable = ConceptUtil.getString(table, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
        if (!Utils.isEmpty(phTable)) {
          if (!phTabs.contains(phTable)) {
            phTabs.add(phTable);
            tables.add(table);
          }
        }
      }
    }

    return tables;
  }

  /**
   * Calculate the next location for a job entry to be placed.
   *
   * @param location
   */
  private void nextLocation(Point location) {
    location.x += GRAPH_MARGIN;
    if (location.x >= GRAPH_MAX_WIDTH) {
      location.x = GRAPH_LEFT;
      location.y += 150;
    }
  }

  /**
   * This method generates a list of transformations: one for each dimension.
   *
   * @return the list of generated transformations
   */
  public List<TransMeta> generateDimensionTransformations() throws KettleException {
    DatabaseMeta databaseMeta = findTargetDatabaseMeta();

    List<TransMeta> transMetas = new ArrayList<TransMeta>();

    List<LogicalTable> logicalTables = getUniqueLogicalTables();

    for (LogicalTable logicalTable : logicalTables) {
      TableType tableType = ConceptUtil.getTableType(logicalTable);
      DimensionType dimensionType = ConceptUtil.getDimensionType(logicalTable);
      if (tableType == TableType.DIMENSION) {
        switch(dimensionType) {
        case SLOWLY_CHANGING_DIMENSION:
        case JUNK_DIMENSION:
          {
            TransMeta transMeta = generateDimensionTransformation(databaseMeta, logicalTable);
            transMetas.add(transMeta);
          }
          break;
        case DATE: // TODO: generate a standard date transformation
          {
            TransMeta transMeta = generateDateTransformation(databaseMeta, logicalTable);
            transMetas.add(transMeta);
          }
          break;
        case TIME: // TODO: generate a standard time transformation
          {
            TransMeta transMeta = generateTimeTransformation(databaseMeta, logicalTable);
            transMetas.add(transMeta);
          }
          break;
        default:
          break;
        }
      }
    }

    return transMetas;
  }

  private TransMeta generateDateTransformation(DatabaseMeta databaseMeta, LogicalTable logicalTable) throws KettleException {
    // We actually load the transformation from a template and then slightly modify it.
    //
    String filename = "/org/pentaho/di/resources/Generate date dimension.ktr";
    InputStream inputStream = getClass().getResourceAsStream(filename);
    TransMeta transMeta = new TransMeta(inputStream, Spoon.getInstance().rep, true, new Variables(), null);

    // Find the table output step and inject the target table name and database...
    //
    StepMeta stepMeta = transMeta.findStep("TARGET");
    if (stepMeta!=null) {
      TableOutputMeta meta = (TableOutputMeta) stepMeta.getStepMetaInterface();
      meta.setDatabaseMeta(databaseMeta);
      String phTable = ConceptUtil.getString(logicalTable, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
      meta.setTableName(phTable);
    }

    return transMeta;
  }

  private TransMeta generateTimeTransformation(DatabaseMeta databaseMeta, LogicalTable logicalTable) throws KettleException {
    // We actually load the transformation from a template and then slightly modify it.
    //
    String filename = "/org/pentaho/di/resources/Generate time dimension.ktr";
    InputStream inputStream = getClass().getResourceAsStream(filename);
    TransMeta transMeta = new TransMeta(inputStream, Spoon.getInstance().rep, true, new Variables(), null);

    // Find the table output step and inject the target table name and database...
    //
    StepMeta stepMeta = transMeta.findStep("TARGET");
    if (stepMeta!=null) {
      TableOutputMeta meta = (TableOutputMeta) stepMeta.getStepMetaInterface();
      meta.setDatabaseMeta(databaseMeta);
      String phTable = ConceptUtil.getString(logicalTable, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
      meta.setTableName(phTable);
    }

    return transMeta;
  }

  /**
   * Generates a template
   * @param databaseMeta
   * @param logicalModel
   * @return
   */
  public TransMeta generateDimensionTransformation(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
    TransMeta transMeta = new TransMeta();

    String tableName = ConceptUtil.getName(logicalTable, locale);
    String tableDescription = ConceptUtil.getDescription(logicalTable, locale);
    DimensionType dimensionType = ConceptUtil.getDimensionType(logicalTable);

    transMeta.setName("Update dimension '"+tableName+"'");
    transMeta.setDescription(tableDescription);

    // Let's not forget to add the target database
    //
    transMeta.addDatabase(databaseMeta);

    Point location = new Point(GRAPH_LEFT, GRAPH_TOP);

    // Find all the source columns and source tables and put them into a table input step...
    //
    StepMeta inputStep = generateTableInputStepFromLogicalTable(logicalTable);
    DatabaseMeta sourceDatabaseMeta = ((TableInputMeta)inputStep.getStepMetaInterface()).getDatabaseMeta();
    if (sourceDatabaseMeta!=null) transMeta.addOrReplaceDatabase(sourceDatabaseMeta);
    inputStep.setLocation(location.x, location.y);
    nextLocation(location);
    transMeta.addStep(inputStep);
    StepMeta lastStep = inputStep;

    // Generate an dimension lookup/update step for each table
    //
    StepMeta dimensionStep;
    if (dimensionType==DimensionType.SLOWLY_CHANGING_DIMENSION) {
      dimensionStep = generateDimensionLookupStepFromLogicalTable(databaseMeta, logicalTable);
    } else {
      dimensionStep = generateCombinationLookupStepFromLogicalTable(databaseMeta, logicalTable);
    }
    dimensionStep.setLocation(location.x, location.y);
    nextLocation(location);
    transMeta.addStep(dimensionStep);

    TransHopMeta transHop = new TransHopMeta(lastStep, dimensionStep);
    transMeta.addTransHop(transHop);

    return transMeta;
  }

  private StepMeta generateTableInputStepFromLogicalTable(LogicalTable logicalTable) {

    String name = ConceptUtil.getName(logicalTable, locale);
    String description = ConceptUtil.getDescription(logicalTable, locale);

    TableInputMeta meta = new TableInputMeta();

    // Source database, retain first
    // Source table, retain first
    // Source columns, retain all
    //
    DatabaseMeta sourceDatabaseMeta = null;
    String sourceTable = null;
    List<String> sourceColumns = new ArrayList<String>();
    for (LogicalColumn column : logicalTable.getLogicalColumns()) {
      String phDb = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_SOURCE_DB);
      String phTable = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_SOURCE_TABLE);
      String phCol = ConceptUtil.getString(column, DefaultIDs.LOGICAL_COLUMN_SOURCE_COLUMN);
      if (!Utils.isEmpty(phDb) && sourceDatabaseMeta==null) {
        sourceDatabaseMeta = DatabaseMeta.findDatabase(databases, phDb);
      }
      if (!Utils.isEmpty(phTable)) {
        sourceTable = phDb;
      }
      if (!Utils.isEmpty(phCol)) {
        sourceColumns.add(phCol);
      }
    }
    String sql = "SELECT * FROM --< Source query for dimension '"+name+"'";

    meta.setDatabaseMeta(sourceDatabaseMeta);

    if (sourceDatabaseMeta!=null && !Utils.isEmpty(sourceTable)) {
      sql = "SELECT ";
      if (sourceColumns.isEmpty()) {
        sql+=" * ";
      } else {
        sql+=Const.CR;
      }
      boolean first=true;
      for (String sourceColumn : sourceColumns) {
        if (first) {
          first=false;
        } else {
          sql+="      , ";
        }
        sql+=sourceDatabaseMeta.quoteField(sourceColumn)+Const.CR;
      }
      sql+="FROM "+sourceDatabaseMeta.getQuotedSchemaTableCombination(null, sourceTable);
    }
    meta.setSQL(sql);

    // Wrap it up...
    //
    StepMeta stepMeta = new StepMeta("Source data for '"+name+"'", meta);
    stepMeta.drawStep();
    stepMeta.setDescription("Reads data for '"+name+"' : "+description);

    return stepMeta;
  }

  protected StepMeta generateDimensionLookupStepFromLogicalTable(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
    String name = ConceptUtil.getName(logicalTable, locale);
    String description = ConceptUtil.getDescription(logicalTable, locale);
    String phTable = ConceptUtil.getString(logicalTable, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
    String schemaTable = databaseMeta.getQuotedSchemaTableCombination(null, Const.NVL(phTable, name));

    DimensionLookupMeta meta = new DimensionLookupMeta();
    meta.setDatabaseMeta(databaseMeta);
    meta.setSchemaName(null); // TODO
    meta.setTableName(schemaTable);
    meta.setAutoIncrement(databaseMeta.supportsAutoinc());
    meta.setCacheSize(5000);
    meta.setCommitSize(500);
    meta.setUpdate(true);

    // Find the technical key (if any defined)
    //
    LogicalColumn keyColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.TECHNICAL_KEY);
    if (keyColumn!=null) {
      ValueMetaInterface keyValue = getValueForLogicalColumn(databaseMeta, keyColumn);
      meta.setKeyField(keyValue.getName());
    }

    // Simply add all the NATURAL_KEY columns...
    //
    List<LogicalColumn> naturalKeys = ConceptUtil.findLogicalColumns(logicalTable, AttributeType.NATURAL_KEY);
    meta.setKeyLookup(new String[naturalKeys.size()]);
    meta.setKeyStream(new String[naturalKeys.size()]);
    for (int i=0;i<naturalKeys.size();i++) {
      LogicalColumn logicalColumn = naturalKeys.get(i);
      ValueMetaInterface valueMeta = getValueForLogicalColumn(databaseMeta, logicalColumn);
      meta.getKeyLookup()[i] = valueMeta.getName();
      meta.getKeyStream()[i] = valueMeta.getName();
    }

    // All other attribute columns go in the fields tab
    //
    List<LogicalColumn> attributes = new ArrayList<LogicalColumn>();
    for (LogicalColumn logicalColumn : logicalTable.getLogicalColumns()) {
      AttributeType attributeType = ConceptUtil.getAttributeType(logicalColumn);
      if (attributeType.isAttribute()) {
          attributes.add(logicalColumn);
      }
    }
    meta.setFieldLookup(new String[attributes.size()]);
    meta.setFieldStream(new String[attributes.size()]);
    meta.setFieldUpdate(new int[attributes.size()]);
    for (int i=0;i<attributes.size();i++) {
      LogicalColumn logicalColumn = attributes.get(i);
      AttributeType attributeType = ConceptUtil.getAttributeType(logicalColumn);
      ValueMetaInterface valueMeta = getValueForLogicalColumn(databaseMeta, logicalColumn);
      meta.getFieldLookup()[i] = valueMeta.getName();
      meta.getFieldStream()[i] = valueMeta.getName();
      if (attributeType == AttributeType.ATTRIBUTE_OVERWRITE) {
        meta.getFieldUpdate()[i] = DimensionLookupMeta.TYPE_UPDATE_DIM_PUNCHTHROUGH;
      } else {
        // Historical or default: keep versions of the dimension records...
        //
        meta.getFieldUpdate()[i] = DimensionLookupMeta.TYPE_UPDATE_DIM_INSERT;
      }
    }

    // The version field...
    //
    LogicalColumn versionColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.VERSION_FIELD);
    if (versionColumn!=null) {
      String phName = ConceptUtil.getString(versionColumn, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
      meta.setVersionField(phName);
    }
    // Start of the date range
    //
    LogicalColumn startRangeColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.DATE_START);
    if (startRangeColumn!=null) {
      String phName = ConceptUtil.getString(startRangeColumn, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
      meta.setDateFrom(phName);
    }
    // End of the date range
    //
    LogicalColumn endRangeColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.DATE_END);
    if (endRangeColumn!=null) {
      String phName = ConceptUtil.getString(endRangeColumn, DefaultIDs.LOGICAL_COLUMN_PHYSICAL_COLUMN_NAME);
      meta.setDateTo(phName);
    }

    StepMeta stepMeta = new StepMeta(name, meta);
    stepMeta.drawStep();
    stepMeta.setDescription(description);

    return stepMeta;
  }

  protected StepMeta generateCombinationLookupStepFromLogicalTable(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
    String name = ConceptUtil.getName(logicalTable, locale);
    String description = ConceptUtil.getDescription(logicalTable, locale);
    String phTable = ConceptUtil.getString(logicalTable, DefaultIDs.LOGICAL_TABLE_PHYSICAL_TABLE_NAME);
    String schemaTable = databaseMeta.getQuotedSchemaTableCombination(null, Const.NVL(phTable, name));

    CombinationLookupMeta meta = new CombinationLookupMeta();
    meta.setDatabaseMeta(databaseMeta);
    meta.setSchemaName(null); // TODO
    meta.setTablename(schemaTable);
    meta.setUseAutoinc(databaseMeta.supportsAutoinc());
    meta.setCacheSize(5000);
    meta.setCommitSize(500);
    meta.setReplaceFields(true); // replace attribute fields with a TK

    // Find the technical key (if any defined)
    //
    LogicalColumn keyColumn = ConceptUtil.findLogicalColumn(logicalTable, AttributeType.TECHNICAL_KEY);
    if (keyColumn!=null) {
      ValueMetaInterface keyValue = getValueForLogicalColumn(databaseMeta, keyColumn);
      meta.setTechnicalKeyField(keyValue.getName());
    }

    // Simply add all the attributes as key columns...
    //
    List<LogicalColumn> attributes = ConceptUtil.findLogicalColumns(logicalTable, AttributeType.ATTRIBUTE);
    meta.setKeyLookup(new String[attributes.size()]);
    meta.setKeyField(new String[attributes.size()]);
    for (int i=0;i<attributes.size();i++) {
      LogicalColumn logicalColumn = attributes.get(i);
      ValueMetaInterface valueMeta = getValueForLogicalColumn(databaseMeta, logicalColumn);
      meta.getKeyLookup()[i] = valueMeta.getName();
      meta.getKeyField()[i] = valueMeta.getName();
    }

    StepMeta stepMeta = new StepMeta(name, meta);
    stepMeta.drawStep();
    stepMeta.setDescription(description);

    return stepMeta;
  }
}