@MetaStoreElementType(
  name = "Kettle Data Set",
  description = "This defines a Kettle data set" )
public class DataSet {

  private String name;

  @MetaStoreAttribute( key = "description" )
  private String description;

  @MetaStoreAttribute( key = "table_name" )
  private String tableName;

  @MetaStoreAttribute( key = "dataset_fields" )
  private List<DataSetField> fields;

  @MetaStoreAttribute( key = "group_name", nameReference = true, nameListKey = DataSetConst.GROUP_LIST_KEY )
  private DataSetGroup group;


  public DataSet() {
    fields = new ArrayList<>();
  }

  public DataSet( String name, String description, DataSetGroup group, String tableName, List<DataSetField> fields ) {
    this();
    this.name = name;
    this.description = description;
    this.group = group;
    this.tableName = tableName;
    this.fields = fields;
  }

  @Override
  public boolean equals( Object obj ) {
    if ( this == obj ) {
      return true;
    }
    if ( !( obj instanceof DataSet ) ) {
      return false;
    }
    DataSet cmp = (DataSet) obj;
    return name.equals( cmp );
  }

  @Override
  public int hashCode() {
    return name.hashCode();
  }


  /**
   * Get standard Kettle row metadata from the defined data set fields
   *
   * @param columnName true if you want the field names to be called after the columns, false if you prefer the field names in the result.
   * @return The row metadata
   * @throws KettlePluginException
   */
  public RowMetaInterface getSetRowMeta( boolean columnName ) throws KettlePluginException {
    RowMetaInterface rowMeta = new RowMeta();
    for ( DataSetField field : getFields() ) {
      ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta(
        columnName ? field.getColumnName() : field.getFieldName(),
        field.getType(),
        field.getLength(),
        field.getPrecision() );
      valueMeta.setComments( field.getComment() );
      valueMeta.setConversionMask( field.getFormat() );
      rowMeta.addValueMeta( valueMeta );
    }
    return rowMeta;
  }

  public DataSetField findFieldWithName( String fieldName ) {
    for ( DataSetField field : fields ) {
      if ( field.getFieldName().equalsIgnoreCase( fieldName ) ) {
        return field;
      }
    }
    return null;
  }

  public String findColumnForField( String dataSetFieldName ) {
    DataSetField field = findFieldWithName( dataSetFieldName );
    if ( field == null ) {
      return null;
    }
    return field.getColumnName();
  }

  public int indexOfField( String fieldName ) {
    for ( int i = 0; i < fields.size(); i++ ) {
      DataSetField field = fields.get( i );
      if ( field.getFieldName().equalsIgnoreCase( fieldName ) ) {
        return i;
      }
    }
    return -1;
  }


  public List<Object[]> getAllRows( LogChannelInterface log, TransUnitTestSetLocation location ) throws KettleException {
    return group.getAllRows( log, this, location );
  }

  public List<Object[]> getAllRows( LogChannelInterface log ) throws KettleException {
    return group.getAllRows( log, this );
  }


  /**
   * Calculate the row metadata for the data set fields needed for the given location.
   *
   * @param location
   * @return The fields metadata for those fields that are mapped against a certain step (location)
   */
  public RowMetaInterface getMappedDataSetFieldsRowMeta( TransUnitTestSetLocation location ) throws KettlePluginException {

    RowMetaInterface setRowMeta = getSetRowMeta( false );
    RowMetaInterface rowMeta = new RowMeta();
    for ( TransUnitTestFieldMapping fieldMapping : location.getFieldMappings() ) {
      ValueMetaInterface valueMeta = setRowMeta.searchValueMeta( fieldMapping.getDataSetFieldName() );
      rowMeta.addValueMeta( valueMeta );
    }
    return rowMeta;
  }


  /**
   * Gets name
   *
   * @return value of name
   */
  public String getName() {
    return name;
  }

  /**
   * @param name The name to set
   */
  public void setName( String name ) {
    this.name = name;
  }

  /**
   * Gets description
   *
   * @return value of description
   */
  public String getDescription() {
    return description;
  }

  /**
   * @param description The description to set
   */
  public void setDescription( String description ) {
    this.description = description;
  }

  /**
   * Gets tableName
   *
   * @return value of tableName
   */
  public String getTableName() {
    return tableName;
  }

  /**
   * @param tableName The tableName to set
   */
  public void setTableName( String tableName ) {
    this.tableName = tableName;
  }

  /**
   * Gets fields
   *
   * @return value of fields
   */
  public List<DataSetField> getFields() {
    return fields;
  }

  /**
   * @param fields The fields to set
   */
  public void setFields( List<DataSetField> fields ) {
    this.fields = fields;
  }

  /**
   * Gets group
   *
   * @return value of group
   */
  public DataSetGroup getGroup() {
    return group;
  }

  /**
   * @param group The group to set
   */
  public void setGroup( DataSetGroup group ) {
    this.group = group;
  }
} public class DataSetField {
  @MetaStoreAttribute( key = "field_name" )
  private String fieldName;

  @MetaStoreAttribute( key = "column_name" )
  private String columnName;

  @MetaStoreAttribute( key = "field_type" )
  private int type;

  @MetaStoreAttribute( key = "field_length" )
  private int length;

  @MetaStoreAttribute( key = "field_precision" )
  private int precision;

  @MetaStoreAttribute( key = "field_comment" )
  private String comment;

  @MetaStoreAttribute( key = "field_format" )
  private String format;

  public DataSetField() {
    // Empty constructor for MetaStoreFactory.
  }

  public DataSetField( String fieldName, String columnName, int type, int length, int precision, String comment, String format ) {
    super();
    this.fieldName = fieldName;
    this.columnName = columnName;
    this.type = type;
    this.length = length;
    this.precision = precision;
    this.comment = comment;
    this.format = format;
  }

  @Override public boolean equals( Object o ) {
    if ( this == o ) {
      return true;
    }
    if ( o == null || getClass() != o.getClass() ) {
      return false;
    }
    DataSetField that = (DataSetField) o;
    return Objects.equals( fieldName, that.fieldName );
  }

  @Override public int hashCode() {
    return Objects.hash( fieldName );
  }

  /**
   * Gets fieldName
   *
   * @return value of fieldName
   */
  public String getFieldName() {
    return fieldName;
  }

  /**
   * @param fieldName The fieldName to set
   */
  public void setFieldName( String fieldName ) {
    this.fieldName = fieldName;
  }

  /**
   * Gets columnName
   *
   * @return value of columnName
   */
  public String getColumnName() {
    return columnName;
  }

  /**
   * @param columnName The columnName to set
   */
  public void setColumnName( String columnName ) {
    this.columnName = columnName;
  }

  /**
   * Gets type
   *
   * @return value of type
   */
  public int getType() {
    return type;
  }

  /**
   * @param type The type to set
   */
  public void setType( int type ) {
    this.type = type;
  }

  /**
   * Gets length
   *
   * @return value of length
   */
  public int getLength() {
    return length;
  }

  /**
   * @param length The length to set
   */
  public void setLength( int length ) {
    this.length = length;
  }

  /**
   * Gets precision
   *
   * @return value of precision
   */
  public int getPrecision() {
    return precision;
  }

  /**
   * @param precision The precision to set
   */
  public void setPrecision( int precision ) {
    this.precision = precision;
  }

  /**
   * Gets comment
   *
   * @return value of comment
   */
  public String getComment() {
    return comment;
  }

  /**
   * @param comment The comment to set
   */
  public void setComment( String comment ) {
    this.comment = comment;
  }

  /**
   * Gets format
   *
   * @return value of format
   */
  public String getFormat() {
    return format;
  }

  /**
   * @param format The format to set
   */
  public void setFormat( String format ) {
    this.format = format;
  }
} @MetaStoreElementType( name = "Kettle Data Set Group", description = "Describes a data set group, the database where the sets are stored." )
public class DataSetGroup {

  private static Class<?> PKG = DataSetGroup.class; // for i18n purposes, needed by Translator2!!


  private String name;

  @MetaStoreAttribute( key = "group_type" )
  private DataSetGroupType type;

  @MetaStoreAttribute( key = "description" )
  private String description;

  @MetaStoreAttribute( key = "database_name", nameReference = true, nameListKey = DataSetConst.DATABASE_LIST_KEY )
  private DatabaseMeta databaseMeta;

  @MetaStoreAttribute( key = "schema_name" )
  private String schemaName;

  @MetaStoreAttribute
  private String folderName;

  public DataSetGroup() {
    // empty constructor for the IMetaStore factory
    //
    type = DataSetGroupType.Database;
  }

  public String getName() {
    return name;
  }

  public DataSetGroup( DataSetGroupType type, String name, String description, DatabaseMeta databaseMeta, String schemaName ) {
    this();
    this.type = type;
    this.name = name;
    this.description = description;
    this.databaseMeta = databaseMeta;
    this.schemaName = schemaName;
  }

  @Override
  public boolean equals( Object obj ) {

    if ( !( obj instanceof DataSetGroup ) ) {
      return false;
    }
    if ( this == obj ) {
      return true;
    }

    DataSetGroup cmp = (DataSetGroup) obj;

    return name.equals( cmp.name );
  }

  @Override
  public int hashCode() {
    return name.hashCode();
  }

  /**
   * Simply get all the rows without fuss in the order of the database.  Just for preview and some such.
   *
   * @param log
   * @param dataSet
   * @return The rows
   * @throws KettleException
   */
  public List<Object[]> getAllRows( LogChannelInterface log, DataSet dataSet ) throws KettleException {
    switch ( type ) {
      case Database:
        return DataSetDatabaseGroup.getAllRows( log, this, dataSet );
      case CSV:
        return DataSetCsvGroup.getAllRows( log, this, dataSet );
      default:
        throw new KettleException( type.name() + " : not supported yet" );
    }
  }

  /**
   * Get the dataset rows from the in a particular format in a particular order, determined by the given location
   *
   * @param log
   * @param dataSet
   * @param location
   * @return
   * @throws KettleException
   */
  public List<Object[]> getAllRows( LogChannelInterface log, DataSet dataSet, TransUnitTestSetLocation location ) throws KettleException {
    switch ( type ) {
      case Database:
        return DataSetDatabaseGroup.getAllRows( log, this, dataSet, location );
      case CSV:
        return DataSetCsvGroup.getAllRows( log, this, dataSet, location );
      default:
        throw new KettleException( type.name() + " : not supported yet" );
    }
  }

  public void writeDataSetData( String tableName, RowMetaInterface rowMeta, List<Object[]> dataRows ) throws KettleException {
    SimpleLoggingObject loggingObject = new SimpleLoggingObject( "Writing Data Set", LoggingObjectType.TRANS, null );

    switch ( type ) {
      case Database:
        DataSetDatabaseGroup.writeDataSetData( loggingObject, this, tableName, rowMeta, dataRows );
        break;
      case CSV:
        DataSetCsvGroup.writeDataSetData( loggingObject, this, tableName, rowMeta, dataRows );
        break;
      default:
        throw new KettleException( type.name() + " : not supported yet" );
    }
  }

  public void verifySettings() throws KettleException {
    if ( type == null ) {
      throw new KettleException( "No group type specified" );
    }
    switch ( type ) {
      case Database:
        if ( databaseMeta == null ) {
          throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.GroupHasNoDatabaseSpecified" ) );
        }
        break;
      case CSV:
        break;
    }
  }

  public void createTable( String tableName, RowMetaInterface rowMeta ) throws KettleException {
    switch ( type ) {
      case Database:
        DataSetDatabaseGroup.createTable( this, tableName, rowMeta );
        break;
      case CSV:
        DataSetCsvGroup.createTable( this, tableName, rowMeta );
        break;
      default:
        throw new KettleException( type.name() + " : not supported yet" );
    }

  }


  /**
   * @param name The name to set
   */
  public void setName( String name ) {
    this.name = name;
  }

  /**
   * Gets description
   *
   * @return value of description
   */
  public String getDescription() {
    return description;
  }

  /**
   * @param description The description to set
   */
  public void setDescription( String description ) {
    this.description = description;
  }

  /**
   * Gets databaseMeta
   *
   * @return value of databaseMeta
   */
  public DatabaseMeta getDatabaseMeta() {
    return databaseMeta;
  }

  /**
   * @param databaseMeta The databaseMeta to set
   */
  public void setDatabaseMeta( DatabaseMeta databaseMeta ) {
    this.databaseMeta = databaseMeta;
  }

  /**
   * Gets schemaName
   *
   * @return value of schemaName
   */
  public String getSchemaName() {
    return schemaName;
  }

  /**
   * @param schemaName The schemaName to set
   */
  public void setSchemaName( String schemaName ) {
    this.schemaName = schemaName;
  }

  /**
   * Gets type
   *
   * @return value of type
   */
  public DataSetGroupType getType() {
    return type;
  }

  /**
   * @param type The type to set
   */
  public void setType( DataSetGroupType type ) {
    this.type = type;
  }

  /**
   * Gets folderName
   *
   * @return value of folderName
   */
  public String getFolderName() {
    return folderName;
  }

  /**
   * @param folderName The folderName to set
   */
  public void setFolderName( String folderName ) {
    this.folderName = folderName;
  }
} public enum TransTweak {

  NONE, BYPASS_STEP, REMOVE_STEP; // , REMOVE_STEPS_AFTER, REMOVE_STEPS_BEFORE;

} @MetaStoreElementType(
  name = "Kettle Transformation Unit Test",
  description = "This describes a golden data unit test for a transformation with defined input data sets" )
public class TransUnitTest {

  private String name;

  @MetaStoreAttribute( key = "description" )
  private String description;

  @MetaStoreAttribute( key = "transformation_rep_object_id" )
  protected String transObjectId; // rep: by reference (1st priority)

  @MetaStoreAttribute( key = "transformation_rep_path" )
  protected String transRepositoryPath; // rep: by name (2nd priority)

  @MetaStoreAttribute( key = "transformation_filename" )
  protected String transFilename; // file (3rd priority)

  @MetaStoreAttribute( key = "input_data_sets" )
  protected List<TransUnitTestSetLocation> inputDataSets;

  @MetaStoreAttribute( key = "golden_data_sets" )
  protected List<TransUnitTestSetLocation> goldenDataSets;

  @MetaStoreAttribute( key = "trans_test_tweaks" )
  protected List<TransUnitTestTweak> tweaks;

  @MetaStoreAttribute( key = "test_type" )
  protected TestType type;

  @MetaStoreAttribute( key = "persist_filename" )
  protected String filename;

  @MetaStoreAttribute
  protected String basePath;

  @MetaStoreAttribute( key = "database_replacements" )
  protected List<TransUnitTestDatabaseReplacement> databaseReplacements;

  @MetaStoreAttribute
  protected List<VariableValue> variableValues;

  @MetaStoreAttribute
  protected boolean autoOpening;


  public TransUnitTest() {
    inputDataSets = new ArrayList<TransUnitTestSetLocation>();
    goldenDataSets = new ArrayList<TransUnitTestSetLocation>();
    tweaks = new ArrayList<TransUnitTestTweak>();
    type = TestType.NONE;
    databaseReplacements = new ArrayList<TransUnitTestDatabaseReplacement>();
    variableValues = new ArrayList<>();
    basePath = null;
    autoOpening = false;
  }

  public TransUnitTest( String name, String description,
                        String transObjectId, String transRepositoryPath, String transFilename,
                        List<TransUnitTestSetLocation> inputDataSets,
                        List<TransUnitTestSetLocation> goldenDataSets,
                        List<TransUnitTestTweak> tweaks,
                        TestType type,
                        String filename,
                        List<TransUnitTestDatabaseReplacement> databaseReplacements,
                        boolean autoOpening ) {
    this();
    this.name = name;
    this.description = description;
    this.transObjectId = transObjectId;
    this.transRepositoryPath = transRepositoryPath;
    this.transFilename = transFilename;
    this.inputDataSets = inputDataSets;
    this.goldenDataSets = goldenDataSets;
    this.tweaks = tweaks;
    this.type = type;
    this.filename = filename;
    this.databaseReplacements = databaseReplacements;
    this.autoOpening = autoOpening;
  }

  @Override
  public boolean equals( Object obj ) {
    if ( obj == this ) {
      return true;
    }
    if ( !( obj instanceof TransUnitTest ) ) {
      return false;
    }
    return ( (TransUnitTest) obj ).name.equalsIgnoreCase( name );
  }

  @Override
  public int hashCode() {
    return name.hashCode();
  }


  public TransUnitTestSetLocation findGoldenLocation( String stepName ) {
    for ( TransUnitTestSetLocation location : goldenDataSets ) {
      if ( stepName.equalsIgnoreCase( location.getStepname() ) ) {
        return location;
      }
    }
    return null;
  }

  public TransUnitTestSetLocation findInputLocation( String stepName ) {
    for ( TransUnitTestSetLocation location : inputDataSets ) {
      if ( stepName.equalsIgnoreCase( location.getStepname() ) ) {
        return location;
      }
    }
    return null;
  }

  /**
   * Retrieve the golden data set for the specified location
   *
   * @param log       the logging channel to log to
   * @param hierarchy The factories to load sets with
   * @param location  the location where we want to check against golden rows
   * @return The golden data set
   * @throws KettleException
   */
  public DataSet getGoldenDataSet( LogChannelInterface log, FactoriesHierarchy hierarchy, TransUnitTestSetLocation location ) throws KettleException {

    String stepName = location.getStepname();
    String goldenDataSetName = location.getDataSetName();

    try {
      // Look in the golden data sets list for the mentioned step name
      //
      if ( goldenDataSetName == null ) {
        throw new KettleException( "Unable to find golden data set for step '" + stepName + "'" );
      }

      DataSet goldenDataSet = hierarchy.getSetFactory().loadElement( goldenDataSetName );
      if ( goldenDataSet == null ) {
        throw new KettleException( "Unable to find golden data set '" + goldenDataSetName + "' for step '" + stepName + "'" );
      }

      return goldenDataSet;

    } catch ( Exception e ) {
      throw new KettleException( "Unable to retrieve sorted golden row data set '" + stepName + "'", e );
    }
  }

  /**
   * Find the first tweak for a certain step
   *
   * @param stepname the name of the step on which a tweak is put
   * @return the first tweak for a certain step or null if nothing was found
   */
  public TransUnitTestTweak findTweak( String stepname ) {
    for ( TransUnitTestTweak tweak : tweaks ) {
      if ( tweak.getStepName() != null && tweak.getStepName().equalsIgnoreCase( stepname ) ) {
        return tweak;
      }
    }
    return null;
  }

  /**
   * Remove all input and golden data sets on the step with the provided name
   *
   * @param stepname the name of the step for which we need to clear out all input and golden data sets
   */
  public void removeInputAndGoldenDataSets( String stepname ) {

    for ( Iterator<TransUnitTestSetLocation> iterator = inputDataSets.iterator(); iterator.hasNext(); ) {
      TransUnitTestSetLocation inputLocation = iterator.next();
      if ( inputLocation.getStepname().equalsIgnoreCase( stepname ) ) {
        iterator.remove();
      }
    }

    for ( Iterator<TransUnitTestSetLocation> iterator = goldenDataSets.iterator(); iterator.hasNext(); ) {
      TransUnitTestSetLocation goldenLocation = iterator.next();
      if ( goldenLocation.getStepname().equalsIgnoreCase( stepname ) ) {
        iterator.remove();
      }
    }
  }

  public String calculateCompleteFilename( VariableSpace space ) {

    String baseFilePath = space.environmentSubstitute( basePath );
    if ( StringUtils.isEmpty( baseFilePath ) ) {
      // See if the base path environment variable is set
      //
      baseFilePath = space.getVariable( DataSetConst.VARIABLE_UNIT_TESTS_BASE_PATH );
    }
    if ( StringUtils.isEmpty( baseFilePath ) ) {
      baseFilePath = "";
    }
    if ( StringUtils.isNotEmpty( baseFilePath ) ) {
      if ( !baseFilePath.endsWith( File.separator ) ) {
        baseFilePath += File.separator;
      }
    }
    return baseFilePath + transFilename;
  }

  /**
   * Gets name
   *
   * @return value of name
   */
  public String getName() {
    return name;
  }

  /**
   * @param name The name to set
   */
  public void setName( String name ) {
    this.name = name;
  }

  /**
   * Gets description
   *
   * @return value of description
   */
  public String getDescription() {
    return description;
  }

  /**
   * @param description The description to set
   */
  public void setDescription( String description ) {
    this.description = description;
  }

  /**
   * Gets transObjectId
   *
   * @return value of transObjectId
   */
  public String getTransObjectId() {
    return transObjectId;
  }

  /**
   * @param transObjectId The transObjectId to set
   */
  public void setTransObjectId( String transObjectId ) {
    this.transObjectId = transObjectId;
  }

  /**
   * Gets transRepositoryPath
   *
   * @return value of transRepositoryPath
   */
  public String getTransRepositoryPath() {
    return transRepositoryPath;
  }

  /**
   * @param transRepositoryPath The transRepositoryPath to set
   */
  public void setTransRepositoryPath( String transRepositoryPath ) {
    this.transRepositoryPath = transRepositoryPath;
  }

  /**
   * Gets transFilename
   *
   * @return value of transFilename
   */
  public String getTransFilename() {
    return transFilename;
  }

  /**
   * @param transFilename The transFilename to set
   */
  public void setTransFilename( String transFilename ) {
    this.transFilename = transFilename;
  }

  /**
   * Gets inputDataSets
   *
   * @return value of inputDataSets
   */
  public List<TransUnitTestSetLocation> getInputDataSets() {
    return inputDataSets;
  }

  /**
   * @param inputDataSets The inputDataSets to set
   */
  public void setInputDataSets( List<TransUnitTestSetLocation> inputDataSets ) {
    this.inputDataSets = inputDataSets;
  }

  /**
   * Gets goldenDataSets
   *
   * @return value of goldenDataSets
   */
  public List<TransUnitTestSetLocation> getGoldenDataSets() {
    return goldenDataSets;
  }

  /**
   * @param goldenDataSets The goldenDataSets to set
   */
  public void setGoldenDataSets( List<TransUnitTestSetLocation> goldenDataSets ) {
    this.goldenDataSets = goldenDataSets;
  }

  /**
   * Gets tweaks
   *
   * @return value of tweaks
   */
  public List<TransUnitTestTweak> getTweaks() {
    return tweaks;
  }

  /**
   * @param tweaks The tweaks to set
   */
  public void setTweaks( List<TransUnitTestTweak> tweaks ) {
    this.tweaks = tweaks;
  }

  /**
   * Gets type
   *
   * @return value of type
   */
  public TestType getType() {
    return type;
  }

  /**
   * @param type The type to set
   */
  public void setType( TestType type ) {
    this.type = type;
  }

  /**
   * Gets filename
   *
   * @return value of filename
   */
  public String getFilename() {
    return filename;
  }

  /**
   * @param filename The filename to set
   */
  public void setFilename( String filename ) {
    this.filename = filename;
  }

  /**
   * Gets basePath
   *
   * @return value of basePath
   */
  public String getBasePath() {
    return basePath;
  }

  /**
   * @param basePath The basePath to set
   */
  public void setBasePath( String basePath ) {
    this.basePath = basePath;
  }

  /**
   * Gets databaseReplacements
   *
   * @return value of databaseReplacements
   */
  public List<TransUnitTestDatabaseReplacement> getDatabaseReplacements() {
    return databaseReplacements;
  }

  /**
   * @param databaseReplacements The databaseReplacements to set
   */
  public void setDatabaseReplacements( List<TransUnitTestDatabaseReplacement> databaseReplacements ) {
    this.databaseReplacements = databaseReplacements;
  }

  /**
   * Gets variableValues
   *
   * @return value of variableValues
   */
  public List<VariableValue> getVariableValues() {
    return variableValues;
  }

  /**
   * @param variableValues The variableValues to set
   */
  public void setVariableValues( List<VariableValue> variableValues ) {
    this.variableValues = variableValues;
  }

  /**
   * Gets autoOpening
   *
   * @return value of autoOpening
   */
  public boolean isAutoOpening() {
    return autoOpening;
  }

  /**
   * @param autoOpening The autoOpening to set
   */
  public void setAutoOpening( boolean autoOpening ) {
    this.autoOpening = autoOpening;
  }
} public class TransUnitTestFieldMapping {

  @MetaStoreAttribute( key = "step_field" )
  private String stepFieldName;

  @MetaStoreAttribute( key = "data_set_field" )
  private String dataSetFieldName;

  public TransUnitTestFieldMapping() {
  }

  public TransUnitTestFieldMapping( String stepFieldName, String dataSetFieldName ) {
    this();
    this.stepFieldName = stepFieldName;
    this.dataSetFieldName = dataSetFieldName;
  }

  public String getStepFieldName() {
    return stepFieldName;
  }

  public void setStepFieldName( String stepFieldName ) {
    this.stepFieldName = stepFieldName;
  }

  public String getDataSetFieldName() {
    return dataSetFieldName;
  }

  public void setDataSetFieldName( String dataSetFieldName ) {
    this.dataSetFieldName = dataSetFieldName;
  }
} public class TransUnitTestSetLocation {

  @MetaStoreAttribute( key = "step_name" )
  protected String stepname;

  @MetaStoreAttribute( key = "data_set_name" )
  protected String dataSetName;

  @MetaStoreAttribute( key = "field_mappings" )
  protected List<TransUnitTestFieldMapping> fieldMappings;

  @MetaStoreAttribute( key = "field_order" )
  protected List<String> fieldOrder;

  public TransUnitTestSetLocation() {
    fieldMappings = new ArrayList<TransUnitTestFieldMapping>();
    fieldOrder = new ArrayList<String>();
  }

  public TransUnitTestSetLocation( String stepname, String dataSetName, List<TransUnitTestFieldMapping> fieldMappings, List<String> fieldOrder ) {
    this();
    this.stepname = stepname;
    this.dataSetName = dataSetName;
    this.fieldMappings = fieldMappings;
    this.fieldOrder = fieldOrder;
  }

  public String findStepField( String dataSetField ) {
    for ( TransUnitTestFieldMapping fieldMapping : fieldMappings ) {
      if ( fieldMapping.getDataSetFieldName().equalsIgnoreCase( dataSetField ) ) {
        return fieldMapping.getStepFieldName();
      }
    }
    return null;
  }

  public String getStepname() {
    return stepname;
  }

  public void setStepname( String stepname ) {
    this.stepname = stepname;
  }

  public String getDataSetName() {
    return dataSetName;
  }

  public void setDataSetName( String dataSetName ) {
    this.dataSetName = dataSetName;
  }

  public List<TransUnitTestFieldMapping> getFieldMappings() {
    return fieldMappings;
  }

  public void setFieldMappings( List<TransUnitTestFieldMapping> fieldMappings ) {
    this.fieldMappings = fieldMappings;
  }

  public List<String> getFieldOrder() {
    return fieldOrder;
  }

  public void setFieldOrder( List<String> fieldOrder ) {
    this.fieldOrder = fieldOrder;
  }

} public class TransUnitTestTweak {

  @MetaStoreAttribute
  private TransTweak tweak;

  @MetaStoreAttribute
  private String stepName;

  public TransUnitTestTweak() {
    tweak = TransTweak.NONE;
  }

  public TransUnitTestTweak( TransTweak tweak, String stepName ) {
    super();
    this.tweak = tweak;
    this.stepName = stepName;
  }

  @Override
  public boolean equals( Object obj ) {
    if ( !( obj instanceof TransUnitTestTweak ) ) {
      return false;
    }
    if ( obj == this ) {
      return true;
    }

    TransUnitTestTweak other = (TransUnitTestTweak) obj;

    return stepName == null ? false : stepName.equals( other.stepName );
  }

  @Override
  public int hashCode() {
    return stepName == null ? 0 : stepName.hashCode();
  }

  public TransTweak getTweak() {
    return tweak;
  }

  public void setTweak( TransTweak tweak ) {
    this.tweak = tweak;
  }

  public String getStepName() {
    return stepName;
  }

  public void setStepName( String stepName ) {
    this.stepName = stepName;
  }
} public class DataSetDialog extends Dialog {
  private static Class<?> PKG = DataSetDialog.class; // for i18n purposes, needed by Translator2!!

  private DataSet dataSet;

  private List<DataSetGroup> groups;

  private Shell shell;

  private Text wName;
  private Text wDescription;
  private Text wTableName;
  private Combo wDataSetGroup;
  private TableView wFieldMapping;

  private Button wOK;
  private Button wGetMetadata;
  private Button wCreateTable;
  private Button wEditData;
  private Button wViewData;
  private Button wCancel;

  private Button wNewGroup;

  private PropsUI props;

  private int middle;
  private int margin;

  private boolean ok;

  private IMetaStore metaStore;

  private Button wEditGroup;

  private List<DatabaseMeta> databases;

  public DataSetDialog( Shell parent, IMetaStore metaStore, DataSet dataSet, List<DataSetGroup> groups, List<DatabaseMeta> databases ) {
    super( parent, SWT.NONE );
    this.metaStore = metaStore;
    this.dataSet = dataSet;
    this.groups = groups;
    this.databases = databases;
    props = PropsUI.getInstance();
    ok = false;
  }

  public boolean open() {
    Shell parent = getParent();
    shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN );
    props.setLook( shell );
    shell.setImage( GUIResource.getInstance().getImageTable() );

    middle = props.getMiddlePct();
    margin = Const.MARGIN;

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setText( BaseMessages.getString( PKG, "DataSetDialog.Shell.Title" ) );
    shell.setLayout( formLayout );

    // The name of the group...
    //
    Label wlName = new Label( shell, SWT.RIGHT );
    props.setLook( wlName );
    wlName.setText( BaseMessages.getString( PKG, "DataSetDialog.Name.Label" ) );
    FormData fdlName = new FormData();
    fdlName.top = new FormAttachment( 0, 0 );
    fdlName.left = new FormAttachment( 0, 0 );
    fdlName.right = new FormAttachment( middle, -margin );
    wlName.setLayoutData( fdlName );
    wName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wName );
    FormData fdName = new FormData();
    fdName.top = new FormAttachment( 0, 0 );
    fdName.left = new FormAttachment( middle, 0 );
    fdName.right = new FormAttachment( 100, 0 );
    wName.setLayoutData( fdName );
    Control lastControl = wName;

    // The description of the group...
    //
    Label wlDescription = new Label( shell, SWT.RIGHT );
    props.setLook( wlDescription );
    wlDescription.setText( BaseMessages.getString( PKG, "DataSetDialog.Description.Label" ) );
    FormData fdlDescription = new FormData();
    fdlDescription.top = new FormAttachment( lastControl, margin );
    fdlDescription.left = new FormAttachment( 0, 0 );
    fdlDescription.right = new FormAttachment( middle, -margin );
    wlDescription.setLayoutData( fdlDescription );
    wDescription = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wDescription );
    FormData fdDescription = new FormData();
    fdDescription.top = new FormAttachment( lastControl, margin );
    fdDescription.left = new FormAttachment( middle, 0 );
    fdDescription.right = new FormAttachment( 100, 0 );
    wDescription.setLayoutData( fdDescription );
    lastControl = wDescription;

    // The table storing the set...
    //
    Label wlTableName = new Label( shell, SWT.RIGHT );
    props.setLook( wlTableName );
    wlTableName.setText( BaseMessages.getString( PKG, "DataSetDialog.TableName.Label" ) );
    FormData fdlTableName = new FormData();
    fdlTableName.top = new FormAttachment( lastControl, margin );
    fdlTableName.left = new FormAttachment( 0, 0 );
    fdlTableName.right = new FormAttachment( middle, -margin );
    wlTableName.setLayoutData( fdlTableName );
    wTableName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wTableName );
    FormData fdTableName = new FormData();
    fdTableName.top = new FormAttachment( lastControl, margin );
    fdTableName.left = new FormAttachment( middle, 0 );
    fdTableName.right = new FormAttachment( 100, 0 );
    wTableName.setLayoutData( fdTableName );
    lastControl = wTableName;

    // The data set group...
    //
    Label wlGroup = new Label( shell, SWT.RIGHT );
    props.setLook( wlGroup );
    wlGroup.setText( BaseMessages.getString( PKG, "DataSetDialog.Group.Label" ) );
    FormData fdlDatabase = new FormData();
    fdlDatabase.top = new FormAttachment( lastControl, margin );
    fdlDatabase.left = new FormAttachment( 0, 0 );
    fdlDatabase.right = new FormAttachment( middle, -margin );
    wlGroup.setLayoutData( fdlDatabase );

    wNewGroup = new Button( shell, SWT.PUSH );
    props.setLook( wNewGroup );
    wNewGroup.setText( BaseMessages.getString( PKG, "DataSetDialog.NewGroup.Label" ) );
    FormData fdlNewGroup = new FormData();
    fdlNewGroup.top = new FormAttachment( lastControl, margin );
    fdlNewGroup.right = new FormAttachment( 100, -margin );
    wNewGroup.setLayoutData( fdlNewGroup );
    wNewGroup.addSelectionListener( new SelectionAdapter() {
      @Override
      public void widgetSelected( SelectionEvent arg0 ) {
        addDataSetGroup();
      }
    } );

    wEditGroup = new Button( shell, SWT.PUSH );
    props.setLook( wEditGroup );
    wEditGroup.setText( BaseMessages.getString( PKG, "DataSetDialog.EditGroup.Label" ) );
    FormData fdlEditGroup = new FormData();
    fdlEditGroup.top = new FormAttachment( lastControl, margin );
    fdlEditGroup.right = new FormAttachment( wNewGroup, -margin );
    wEditGroup.setLayoutData( fdlEditGroup );
    wEditGroup.addSelectionListener( new SelectionAdapter() {
      @Override
      public void widgetSelected( SelectionEvent arg0 ) {
        editDataSetGroup();
      }
    } );

    wDataSetGroup = new Combo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    FormData fdDatabase = new FormData();
    fdDatabase.top = new FormAttachment( lastControl, margin );
    fdDatabase.left = new FormAttachment( middle, 0 );
    fdDatabase.right = new FormAttachment( wEditGroup, -margin );
    wDataSetGroup.setLayoutData( fdDatabase );
    lastControl = wDataSetGroup;

    // The field mapping from the input to the data set...
    //
    Label wlFieldMapping = new Label( shell, SWT.NONE );
    wlFieldMapping.setText( BaseMessages.getString( PKG, "DataSetDialog.FieldMapping.Label" ) );
    props.setLook( wlFieldMapping );
    FormData fdlUpIns = new FormData();
    fdlUpIns.left = new FormAttachment( 0, 0 );
    fdlUpIns.top = new FormAttachment( lastControl, margin * 2 );
    wlFieldMapping.setLayoutData( fdlUpIns );
    lastControl = wlFieldMapping;

    // Buttons at the bottom...
    //
    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) );

    wGetMetadata = new Button( shell, SWT.PUSH );
    wGetMetadata.setText( BaseMessages.getString( PKG, "DataSetDialog.GetMetadata.Button" ) );

    wCreateTable = new Button( shell, SWT.PUSH );
    wCreateTable.setText( BaseMessages.getString( PKG, "DataSetDialog.CreateTable.Button" ) );

    wEditData = new Button( shell, SWT.PUSH );
    wEditData.setText( BaseMessages.getString( PKG, "DataSetDialog.EditData.Button" ) );

    wViewData = new Button( shell, SWT.PUSH );
    wViewData.setText( BaseMessages.getString( PKG, "DataSetDialog.ViewData.Button" ) );

    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) );

    Button[] buttons = new Button[] { wOK, wGetMetadata, wCreateTable, wEditData, wViewData, wCancel };
    BaseStepDialog.positionBottomButtons( shell, buttons, margin, null );

    // the field mapping grid in between
    //
    ColumnInfo[] columns = new ColumnInfo[] {
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldName" ),
        ColumnInfo.COLUMN_TYPE_CCOMBO, new String[] { "" }, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.ColumnName" ),
        ColumnInfo.COLUMN_TYPE_CCOMBO, new String[] { "" }, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldType" ),
        ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMetaFactory.getAllValueMetaNames(), false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldFormat" ),
        ColumnInfo.COLUMN_TYPE_TEXT, true, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldLength" ),
        ColumnInfo.COLUMN_TYPE_TEXT, true, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldPrecision" ),
        ColumnInfo.COLUMN_TYPE_TEXT, true, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.Comment" ),
        ColumnInfo.COLUMN_TYPE_TEXT, false, false ),
    };

    wFieldMapping = new TableView(
      new Variables(),
      shell,
      SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL,
      columns,
      dataSet.getFields().size(),
      null, props );

    FormData fdFieldMapping = new FormData();
    fdFieldMapping.left = new FormAttachment( 0, 0 );
    fdFieldMapping.top = new FormAttachment( lastControl, margin );
    fdFieldMapping.right = new FormAttachment( 100, 0 );
    fdFieldMapping.bottom = new FormAttachment( wOK, -2 * margin );
    wFieldMapping.setLayoutData( fdFieldMapping );

    // Add listeners
    wOK.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        ok();
      }
    } );
    wGetMetadata.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        getMetadataFromTable();
      }
    } );
    wCreateTable.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        createTable();
      }
    } );
    wEditData.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        editData();
      }
    } );
    wViewData.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        viewData();
      }
    } );
    wCancel.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        cancel();
      }
    } );

    SelectionAdapter selAdapter = new SelectionAdapter() {
      public void widgetDefaultSelected( SelectionEvent e ) {
        ok();
      }
    };
    wName.addSelectionListener( selAdapter );
    wDescription.addSelectionListener( selAdapter );
    wTableName.addSelectionListener( selAdapter );

    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener( new ShellAdapter() {
      public void shellClosed( ShellEvent e ) {
        cancel();
      }
    } );

    getData();

    BaseStepDialog.setSize( shell );

    shell.open();
    Display display = parent.getDisplay();
    while ( !shell.isDisposed() ) {
      if ( !display.readAndDispatch() ) {
        display.sleep();
      }
    }
    return ok;
  }

  protected void addDataSetGroup() {

    try {
      DataSetHelper.getInstance().addDataSetGroup();
      refreshGroups( metaStore );
    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error creating new data set group", e );
    }
  }

  protected void editDataSetGroup() {

    String groupName = wDataSetGroup.getText();
    if ( StringUtils.isEmpty( groupName ) ) {
      return;
    }

    try {
      DataSetHelper.getInstance().editDataSetGroup( groupName );
      refreshGroups( metaStore );
    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error editing data set group", e );
    }
  }

  private void refreshGroups( IMetaStore metaStore ) throws MetaStoreException {

    MetaStoreFactory<DataSetGroup> factory = new MetaStoreFactory<DataSetGroup>( DataSetGroup.class, metaStore, PentahoDefaults.NAMESPACE );
    factory.addNameList( DataSetConst.DATABASE_LIST_KEY, databases );

    groups = factory.getElements();

    List<String> names = factory.getElementNames();
    Collections.sort( names );
    wDataSetGroup.setItems( names.toArray( new String[ names.size() ] ) );

  }

  /**
   * Look at the metadata specified here and create the table accordingly...
   */
  protected void getMetadataFromTable() {
    try {
      verifySettings();

      DataSet set = new DataSet();
      getInfo( set );
      DataSetGroup group = set.getGroup();

      if ( group.getType() != DataSetGroupType.Database ) {
        return;
      }
      group.verifySettings();

      String schemaTable = group.getDatabaseMeta().getQuotedSchemaTableCombination( group.getSchemaName(), wTableName.getText() );

      Database database = null;
      try {
        database = new Database( new LoggingObject( "DataSetDialog" ), group.getDatabaseMeta() );
        database.connect();
        if ( database.checkTableExists( schemaTable ) ) {
          RowMetaInterface rowMeta = database.getTableFields( schemaTable );

          BaseStepDialog.getFieldsFromPrevious( rowMeta, wFieldMapping, 2, new int[] { 1, 2 }, new int[] { 3 }, 4, 5, new TableItemInsertListener() {
            public boolean tableItemInserted( TableItem tableItem, ValueMetaInterface v ) {

              tableItem.setText( 4, v.getLength() < 0 ? "" : Integer.toString( v.getLength() ) );
              tableItem.setText( 5, v.getPrecision() < 0 ? "" : Integer.toString( v.getPrecision() ) );

              return true;
            }
          } );

        } else {
          throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.TableDoesNotExist", schemaTable ) );
        }

      } finally {
        if ( database != null ) {
          database.disconnect();
        }
      }

    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error retrieving metadata from dataset table", e );
    }
  }

  private void verifySettings() throws KettleException {
    DataSetGroup group = DataSetConst.findDataSetGroup( groups, wDataSetGroup.getText() );

    if ( group == null ) {
      throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.NoGroupSpecified" ) );
    }
    if ( StringUtil.isEmpty( wTableName.getText() ) ) {
      throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.NoTableSpecified" ) );
    }
    group.verifySettings();

  }

  /**
   * Look at the metadata specified here and create the table accordingly...
   * - Create a table for the Database Group type
   * - Create a metadata file for the CSV group type
   */
  protected void createTable() {
    try {
      verifySettings();

      DataSet set = new DataSet();
      getInfo( set );
      DataSetGroup group = set.getGroup();

      String tableName = wTableName.getText();

      // Calculate the row metadata of the table
      //
      RowMetaInterface rowMeta = new RowMeta();
      int nrFields = wFieldMapping.nrNonEmpty();
      for ( int i = 0; i < nrFields; i++ ) {
        TableItem item = wFieldMapping.getNonEmpty( i );
        int colnr = 2;
        String columnName = item.getText( colnr++ );
        if ( StringUtil.isEmpty( columnName ) ) {
          throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.NoColumnName", Integer.toString( i ) ) );
        }
        int fieldType = ValueMetaFactory.getIdForValueMeta( item.getText( colnr++ ) );
        if ( fieldType == ValueMetaInterface.TYPE_NONE ) {
          throw new KettleException( BaseMessages.getString( PKG, "DataSetDialog.Error.NoDataType", Integer.toString( i ) ) );
        }
        int length = Const.toInt( item.getText( colnr++ ), -1 );
        int precision = Const.toInt( item.getText( colnr++ ), -1 );

        ValueMetaInterface valueMeta = ValueMetaFactory.createValueMeta( columnName, fieldType, length, precision );
        rowMeta.addValueMeta( valueMeta );
      }

      group.createTable( tableName, rowMeta );

    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error retrieving metadata from dataset table", e );
    }

  }

  protected void editData() {

    // If the row count is too high, we don't want to load it into memory...
    // Too high simply means: above the preview size...
    //
    int previewSize = props.getDefaultPreviewSize();
    try {

      verifySettings();

      DataSet set = new DataSet();
      getInfo( set );
      DataSetGroup group = set.getGroup();

      // get rows from the data set...
      //
      List<Object[]> rows = set.getAllRows( LogChannel.UI );

      RowMetaInterface fieldsRowMeta = set.getSetRowMeta( false );
      RowMetaInterface columnsRowMeta = set.getSetRowMeta( true );

      EditRowsDialog editRowsDialog = new EditRowsDialog( shell, SWT.NONE,
        BaseMessages.getString( PKG, "DataSetDialog.EditRows.Title" ),
        BaseMessages.getString( PKG, "DataSetDialog.EditRows.Message", set.getName() ),
        fieldsRowMeta,
        rows );
      List<Object[]> newList = editRowsDialog.open();
      if ( newList != null ) {
        // Write the rows back to the data set
        //
        group.writeDataSetData( set.getTableName(), columnsRowMeta, newList );
      }

    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error previewing data from dataset table", e );
    }
  }

  protected void viewData() {
    try {
      DataSet set = new DataSet();
      getInfo( set );
      verifySettings();

      List<Object[]> setRows = set.getAllRows( LogChannel.UI );
      RowMetaInterface setRowMeta = set.getSetRowMeta( false );

      PreviewRowsDialog previewRowsDialog = new PreviewRowsDialog( shell, new Variables(), SWT.NONE, set.getName(), setRowMeta, setRows );
      previewRowsDialog.open();

    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", "Error previewing data from dataset table", e );
    }
  }

  public void dispose() {
    props.setScreen( new WindowProperty( shell ) );
    shell.dispose();
  }

  public void getData() {

    for ( DataSetGroup group : groups ) {
      wDataSetGroup.add( group.getName() );
    }

    wName.setText( Const.NVL( dataSet.getName(), "" ) );
    wDescription.setText( Const.NVL( dataSet.getDescription(), "" ) );
    wDataSetGroup.setText( Const.NVL( dataSet.getGroup() == null ? null : dataSet.getGroup().getName(), "" ) );
    wTableName.setText( Const.NVL( dataSet.getTableName(), "" ) );
    for ( int i = 0; i < dataSet.getFields().size(); i++ ) {
      DataSetField field = dataSet.getFields().get( i );
      int colnr = 1;
      wFieldMapping.setText( Const.NVL( field.getFieldName(), "" ), colnr++, i );
      wFieldMapping.setText( Const.NVL( field.getColumnName(), "" ), colnr++, i );
      wFieldMapping.setText( ValueMetaFactory.getValueMetaName( field.getType() ), colnr++, i );
      wFieldMapping.setText( Const.NVL( field.getFormat(), ""), colnr++, i );
      wFieldMapping.setText( field.getLength() >= 0 ? Integer.toString( field.getLength() ) : "", colnr++, i );
      wFieldMapping.setText( field.getPrecision() >= 0 ? Integer.toString( field.getPrecision() ) : "", colnr++, i );
      wFieldMapping.setText( Const.NVL( field.getComment(), "" ), colnr++, i );
    }
    wName.setFocus();
  }

  private void cancel() {
    ok = false;
    dispose();
  }

  /**
   * @param set The data set to load the dialog information into
   */
  public void getInfo( DataSet set ) {

    set.setName( wName.getText() );
    set.setDescription( wDescription.getText() );
    set.setGroup( DataSetConst.findDataSetGroup( groups, wDataSetGroup.getText() ) );
    set.setTableName( wTableName.getText() );
    set.getFields().clear();
    int nrFields = wFieldMapping.nrNonEmpty();
    for ( int i = 0; i < nrFields; i++ ) {
      TableItem item = wFieldMapping.getNonEmpty( i );
      int colnr = 1;
      String fieldName = item.getText( colnr++ );
      String columnName = item.getText( colnr++ );
      int type = ValueMetaFactory.getIdForValueMeta( item.getText( colnr++ ) );
      String format = item.getText( colnr++ );
      int length = Const.toInt( item.getText( colnr++ ), -1 );
      int precision = Const.toInt( item.getText( colnr++ ), -1 );
      String comment = item.getText( colnr++ );

      DataSetField field = new DataSetField( fieldName, columnName, type, length, precision, comment, format );
      set.getFields().add( field );
    }

  }

  public void ok() {

    try {
      verifySettings();
    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", BaseMessages.getString( PKG, "DataSetDialog.Error.ValidationError" ), e );
    }

    getInfo( dataSet );

    ok = true;
    dispose();

  }

  public IMetaStore getMetaStore() {
    return metaStore;
  }

  public void setMetaStore( IMetaStore metaStore ) {
    this.metaStore = metaStore;
  }

} public class DataSetGroupDialog extends Dialog {
  private static Class<?> PKG = DataSetGroupDialog.class; // for i18n purposes, needed by Translator2!!

  private DataSetGroup dataSetGroup;

  private Group wgDatabase;
  private Group wgCsv;

  private List<DatabaseMeta> databases;

  private Shell shell;

  private Text wName;
  private Text wDescription;
  private Combo wGroupType;

  // Database type
  //
  private Combo wDatabase;
  private Text wSchemaName;

  // CSV type
  //
  private TextVar wFolderName;

  private Button wOK, wCancel;

  private PropsUI props;

  private int middle;
  private int margin;

  private boolean ok;

  private VariableSpace space;

  public DataSetGroupDialog( Shell parent, DataSetGroup dataSetGroup, List<DatabaseMeta> databases ) {
    super( parent, SWT.NONE );
    this.dataSetGroup = dataSetGroup;
    this.databases = databases;
    props = PropsUI.getInstance();
    ok = false;

    space = new Variables();
    space.initializeVariablesFrom( null );
  }

  public boolean open() {
    Shell parent = getParent();
    shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN );
    props.setLook( shell );
    shell.setImage( GUIResource.getInstance().getImageTable() );

    middle = props.getMiddlePct();
    margin = Const.MARGIN;

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.Shell.Title" ) );
    shell.setLayout( formLayout );

    // The name of the group...
    //
    Label wlName = new Label( shell, SWT.RIGHT );
    props.setLook( wlName );
    wlName.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupName.Label" ) );
    FormData fdlName = new FormData();
    fdlName.top = new FormAttachment( 0, 0 );
    fdlName.left = new FormAttachment( 0, 0 );
    fdlName.right = new FormAttachment( middle, -margin );
    wlName.setLayoutData( fdlName );
    wName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wName );
    FormData fdName = new FormData();
    fdName.top = new FormAttachment( wlName, 0, SWT.CENTER );
    fdName.left = new FormAttachment( middle, 0 );
    fdName.right = new FormAttachment( 100, 0 );
    wName.setLayoutData( fdName );
    Control lastControl = wName;

    // The description of the group...
    //
    Label wlDescription = new Label( shell, SWT.RIGHT );
    props.setLook( wlDescription );
    wlDescription.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupDescription.Label" ) );
    FormData fdlDescription = new FormData();
    fdlDescription.top = new FormAttachment( lastControl, margin );
    fdlDescription.left = new FormAttachment( 0, 0 );
    fdlDescription.right = new FormAttachment( middle, -margin );
    wlDescription.setLayoutData( fdlDescription );
    wDescription = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wDescription );
    FormData fdDescription = new FormData();
    fdDescription.top = new FormAttachment( wlDescription, 0, SWT.CENTER );
    fdDescription.left = new FormAttachment( middle, 0 );
    fdDescription.right = new FormAttachment( 100, 0 );
    wDescription.setLayoutData( fdDescription );
    lastControl = wDescription;


    // The type of the group...
    //
    Label wlGroupType = new Label( shell, SWT.RIGHT );
    props.setLook( wlGroupType );
    wlGroupType.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupType.Label" ) );
    FormData fdlGroupType = new FormData();
    fdlGroupType.top = new FormAttachment( lastControl, margin );
    fdlGroupType.left = new FormAttachment( 0, 0 );
    fdlGroupType.right = new FormAttachment( middle, -margin );
    wlGroupType.setLayoutData( fdlGroupType );
    wGroupType = new Combo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    wGroupType.setItems( DataSetGroupType.getNames() );
    FormData fdGroupType = new FormData();
    fdGroupType.top = new FormAttachment( wlGroupType, 0, SWT.CENTER );
    fdGroupType.left = new FormAttachment( middle, 0 );
    fdGroupType.right = new FormAttachment( 100, 0 );
    wGroupType.setLayoutData( fdGroupType );
    lastControl = wGroupType;

    wGroupType.addListener( SWT.Selection, e -> enableGroups() );


    wgDatabase = new Group( shell, SWT.NO_BACKGROUND | SWT.SHADOW_ETCHED_IN );
    props.setLook( wgDatabase );
    wgDatabase.setText( "Database settings" );
    FormLayout databaseGroupLayout = new FormLayout();
    databaseGroupLayout.spacing = 10;
    databaseGroupLayout.marginTop = 10;
    databaseGroupLayout.marginBottom = 10;
    wgDatabase.setLayout( databaseGroupLayout );

    // The database of the group...
    //
    Label wlDatabase = new Label( wgDatabase, SWT.RIGHT );
    props.setLook( wlDatabase );
    wlDatabase.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupDatabase.Label" ) );
    FormData fdlDatabase = new FormData();
    fdlDatabase.top = new FormAttachment( 0, 0 );
    fdlDatabase.left = new FormAttachment( 0, 0 );
    fdlDatabase.right = new FormAttachment( middle, -margin );
    wlDatabase.setLayoutData( fdlDatabase );
    wDatabase = new Combo( wgDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    FormData fdDatabase = new FormData();
    fdDatabase.top = new FormAttachment( wlDatabase, 0, SWT.CENTER );
    fdDatabase.left = new FormAttachment( middle, 0 );
    fdDatabase.right = new FormAttachment( 100, 0 );
    wDatabase.setLayoutData( fdDatabase );
    Control lastGroupControl = wDatabase;

    // The schema in the database...
    //
    Label wlSchemaName = new Label( wgDatabase, SWT.RIGHT );
    props.setLook( wlSchemaName );
    wlSchemaName.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupSchemaName.Label" ) );
    FormData fdlSchemaName = new FormData();
    fdlSchemaName.top = new FormAttachment( lastGroupControl, margin );
    fdlSchemaName.left = new FormAttachment( 0, 0 );
    fdlSchemaName.right = new FormAttachment( middle, -margin );
    wlSchemaName.setLayoutData( fdlSchemaName );
    wSchemaName = new Text( wgDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wSchemaName );
    FormData fdSchemaName = new FormData();
    fdSchemaName.top = new FormAttachment( wlSchemaName, 0, SWT.CENTER );
    fdSchemaName.left = new FormAttachment( middle, 0 );
    fdSchemaName.right = new FormAttachment( 100, 0 );
    wSchemaName.setLayoutData( fdSchemaName );

    FormData fdgDatabase = new FormData();
    fdgDatabase.top = new FormAttachment( lastControl, margin * 2 );
    fdgDatabase.left = new FormAttachment( 0, 0 );
    fdgDatabase.right = new FormAttachment( 100, 0 );
    wgDatabase.setLayoutData( fdgDatabase );
    lastControl = wgDatabase;

    wgCsv = new Group( shell, SWT.NO_BACKGROUND | SWT.SHADOW_ETCHED_IN );
    props.setLook( wgCsv );
    wgCsv.setText( "CSV settings" );
    FormLayout csvGroupLayout = new FormLayout();
    csvGroupLayout.spacing = 10;
    csvGroupLayout.marginTop = 10;
    csvGroupLayout.marginBottom = 10;
    wgCsv.setLayout( csvGroupLayout );


    // The folder for the CSV files...
    //
    Label wlFolderName = new Label( wgCsv, SWT.RIGHT );
    props.setLook( wlFolderName );
    wlFolderName.setText( BaseMessages.getString( PKG, "DataSetGroupDialog.GroupFolderName.Label" ) );
    FormData fdlFolderName = new FormData();
    fdlFolderName.top = new FormAttachment( 0, 0 );
    fdlFolderName.left = new FormAttachment( 0, 0 );
    fdlFolderName.right = new FormAttachment( middle, -margin );
    wlFolderName.setLayoutData( fdlFolderName );
    wFolderName = new TextVar( space, wgCsv, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wFolderName );
    FormData fdFolderName = new FormData();
    fdFolderName.top = new FormAttachment( wlFolderName, 0, SWT.CENTER );
    fdFolderName.left = new FormAttachment( middle, 0 );
    fdFolderName.right = new FormAttachment( 100, 0 );
    wFolderName.setLayoutData( fdFolderName );

    FormData fdgCsv = new FormData();
    fdgCsv.top = new FormAttachment( lastControl, margin * 2 );
    fdgCsv.left = new FormAttachment( 0, 0 );
    fdgCsv.right = new FormAttachment( 100, 0 );
    wgCsv.setLayoutData( fdgCsv );
    lastControl = wgCsv;

    // Buttons
    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) );

    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) );

    Button[] buttons = new Button[] { wOK, wCancel };
    BaseStepDialog.positionBottomButtons( shell, buttons, margin, lastControl );

    // Add listeners
    wOK.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        ok();
      }
    } );
    wCancel.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        cancel();
      }
    } );

    SelectionAdapter selAdapter = new SelectionAdapter() {
      public void widgetDefaultSelected( SelectionEvent e ) {
        ok();
      }
    };
    wName.addSelectionListener( selAdapter );
    wDescription.addSelectionListener( selAdapter );
    wSchemaName.addSelectionListener( selAdapter );
    wFolderName.addSelectionListener( selAdapter );
    wGroupType.addSelectionListener( selAdapter );

    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener( new ShellAdapter() {
      public void shellClosed( ShellEvent e ) {
        cancel();
      }
    } );

    getData();

    BaseStepDialog.setSize( shell );

    shell.open();
    Display display = parent.getDisplay();
    while ( !shell.isDisposed() ) {
      if ( !display.readAndDispatch() ) {
        display.sleep();
      }
    }
    return ok;
  }

  private void enableGroups() {
    DataSetGroupType type = DataSetGroupType.fromName( wGroupType.getText() );

    wgDatabase.setEnabled( type == DataSetGroupType.Database );
    wDatabase.setEnabled( type == DataSetGroupType.Database );
    wSchemaName.setEnabled( type == DataSetGroupType.Database );

    wgCsv.setEnabled( type == DataSetGroupType.CSV );
    wFolderName.setEnabled( type == DataSetGroupType.CSV );

  }

  public void dispose() {
    props.setScreen( new WindowProperty( shell ) );
    shell.dispose();
  }

  public void getData() {

    for ( DatabaseMeta database : databases ) {
      wDatabase.add( database.getName() );
    }

    wName.setText( Const.NVL( dataSetGroup.getName(), "" ) );
    wDescription.setText( Const.NVL( dataSetGroup.getDescription(), "" ) );
    wGroupType.setText( dataSetGroup.getType() == null ? DataSetGroupType.Database.name() : dataSetGroup.getType().name() );
    wDatabase.setText( Const.NVL( dataSetGroup.getDatabaseMeta() == null ? null : dataSetGroup.getDatabaseMeta().getName(), "" ) );
    wSchemaName.setText( Const.NVL( dataSetGroup.getSchemaName(), "" ) );
    wFolderName.setText( Const.NVL( dataSetGroup.getFolderName(), "" ) );

    enableGroups();

    wName.setFocus();
  }

  private void cancel() {
    ok = false;
    dispose();
  }

  public void ok() {

    dataSetGroup.setName( wName.getText() );
    dataSetGroup.setDescription( wDescription.getText() );
    dataSetGroup.setType( DataSetGroupType.fromName( wGroupType.getText() ) );
    dataSetGroup.setDatabaseMeta( DatabaseMeta.findDatabase( databases, wDatabase.getText() ) );
    dataSetGroup.setSchemaName( wSchemaName.getText() );
    dataSetGroup.setFolderName( wFolderName.getText() );

    ok = true;

    dispose();
  }
} public class EditRowsDialog {
  private static Class<?> PKG = EditRowsDialog.class; // for i18n purposes, needed by Translator2!!

  public static final int MAX_BINARY_STRING_PREVIEW_SIZE = 1000000;

  private Label wlMessage;

  private TableView wFields;

  private FormData fdlFields, fdFields;

  private Button wOK;

  private Button wCancel;

  private Shell shell;

  private List<Object[]> rowBuffer;

  private PropsUI props;

  private String title, message;

  private Rectangle bounds;

  private int hscroll, vscroll;

  private int hmax, vmax;

  private RowMetaInterface rowMeta;

  private LogChannelInterface log;

  protected int lineNr;

  private int style = SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN;

  private Shell parentShell;

  private List<Object[]> outputList;

  private RowMetaInterface stringRowMeta;

  public EditRowsDialog( Shell parent, int style,
                         String title, String message, RowMetaInterface rowMeta, List<Object[]> rowBuffer ) {
    this.title = title;
    this.message = message;
    this.rowBuffer = rowBuffer;
    this.rowMeta = rowMeta;
    this.parentShell = parent;
    this.style = ( style != SWT.None ) ? style : this.style;

    props = PropsUI.getInstance();
    bounds = null;
    hscroll = -1;
    vscroll = -1;
    title = null;
    message = null;

    this.log = LogChannel.GENERAL;
  }

  public void setTitleMessage( String title, String message ) {
    this.title = title;
    this.message = message;
  }

  public List<Object[]> open() {
    shell = new Shell( parentShell, style );
    props.setLook( shell );
    shell.setImage( GUIResource.getInstance().getImageSpoon() );

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setLayout( formLayout );
    shell.setText( title );

    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) );
    wOK.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        ok();
      }
    } );

    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) );
    wCancel.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        cancel();
      }
    } );

    // Position the buttons...
    //
    BaseStepDialog.positionBottomButtons( shell, new Button[] { wOK, wCancel, }, Const.MARGIN, null );

    if ( addFields() ) {
      return null;
    }

    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener( new ShellAdapter() {
      public void shellClosed( ShellEvent e ) {
        cancel();
      }
    } );

    getData();

    BaseStepDialog.setSize( shell );

    shell.open();

    while ( !shell.isDisposed() ) {
      if ( !shell.getDisplay().readAndDispatch() ) {
        shell.getDisplay().sleep();
      }
    }

    return outputList;

  }

  private boolean addFields() {
    // int middle = props.getMiddlePct();
    int margin = Const.MARGIN;

    if ( wlMessage == null ) {
      wlMessage = new Label( shell, SWT.LEFT );
      wlMessage.setText( message );
      props.setLook( wlMessage );
      fdlFields = new FormData();
      fdlFields.left = new FormAttachment( 0, 0 );
      fdlFields.right = new FormAttachment( 100, 0 );
      fdlFields.top = new FormAttachment( 0, margin );
      wlMessage.setLayoutData( fdlFields );
    } else {
      wFields.dispose();
    }

    // Mmm, if we don't get any row metadata: show a dialog box.
    if ( rowMeta == null || rowMeta.size() == 0 ) {
      ShowMessageDialog dialog = new ShowMessageDialog( shell, SWT.OK | SWT.ICON_WARNING,
        BaseMessages.getString( PKG, "EditRowsDialog.NoRowMeta.Text" ),
        BaseMessages.getString( PKG, "EditRowsDialog.NoRowMeta.Message" ) );
      dialog.open();
      shell.dispose();
      return true;
    }

    // ColumnInfo[] colinf = new ColumnInfo[rowMeta==null ? 0 : rowMeta.size()];
    ColumnInfo[] colinf = new ColumnInfo[ rowMeta.size() ];
    for ( int i = 0; i < rowMeta.size(); i++ ) {
      ValueMetaInterface v = rowMeta.getValueMeta( i );
      colinf[ i ] = new ColumnInfo( v.getName(), ColumnInfo.COLUMN_TYPE_TEXT, v.isNumeric() );
      colinf[ i ].setToolTip( v.toStringMeta() );
      colinf[ i ].setValueMeta( v );
    }

    wFields = new TableView(
      new Variables(),
      shell,
      SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI,
      colinf,
      rowBuffer.size(),
      null,
      props );
    wFields.setShowingBlueNullValues( true );

    fdFields = new FormData();
    fdFields.left = new FormAttachment( 0, 0 );
    fdFields.top = new FormAttachment( wlMessage, margin );
    fdFields.right = new FormAttachment( 100, 0 );
    fdFields.bottom = new FormAttachment( 100, -50 );
    wFields.setLayoutData( fdFields );

    shell.layout( true, true );

    return false;
  }

  public void dispose() {
    props.setScreen( new WindowProperty( shell ) );
    bounds = shell.getBounds();
    hscroll = wFields.getHorizontalBar().getSelection();
    vscroll = wFields.getVerticalBar().getSelection();
    shell.dispose();
  }

  /**
   * Copy information from the meta-data input to the dialog fields.
   */
  private void getData() {
    shell.getDisplay().asyncExec( new Runnable() {
      public void run() {
        lineNr = 0;
        for ( int i = 0; i < rowBuffer.size(); i++ ) {
          TableItem item = wFields.table.getItem( i );
          Object[] row = rowBuffer.get( i );
          getDataForRow( item, row );
        }
        wFields.optWidth( true, 200 );
      }
    } );
  }

  protected int getDataForRow( TableItem item, Object[] row ) {
    int nrErrors = 0;

    // Display the correct line item...
    //
    String strNr;
    lineNr++;
    try {
      strNr = wFields.getNumberColumn().getValueMeta().getString( new Long( lineNr ) );
    } catch ( Exception e ) {
      strNr = Integer.toString( lineNr );
    }
    item.setText( 0, strNr );

    for ( int c = 0; c < rowMeta.size(); c++ ) {
      ValueMetaInterface v = rowMeta.getValueMeta( c );
      String show;
      try {
        show = v.getString( row[ c ] );
        if ( v.isBinary() && show != null && show.length() > MAX_BINARY_STRING_PREVIEW_SIZE ) {
          // We want to limit the size of the strings during preview to keep all SWT widgets happy.
          //
          show = show.substring( 0, MAX_BINARY_STRING_PREVIEW_SIZE );
        }
      } catch ( KettleValueException e ) {
        nrErrors++;
        if ( nrErrors < 25 ) {
          log.logError( Const.getStackTracker( e ) );
        }
        show = null;
      } catch ( ArrayIndexOutOfBoundsException e ) {
        nrErrors++;
        if ( nrErrors < 25 ) {
          log.logError( Const.getStackTracker( e ) );
        }
        show = null;
      }

      if ( show != null ) {
        item.setText( c + 1, show );
        item.setForeground( c + 1, GUIResource.getInstance().getColorBlack() );
      } else {
        // Set null value
        item.setText( c + 1, "<null>" );
        item.setForeground( c + 1, GUIResource.getInstance().getColorBlue() );
      }
    }

    return nrErrors;

  }

  private Object[] getRowForData( TableItem item, int rowNr ) throws KettleException {
    try {
      Object[] row = RowDataUtil.allocateRowData( rowMeta.size() );
      for ( int i = 0; i < rowMeta.size(); i++ ) {
        ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
        ValueMetaInterface stringValueMeta = stringRowMeta.getValueMeta( i );

        int colnr = i + 1;
        if ( GUIResource.getInstance().getColorBlue().equals( item.getForeground( colnr ) ) ) {
          row[ i ] = null; // <null> value
        } else {
          String string = item.getText( colnr );
          row[ i ] = valueMeta.convertDataFromString( string, stringValueMeta,
            null, null, ValueMetaInterface.TRIM_TYPE_NONE );
        }
      }
      return row;
    } catch ( KettleException e ) {
      throw new KettleException( BaseMessages.getString( PKG, "EditRowsDialog.Error.ErrorGettingRowForData",
        Integer.toString( rowNr ) ), e );
    }
  }

  private void ok() {

    try {
      stringRowMeta = new RowMeta();
      for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
        ValueMetaInterface stringValueMeta = ValueMetaFactory.cloneValueMeta( valueMeta,
          ValueMetaInterface.TYPE_STRING );
        stringRowMeta.addValueMeta( stringValueMeta );
      }

      List<Object[]> list = new ArrayList<Object[]>();

      // Now read all the rows in the dialog, including the empty rows...
      //
      for ( int i = 0; i < wFields.getItemCount(); i++ ) {
        TableItem item = wFields.getTable().getItem( i );
        Object[] row = getRowForData( item, i + 1 );
        list.add( row );
      }

      outputList = list;
      dispose();

    } catch ( Exception e ) {
      new ErrorDialog( shell, "Error", BaseMessages.getString( PKG, "EditRowsDialog.ErrorConvertingData" ), e );
    }
  }

  private void cancel() {
    outputList = null;
    dispose();
  }

  public Rectangle getBounds() {
    return bounds;
  }

  public void setBounds( Rectangle b ) {
    bounds = b;
  }

  public int getHScroll() {
    return hscroll;
  }

  public void setHScroll( int s ) {
    hscroll = s;
  }

  public int getVScroll() {
    return vscroll;
  }

  public void setVScroll( int s ) {
    vscroll = s;
  }

  public int getHMax() {
    return hmax;
  }

  public void setHMax( int m ) {
    hmax = m;
  }

  public int getVMax() {
    return vmax;
  }

  public void setVMax( int m ) {
    vmax = m;
  }
} public class TransUnitTestDialog extends Dialog {
  private static Class<?> PKG = TransUnitTestDialog.class; // for i18n purposes, needed by Translator2!!

  private TransUnitTest transUnitTest;
  protected TransMeta transMeta;

  private Shell shell;

  private Text wName;
  private Text wDescription;
  private Combo wTestType;
  private TextVar wFilename;
  private TextVar wBasePath;
  private Button wAutoOpen;

  private TableView wDbReplacements;
  private TableView wVariableValues;


  private Button wOK;
  private Button wCancel;

  private PropsUI props;

  private int middle;
  private int margin;

  private boolean ok;

  protected IMetaStore metaStore;
  MetaStoreFactory<TransUnitTest> setFactory;

  public TransUnitTestDialog( Shell parent, TransMeta transMeta, IMetaStore metaStore, TransUnitTest transUnitTest ) throws KettleException, MetaStoreException {
    super( parent, SWT.NONE );
    this.transMeta = transMeta;
    this.metaStore = metaStore;
    this.transUnitTest = transUnitTest;
    props = PropsUI.getInstance();
    ok = false;

    setFactory = new MetaStoreFactory<TransUnitTest>( TransUnitTest.class, metaStore, PentahoDefaults.NAMESPACE );
  }

  public boolean open() {
    Shell parent = getParent();
    shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN );
    props.setLook( shell );
    shell.setImage( GUIResource.getInstance().getImageTable() );

    middle = props.getMiddlePct();
    margin = Const.MARGIN;

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.Shell.Title" ) );
    shell.setLayout( formLayout );

    // The name of the unit test...
    //
    Label wlName = new Label( shell, SWT.RIGHT );
    props.setLook( wlName );
    wlName.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.Name.Label" ) );
    FormData fdlName = new FormData();
    fdlName.top = new FormAttachment( 0, 0 );
    fdlName.left = new FormAttachment( 0, 0 );
    fdlName.right = new FormAttachment( middle, -margin );
    wlName.setLayoutData( fdlName );
    wName = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wName );
    FormData fdName = new FormData();
    fdName.top = new FormAttachment( 0, 0 );
    fdName.left = new FormAttachment( middle, 0 );
    fdName.right = new FormAttachment( 100, 0 );
    wName.setLayoutData( fdName );
    Control lastControl = wName;

    // The description of the test...
    //
    Label wlDescription = new Label( shell, SWT.RIGHT );
    props.setLook( wlDescription );
    wlDescription.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.Description.Label" ) );
    FormData fdlDescription = new FormData();
    fdlDescription.top = new FormAttachment( lastControl, margin );
    fdlDescription.left = new FormAttachment( 0, 0 );
    fdlDescription.right = new FormAttachment( middle, -margin );
    wlDescription.setLayoutData( fdlDescription );
    wDescription = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wDescription );
    FormData fdDescription = new FormData();
    fdDescription.top = new FormAttachment( lastControl, margin );
    fdDescription.left = new FormAttachment( middle, 0 );
    fdDescription.right = new FormAttachment( 100, 0 );
    wDescription.setLayoutData( fdDescription );
    lastControl = wDescription;

    // The type of test...
    //
    Label wlTestType = new Label( shell, SWT.RIGHT );
    props.setLook( wlTestType );
    wlTestType.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.TestType.Label" ) );
    FormData fdlTestType = new FormData();
    fdlTestType.top = new FormAttachment( lastControl, margin );
    fdlTestType.left = new FormAttachment( 0, 0 );
    fdlTestType.right = new FormAttachment( middle, -margin );
    wlTestType.setLayoutData( fdlTestType );
    wTestType = new Combo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    FormData fdTestType = new FormData();
    fdTestType.top = new FormAttachment( lastControl, margin );
    fdTestType.left = new FormAttachment( middle, 0 );
    fdTestType.right = new FormAttachment( 100, 0 );
    wTestType.setLayoutData( fdTestType );
    wTestType.setItems( DataSetConst.getTestTypeDescriptions() );
    lastControl = wTestType;

    // The optional filename of the test result...
    //
    Label wlFilename = new Label( shell, SWT.RIGHT );
    props.setLook( wlFilename );
    wlFilename.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.Filename.Label" ) );
    FormData fdlFilename = new FormData();
    fdlFilename.top = new FormAttachment( lastControl, margin );
    fdlFilename.left = new FormAttachment( 0, 0 );
    fdlFilename.right = new FormAttachment( middle, -margin );
    wlFilename.setLayoutData( fdlFilename );
    wFilename = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wFilename );
    FormData fdFilename = new FormData();
    fdFilename.top = new FormAttachment( lastControl, margin );
    fdFilename.left = new FormAttachment( middle, 0 );
    fdFilename.right = new FormAttachment( 100, 0 );
    wFilename.setLayoutData( fdFilename );
    lastControl = wFilename;

    // The base path for relative test path resolution
    //
    Label wlBasePath = new Label( shell, SWT.RIGHT );
    props.setLook( wlBasePath );
    wlBasePath.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.BasePath.Label" ) );
    FormData fdlBasePath = new FormData();
    fdlBasePath.top = new FormAttachment( lastControl, margin );
    fdlBasePath.left = new FormAttachment( 0, 0 );
    fdlBasePath.right = new FormAttachment( middle, -margin );
    wlBasePath.setLayoutData( fdlBasePath );
    wBasePath = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wBasePath );
    FormData fdBasePath = new FormData();
    fdBasePath.top = new FormAttachment( lastControl, margin );
    fdBasePath.left = new FormAttachment( middle, 0 );
    fdBasePath.right = new FormAttachment( 100, 0 );
    wBasePath.setLayoutData( fdBasePath );
    lastControl = wBasePath;

    // The base path for relative test path resolution
    //
    Label wlAutoOpen = new Label( shell, SWT.RIGHT );
    props.setLook( wlAutoOpen );
    wlAutoOpen.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.AutoOpen.Label" ) );
    FormData fdlAutoOpen = new FormData();
    fdlAutoOpen.top = new FormAttachment( lastControl, margin );
    fdlAutoOpen.left = new FormAttachment( 0, 0 );
    fdlAutoOpen.right = new FormAttachment( middle, -margin );
    wlAutoOpen.setLayoutData( fdlAutoOpen );
    wAutoOpen = new Button( shell, SWT.CHECK );
    props.setLook( wAutoOpen );
    FormData fdAutoOpen = new FormData();
    fdAutoOpen.top = new FormAttachment( lastControl, margin );
    fdAutoOpen.left = new FormAttachment( middle, 0 );
    fdAutoOpen.right = new FormAttachment( 100, 0 );
    wAutoOpen.setLayoutData( fdAutoOpen );
    lastControl = wAutoOpen;

    // The list of database replacements in the unit test transformation
    //
    Label wlFieldMapping = new Label( shell, SWT.NONE );
    wlFieldMapping.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.DbReplacements.Label" ) );
    props.setLook( wlFieldMapping );
    FormData fdlUpIns = new FormData();
    fdlUpIns.left = new FormAttachment( 0, 0 );
    fdlUpIns.top = new FormAttachment( lastControl, margin );
    wlFieldMapping.setLayoutData( fdlUpIns );
    lastControl = wlFieldMapping;

    // Buttons at the bottom...
    //
    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( BaseMessages.getString( PKG, "System.Button.OK" ) );

    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( BaseMessages.getString( PKG, "System.Button.Cancel" ) );

    Button[] buttons = new Button[] { wOK, wCancel };
    BaseStepDialog.positionBottomButtons( shell, buttons, margin, null );

    // the database replacements
    //
    String[] dbNames = transMeta.getDatabaseNames();
    Arrays.sort( dbNames );
    ColumnInfo[] columns = new ColumnInfo[] {
      new ColumnInfo( BaseMessages.getString( PKG, "TransUnitTestDialog.DbReplacement.ColumnInfo.OriginalDb" ),
        ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "TransUnitTestDialog.DbReplacement.ColumnInfo.ReplacementDb" ),
        ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames, false ), };
    columns[ 0 ].setUsingVariables( true );
    columns[ 1 ].setUsingVariables( true );

    wDbReplacements = new TableView( new Variables(), shell,
      SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, columns,
      transUnitTest.getTweaks().size(), null, props );

    FormData fdDbReplacements = new FormData();
    fdDbReplacements.left = new FormAttachment( 0, 0 );
    fdDbReplacements.top = new FormAttachment( lastControl, margin );
    fdDbReplacements.right = new FormAttachment( 100, 0 );
    fdDbReplacements.bottom = new FormAttachment( lastControl, 250 );
    wDbReplacements.setLayoutData( fdDbReplacements );
    lastControl = wDbReplacements;

    Label wlVariableValues = new Label( shell, SWT.NONE );
    wlVariableValues.setText( BaseMessages.getString( PKG, "TransUnitTestDialog.VariableValues.Label" ) );
    props.setLook( wlVariableValues );
    FormData fdlVariableValues = new FormData();
    fdlVariableValues.left = new FormAttachment( 0, 0 );
    fdlVariableValues.top = new FormAttachment( lastControl, margin );
    wlVariableValues.setLayoutData( fdlVariableValues );
    lastControl = wlVariableValues;

    ColumnInfo[] varValColumns = new ColumnInfo[] {
      new ColumnInfo( BaseMessages.getString( PKG, "TransUnitTestDialog.VariableValues.ColumnInfo.VariableName" ), ColumnInfo.COLUMN_TYPE_TEXT, false ),
      new ColumnInfo( BaseMessages.getString( PKG, "TransUnitTestDialog.VariableValues.ColumnInfo.VariableValue" ), ColumnInfo.COLUMN_TYPE_TEXT, false ),
    };
    varValColumns[ 0 ].setUsingVariables( true );
    varValColumns[ 1 ].setUsingVariables( true );

    wVariableValues = new TableView( new Variables(), shell,
      SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, varValColumns,
      transUnitTest.getVariableValues().size(), null, props );

    FormData fdVariableValues = new FormData();
    fdVariableValues.left = new FormAttachment( 0, 0 );
    fdVariableValues.top = new FormAttachment( lastControl, margin );
    fdVariableValues.right = new FormAttachment( 100, 0 );
    fdVariableValues.bottom = new FormAttachment( wOK, -2 * margin );
    wVariableValues.setLayoutData( fdVariableValues );

    // Add listeners
    wOK.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        ok();
      }
    } );
    wCancel.addListener( SWT.Selection, new Listener() {
      public void handleEvent( Event e ) {
        cancel();
      }
    } );

    SelectionAdapter selAdapter = new SelectionAdapter() {
      public void widgetDefaultSelected( SelectionEvent e ) {
        ok();
      }
    };
    wName.addSelectionListener( selAdapter );
    wDescription.addSelectionListener( selAdapter );
    wTestType.addSelectionListener( selAdapter );
    wFilename.addSelectionListener( selAdapter );
    wBasePath.addSelectionListener( selAdapter );

    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener( new ShellAdapter() {
      public void shellClosed( ShellEvent e ) {
        cancel();
      }
    } );

    getData();

    BaseStepDialog.setSize( shell );

    shell.open();
    Display display = parent.getDisplay();
    while ( !shell.isDisposed() ) {
      if ( !display.readAndDispatch() ) {
        display.sleep();
      }
    }
    return ok;
  }

  public void dispose() {
    props.setScreen( new WindowProperty( shell ) );
    shell.dispose();
  }

  public void getData() {

    wName.setText( Const.NVL( transUnitTest.getName(), "" ) );
    wDescription.setText( Const.NVL( transUnitTest.getDescription(), "" ) );
    wTestType.setText( Const.NVL( DataSetConst.getTestTypeDescription( transUnitTest.getType() ), "" ) );
    wFilename.setText( Const.NVL( transUnitTest.getFilename(), "" ) );
    wBasePath.setText( Const.NVL( transUnitTest.getBasePath(), "" ) );
    wAutoOpen.setSelection( transUnitTest.isAutoOpening() );

    for ( int i = 0; i < transUnitTest.getDatabaseReplacements().size(); i++ ) {
      TransUnitTestDatabaseReplacement dbReplacement = transUnitTest.getDatabaseReplacements().get( i );
      wDbReplacements.setText( Const.NVL( dbReplacement.getOriginalDatabaseName(), "" ), 1, i );
      wDbReplacements.setText( Const.NVL( dbReplacement.getReplacementDatabaseName(), "" ), 2, i );
    }

    for ( int i = 0; i < transUnitTest.getVariableValues().size(); i++ ) {
      VariableValue variableValue = transUnitTest.getVariableValues().get( i );
      wVariableValues.setText( Const.NVL( variableValue.getKey(), "" ), 1, i );
      wVariableValues.setText( Const.NVL( variableValue.getValue(), "" ), 2, i );
    }

    wDbReplacements.removeEmptyRows();
    wDbReplacements.setRowNums();

    wName.setFocus();
  }

  private void cancel() {
    ok = false;
    dispose();
  }

  /**
   * @param test The trans unit test to load the dialog information into
   */
  public void getInfo( TransUnitTest test ) {

    test.setName( wName.getText() );
    test.setDescription( wDescription.getText() );
    test.setType( DataSetConst.getTestTypeForDescription( wTestType.getText() ) );
    test.setFilename( wFilename.getText() );
    test.setBasePath( wBasePath.getText() );
    test.setAutoOpening( wAutoOpen.getSelection() );

    test.getDatabaseReplacements().clear();
    int nrFields = wDbReplacements.nrNonEmpty();
    for ( int i = 0; i < nrFields; i++ ) {
      TableItem item = wDbReplacements.getNonEmpty( i );
      String sourceDb = item.getText( 1 );
      String replaceDb = item.getText( 2 );
      TransUnitTestDatabaseReplacement dbReplacement = new TransUnitTestDatabaseReplacement( sourceDb, replaceDb );
      test.getDatabaseReplacements().add( dbReplacement );
    }
    test.getVariableValues().clear();
    int nrVars = wVariableValues.nrNonEmpty();
    for ( int i = 0; i < nrVars; i++ ) {
      TableItem item = wVariableValues.getNonEmpty( i );
      String key = item.getText( 1 );
      String value = item.getText( 2 );
      VariableValue variableValue = new VariableValue( key, value );
      test.getVariableValues().add( variableValue );
    }
  }

  public void ok() {

    getInfo( transUnitTest );

    ok = true;
    dispose();

  }


} public class TransMetaModifier {

  private TransMeta transMeta;
  private TransUnitTest unitTest;

  public TransMetaModifier() {
  }

  public TransMetaModifier( TransMeta transMeta, TransUnitTest unitTest ) {
    this.transMeta = transMeta;
    this.unitTest = unitTest;
  }

  public TransMeta getTestTransformation( LogChannelInterface log, VariableSpace space, FactoriesHierarchy factoriesHierarchy ) throws KettleException {
    // OK, so now replace an input step with a data set attached with an Injector step...
    // However, we don't want to have the user see this so we need to copy trans.transMeta first...
    //
    // Clone seems to has problems so we'll take the long (XML) way around...
    InputStream stream;
    try {
      stream = new ByteArrayInputStream( transMeta.getXML().getBytes( Const.XML_ENCODING ) );
    } catch ( UnsupportedEncodingException e ) {
      throw new KettleException( "Encoding error", e );
    }
    TransMeta copyTransMeta = new TransMeta( stream, transMeta.getRepository(), true, transMeta, null );
    // Pass the metadata references...
    //
    copyTransMeta.setRepository( transMeta.getRepository() );
    copyTransMeta.setMetaStore( transMeta.getMetaStore() );
    copyTransMeta.setSharedObjects( transMeta.getSharedObjects() );

    // Replace certain connections with another
    //
    for ( TransUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements() ) {
      String sourceDatabaseName = transMeta.environmentSubstitute( dbReplacement.getOriginalDatabaseName() );
      String replacementDatabaseName = transMeta.environmentSubstitute( dbReplacement.getReplacementDatabaseName() );

      DatabaseMeta sourceDatabaseMeta = copyTransMeta.findDatabase( sourceDatabaseName );
      DatabaseMeta replacementDatabaseMeta = copyTransMeta.findDatabase( replacementDatabaseName );
      if ( sourceDatabaseMeta == null ) {
        throw new KettleException( "Unable to find source database connection '" + sourceDatabaseName + "', can not be replaced" );
      }
      if ( replacementDatabaseMeta == null ) {
        throw new KettleException( "Unable to find replacement database connection '" + replacementDatabaseName + "', can not be used to replace" );
      }

      if ( log.isDetailed() ) {
        log.logDetailed( "Replaced database connection '" + sourceDatabaseName + "' with connection '" + replacementDatabaseName + "'" );
      }
      sourceDatabaseMeta.replaceMeta( replacementDatabaseMeta );
    }

    // Set parameters and variables...
    //
    String[] parameters = copyTransMeta.listParameters();
    List<VariableValue> variableValues = unitTest.getVariableValues();
    for ( VariableValue variableValue : variableValues ) {
      String key = space.environmentSubstitute( variableValue.getKey() );
      String value = space.environmentSubstitute( variableValue.getValue() );

      if ( StringUtils.isEmpty( key ) ) {
        continue;
      }
      if ( Const.indexOfString( key, parameters ) < 0 ) {
        // set the variable in the transformation metadata...
        //
        copyTransMeta.setVariable( key, value );
      } else {
        // Set the parameter value...
        //
        copyTransMeta.setParameterValue( key, value );
      }
    }

    // Replace all steps with an Input Data Set marker with an Injector
    // Replace all steps with a Golden Data Set marker with a Dummy
    // Apply the tweaks to the steps:
    //   - Bypass : replace with Dummy
    //   - Remove : remove step and all connected hops.
    //
    // Loop over the original transformation to allow us to safely modify the copy
    //
    List<StepMeta> steps = transMeta.getSteps();
    for ( StepMeta step : steps ) {
      StepMeta stepMeta = copyTransMeta.findStep( step.getName() );
      TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() );
      TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() );
      TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() );

      // See if there's a unit test if the step isn't flagged...
      //
      if ( inputLocation != null ) {
        handleInputDataSet( log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy );
      }

      // Capture golden data in a dummy step instead of the regular one?
      //
      if ( goldenLocation != null ) {
        handleGoldenDataSet( log, goldenLocation, stepMeta );
      }

      if ( stepTweak != null && stepTweak.getTweak() != null ) {
        switch ( stepTweak.getTweak() ) {
          case NONE:
            break;
          case REMOVE_STEP:
            handleTweakRemoveStep( log, copyTransMeta, stepMeta );
            break;
          case BYPASS_STEP:
            handleTweakBypassStep( log, stepMeta );
            break;
          default:
            break;
        }
      }
    }

    return copyTransMeta;
  }

  private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta,
                                   FactoriesHierarchy factoriesHierarchy ) throws KettleException {

    String inputSetName = inputLocation.getDataSetName();

    if ( log.isDetailed() ) {
      log.logDetailed( "Replacing step '" + stepMeta.getName() + "' with an Injector for dataset '" + inputSetName + "'" );
    }

    DataSet dataSet;
    try {
      dataSet = factoriesHierarchy.getSetFactory().loadElement( inputSetName );
    } catch ( MetaStoreException e ) {
      throw new KettleException( "Unable to load data set '" + inputSetName + "'" );
    }

    // OK, this step needs to be replaced by an Injector step...
    // Which fields do we need to use?
    //
    final RowMetaInterface stepFields = DataSetConst.getStepOutputFields( log, dataSet, inputLocation );

    if ( log.isDetailed() ) {
      log.logDetailed( "Input Data Set '" + inputSetName + "' Injector fields : '" + stepFields.toString() );
    }

    InjectorMeta injectorMeta = new InjectorMeta();
    injectorMeta.allocate( stepFields.size() );
    for ( int x = 0; x < stepFields.size(); x++ ) {
      injectorMeta.getFieldname()[ x ] = stepFields.getValueMeta( x ).getName();
      injectorMeta.getType()[ x ] = stepFields.getValueMeta( x ).getType();
      injectorMeta.getLength()[ x ] = stepFields.getValueMeta( x ).getLength();
      injectorMeta.getPrecision()[ x ] = stepFields.getValueMeta( x ).getPrecision();

      // Only the step metadata, type...
      stepMeta.setStepMetaInterface( injectorMeta );
      stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, injectorMeta ) );
    }
  }

  private void handleGoldenDataSet( LogChannelInterface log, TransUnitTestSetLocation goldenSetName, StepMeta stepMeta ) {

    if ( log.isDetailed() ) {
      log.logDetailed( "Replacing step '" + stepMeta.getName() + "' with an Dummy for golden dataset '" + goldenSetName + "'" );
    }

    replaceStepWithDummy( log, stepMeta );
  }

  private void replaceStepWithDummy( LogChannelInterface log, StepMeta stepMeta ) {
    DummyTransMeta dummyTransMeta = new DummyTransMeta();
    stepMeta.setStepMetaInterface( dummyTransMeta );
    stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyTransMeta ) );
  }

  private void handleTweakBypassStep( LogChannelInterface log, StepMeta stepMeta ) {
    if ( log.isDetailed() ) {
      log.logDetailed( "Replacing step '" + stepMeta.getName() + "' with an Dummy for Bypass step tweak" );
    }

    replaceStepWithDummy( log, stepMeta );
  }

  private void handleTweakRemoveStep( LogChannelInterface log, TransMeta copyTransMeta, StepMeta stepMeta ) {
    if ( log.isDetailed() ) {
      log.logDetailed( "Removing step '" + stepMeta.getName() + "' for Remove step tweak" );
    }

    // Remove all hops connecting to the step to be removed...
    //
    List<StepMeta> prevSteps = copyTransMeta.findPreviousSteps( stepMeta );
    for ( StepMeta prevStep : prevSteps ) {
      TransHopMeta hop = copyTransMeta.findTransHop( prevStep, stepMeta );
      if ( hop != null ) {
        int hopIndex = copyTransMeta.indexOfTransHop( hop );
        copyTransMeta.removeTransHop( hopIndex );
      }
    }
    List<StepMeta> nextSteps = copyTransMeta.findNextSteps( stepMeta );
    for ( StepMeta nextStep : nextSteps ) {
      TransHopMeta hop = copyTransMeta.findTransHop( stepMeta, nextStep );
      if ( hop != null ) {
        int hopIndex = copyTransMeta.indexOfTransHop( hop );
        copyTransMeta.removeTransHop( hopIndex );
      }
    }

    int idx = copyTransMeta.indexOfStep( stepMeta );
    if ( idx >= 0 ) {
      copyTransMeta.removeStep( idx );
    }
  }


  /**
   * Gets transMeta
   *
   * @return value of transMeta
   */
  public TransMeta getTransMeta() {
    return transMeta;
  }

  /**
   * @param transMeta The transMeta to set
   */
  public void setTransMeta( TransMeta transMeta ) {
    this.transMeta = transMeta;
  }

  /**
   * Gets unitTest
   *
   * @return value of unitTest
   */
  public TransUnitTest getUnitTest() {
    return unitTest;
  }

  /**
   * @param unitTest The unitTest to set
   */
  public void setUnitTest( TransUnitTest unitTest ) {
    this.unitTest = unitTest;
  }
} @ExtensionPoint(
  extensionPointId = "TransformationStartThreads",
  id = "WriteToDataSetExtensionPoint",
  description = "Writes rows of data from a step into a data set" )
public class WriteToDataSetExtensionPoint implements ExtensionPointInterface {

  public static Map<String, StepMeta> stepsMap = new HashMap<String, StepMeta>();
  public static Map<String, List<SourceToTargetMapping>> mappingsMap = new HashMap<String, List<SourceToTargetMapping>>();
  public static Map<String, DataSet> setsMap = new HashMap<String, DataSet>();

  @Override
  public void callExtensionPoint( LogChannelInterface log, Object object ) throws KettleException {
    if ( !( object instanceof Trans ) ) {
      return;
    }

    final Trans trans = (Trans) object;
    final TransMeta transMeta = trans.getTransMeta();
    boolean writeToDataSet = "Y".equalsIgnoreCase( transMeta.getVariable( DataSetConst.VAR_WRITE_TO_DATASET ) );
    if ( !writeToDataSet ) {
      return;
    }

    trans.addTransListener( new TransAdapter() {
      @Override public void transFinished( Trans trans ) throws KettleException {
        // Remove the flag when done.
        // We don't want to write to the data set every time we run
        //
        transMeta.setVariable( DataSetConst.VAR_WRITE_TO_DATASET, null );

        // Prevent memory leaking as well
        //
        WriteToDataSetExtensionPoint.stepsMap.remove( transMeta.getName() );
        WriteToDataSetExtensionPoint.mappingsMap.remove( transMeta.getName() );
        WriteToDataSetExtensionPoint.setsMap.remove( transMeta.getName() );
      }
    } );

    try {
      IMetaStore metaStore = transMeta.getMetaStore();

      if ( metaStore == null ) {
        return; // Nothing to do here, we can't reference data sets.
      }

      // Replace all steps with input data sets with Injector steps.
      // Replace all steps with a golden data set, attached to a unit test, with a Dummy
      // Apply tweaks
      //
      for ( final StepMeta stepMeta : trans.getTransMeta().getSteps() ) {

        // We might want to pass the data from this step into a data set all by itself...
        // For this we want to attach a row listener which writes the data.
        //
        StepMeta injectMeta = stepsMap.get( transMeta.getName() );
        if ( injectMeta != null && injectMeta.equals( stepMeta ) ) {
          final List<SourceToTargetMapping> mappings = mappingsMap.get( transMeta.getName() );
          final DataSet dataSet = setsMap.get( transMeta.getName() );
          if ( mappings != null && dataSet != null ) {
            passStepRowsToDataSet( trans, transMeta, stepMeta, mappings, dataSet );
          }
        }
      }
    } catch ( Throwable e ) {
      throw new KettleException( "Unable to pass rows to data set", e );
    }

  }


  private void passStepRowsToDataSet( final Trans trans, final TransMeta transMeta, final StepMeta stepMeta, final List<SourceToTargetMapping> mappings, final DataSet dataSet )
    throws KettleException {

    // This is the step to inject into the specified data set
    //
    final RowMetaInterface columnsRowMeta = dataSet.getSetRowMeta( true );

    StepInterface stepInterface = trans.findStepInterface( stepMeta.getName(), 0 );

    final List<Object[]> stepForDbRows = new ArrayList<>();

    stepInterface.addRowListener( new RowAdapter() {
      public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
        Object[] stepForDbRow = RowDataUtil.allocateRowData( columnsRowMeta.size() );
        for ( SourceToTargetMapping mapping : mappings ) {
          stepForDbRow[ mapping.getTargetPosition() ] = row[ mapping.getSourcePosition() ];
        }
        stepForDbRows.add( stepForDbRow );
      }
    } );

    // At the end of the transformation, write it...
    //
    trans.addTransListener( new TransAdapter() {
      @Override public void transFinished( Trans trans ) throws KettleException {

        // Write it
        //
        dataSet.getGroup().writeDataSetData( dataSet.getTableName(), columnsRowMeta, stepForDbRows );
      }
    } );

  }
} public class DataSetConst {
  private static Class<?> PKG = DataSetConst.class; // for i18n purposes, needed by Translator2!!

  public static final String DATABASE_LIST_KEY = "Databases";
  public static final String GROUP_LIST_KEY = "DataSetGroups";
  public static final String SET_LIST_KEY = "DataSets";

  // Variables during execution to indicate the selected test to run
  //
  public static final String VAR_RUN_UNIT_TEST = "__UnitTest_Run__";
  public static final String VAR_UNIT_TEST_NAME = "__UnitTest_Name__";
  public static final String VAR_WRITE_TO_DATASET = "__UnitTest_WriteDataSet__";
  public static final String VAR_DO_NOT_SHOW_UNIT_TEST_ERRORS = "__UnitTest_DontShowUnitTestErrors__";

  public static final String AREA_DRAWN_UNIT_TEST_ICON = "Drawn_UnitTestIcon";
  public static final String AREA_DRAWN_UNIT_TEST_NAME = "Drawn_UnitTestName";
  public static final String AREA_DRAWN_INPUT_DATA_SET = "Input_DataSet";
  public static final String AREA_DRAWN_GOLDEN_DATA_SET = "Golden_DataSet";


  public static final String ROW_COLLECTION_MAP = "RowCollectionMap";
  public static final String UNIT_TEST_RESULTS = "UnitTestResults";

  public static final String VARIABLE_UNIT_TESTS_BASE_PATH = "UNIT_TESTS_BASE_PATH";

  private static final String[] tweakDesc = new String[] {
    BaseMessages.getString( PKG, "DataSetConst.Tweak.NONE.Desc" ),
    BaseMessages.getString( PKG, "DataSetConst.Tweak.BYPASS_STEP.Desc" ),
    BaseMessages.getString( PKG, "DataSetConst.Tweak.REMOVE_STEP.Desc" ),
  };

  private static final String[] testTypeDesc = new String[] {
    BaseMessages.getString( PKG, "DataSetConst.TestType.NONE.Desc" ),
    BaseMessages.getString( PKG, "DataSetConst.TestType.CONCEPTUAL.Desc" ),
    BaseMessages.getString( PKG, "DataSetConst.TestType.DEVELOPMENT.Desc" ),
    BaseMessages.getString( PKG, "DataSetConst.TestType.UNIT_TEST.Desc" ),
  };


  public static final DataSetGroup findDataSetGroup( List<DataSetGroup> list, String dataSetGroupName ) {
    if ( StringUtil.isEmpty( dataSetGroupName ) ) {
      return null;
    }
    for ( DataSetGroup dataSetGroup : list ) {
      if ( dataSetGroupName.equals( dataSetGroup.getName() ) ) {
        return dataSetGroup;
      }
    }
    return null;
  }

  public static List<DatabaseMeta> getAvailableDatabases( Repository repository, SharedObjects sharedObjects ) throws KettleException {
    List<DatabaseMeta> list = new ArrayList<DatabaseMeta>();

    // Load database connections from the central repository if we're connected to one
    //
    if ( repository != null ) {
      ObjectId[] databaseIDs = repository.getDatabaseIDs( false );
      for ( ObjectId databaseId : databaseIDs ) {
        list.add( repository.loadDatabaseMeta( databaseId, null ) );
      }
    }

    // Also load from the shared objects file of the transformation
    //
    Collection<SharedObjectInterface> localSharedObjects = sharedObjects.getObjectsMap().values();

    for ( SharedObjectInterface localSharedObject : localSharedObjects ) {
      if ( localSharedObject instanceof DatabaseMeta ) {
        DatabaseMeta databaseMeta = (DatabaseMeta) localSharedObject;
        // Only add a local database if it doesn't exist in the central repository
        //
        if ( !list.contains( databaseMeta ) ) {
          list.add( databaseMeta );
        }
      }
    }

    return list;
  }

  public static final DataSet writeDataSet( String name, String description, DataSetGroup dataSetGroup, String tableName, List<DataSetField> fields, List<Object[]> dataRows ) throws KettleException {

    DataSet dataSet = new DataSet( name, description, dataSetGroup, tableName, fields );
    RowMetaInterface rowMeta = dataSet.getSetRowMeta( true );

    // Write the rows to the data set we just created...
    //
    dataSetGroup.writeDataSetData( tableName, rowMeta, dataRows );

    return dataSet;
  }

  /**
   * Validate the execution results of a transformation against the golden data sets of a unit test.
   *
   * @param trans     The transformation after execution
   * @param unitTest  The unit test
   * @param hierarchy The factories to load unit test and data set information
   * @param results   The results list to add comments to
   * @return The nr of errors, 0 if no errors found
   * @throws KettleException In case there was an error loading data or metadata.
   */
  public static final int validateTransResultAgainstUnitTest( Trans trans, TransUnitTest unitTest, FactoriesHierarchy hierarchy, List<UnitTestResult> results ) throws KettleException {
    int nrErrors = 0;

    LogChannelInterface log = trans.getLogChannel();

    @SuppressWarnings( "unchecked" )
    Map<String, RowCollection> collectionMap = (Map<String, RowCollection>) trans.getExtensionDataMap().get( DataSetConst.ROW_COLLECTION_MAP );
    if ( collectionMap == null ) {

      String comment = "No step output result data found to validate against";
      results.add( new UnitTestResult( trans.getName(), unitTest.getName(), null, null, false, comment ) );
      return nrErrors;
    }

    for ( TransUnitTestSetLocation location : unitTest.getGoldenDataSets() ) {

      // Sometimes we deleted a step and it's still in the list:
      // Simply skip that one
      //
      if ( trans.getTransMeta().findStep( location.getStepname() ) == null ) {
        continue;
      }

      int nrLocationErrors = 0;
      RowCollection resultCollection = collectionMap.get( location.getStepname() );
      if ( resultCollection == null || resultCollection.getRows() == null || resultCollection.getRowMeta() == null ) {
        // error occurred somewhere, we don't have results, provide dummy values to avoid exceptions, flag error
        //
        resultCollection = new RowCollection();
        resultCollection.setRowMeta( new RowMeta() );
        resultCollection.setRows( new ArrayList<Object[]>() );

        String comment = "WARNING: no test results found for step '" + location.getStepname() + "' : check disabled hops, input and so on.";
        results.add( new UnitTestResult(
          trans.getName(), unitTest.getName(), location.getDataSetName(), location.getStepname(),
          false, comment ) );
      }
      final RowMetaInterface resultRowMeta = resultCollection.getRowMeta();

      log.logDetailed( "Found " + resultCollection.getRows().size() + " results for data comparing in step '" + location.getStepname() + "', fields: " + resultRowMeta.toString() );

      DataSet goldenDataSet = unitTest.getGoldenDataSet( log, hierarchy, location );
      List<Object[]> goldenRows = goldenDataSet.getAllRows( log, location );
      RowMetaInterface goldenRowMeta = goldenDataSet.getMappedDataSetFieldsRowMeta( location );

      log.logDetailed( "Found " + goldenRows.size() + " golden rows '" + location.getStepname() + "', fields: " + goldenRowMeta );

      List<Object[]> resultRows = resultCollection.getRows();

      if ( resultRows.size() != goldenRows.size() ) {
        String comment =
          "Incorrect number of rows received from step, golden data set '" + location.getDataSetName() + "' has " + goldenRows.size() + " rows in it and we received " + resultRows.size();
        results.add( new UnitTestResult(
          trans.getName(), unitTest.getName(), location.getDataSetName(), location.getStepname(),
          true, comment ) );
        nrLocationErrors++;
      } else {

        // To compare the 2 data sets they need to be explicitly sorted on the same keys
        // The added problem is that the user provided a field mapping.
        // So for every "location field order" we need to find the step source field
        //
        // Sort step result rows
        //
        final int[] resultFieldIndexes = new int[ location.getFieldOrder().size() ];
        for ( int i = 0; i < resultFieldIndexes.length; i++ ) {
          String dataSetOrderField = location.getFieldOrder().get( i );
          String stepOrderField = location.findStepField( dataSetOrderField );
          if ( stepOrderField == null ) {
            throw new KettleException( "There is no step field provided in the mappings so I don't know which field to use to sort '" + dataSetOrderField + "'" );
          }
          resultFieldIndexes[ i ] = resultRowMeta.indexOfValue( stepOrderField );
          if ( resultFieldIndexes[ i ] < 0 ) {
            throw new KettleException( "Unable to find sort field '" + stepOrderField + "' in step results : " + Arrays.toString( resultRowMeta.getFieldNames() ) );
          }
        }
        try {
          log.logDetailed( "Sorting result rows collection on fields: " + location.getFieldOrder() );
          resultCollection.getRows().sort( new Comparator<Object[]>() {
            @Override public int compare( Object[] row1, Object[] row2 ) {
              try {
                return resultRowMeta.compare( row1, row2, resultFieldIndexes );
              } catch ( KettleValueException e ) {
                throw new RuntimeException( "Error comparing golden data result rows", e );
              }
            }
          } );
        } catch ( RuntimeException e ) {
          throw new KettleException( "Error sorting result rows for golden data set '" + location.getDataSetName() + "'", e );
        }

        // Print the first 10 result rows
        //
        if ( log.isDebug() ) {
          for ( int i = 0; i < 10 && i < resultCollection.getRows().size(); i++ ) {
            log.logDetailed( "Result row #" + ( i + 1 ) + " : " + resultRowMeta.getString( resultCollection.getRows().get( i ) ) );
          }
        }


        // Golden rows
        //
        final int[] goldenFieldIndexes = new int[ location.getFieldOrder().size() ];
        for ( int i = 0; i < goldenFieldIndexes.length; i++ ) {
          goldenFieldIndexes[ i ] = goldenRowMeta.indexOfValue( location.getFieldOrder().get( i ) );
          if ( goldenFieldIndexes[ i ] < 0 ) {
            throw new KettleException( "Unable to find sort field '" + location.getFieldOrder().get( i ) + "' in golden rows : " + Arrays.toString( goldenRowMeta.getFieldNames() ) );
          }
        }
        try {
          log.logDetailed( "Sorting golden rows collection on fields: " + location.getFieldOrder() );

          goldenRows.sort( new Comparator<Object[]>() {
            @Override public int compare( Object[] row1, Object[] row2 ) {
              try {
                return goldenRowMeta.compare( row1, row2, goldenFieldIndexes );
              } catch ( KettleValueException e ) {
                throw new RuntimeException( "Error comparing golden data set rows", e );
              }
            }
          } );
        } catch ( RuntimeException e ) {
          throw new KettleException( "Error sorting golden data rows for golden data set '" + location.getDataSetName() + "'", e );
        }

        // Print the first 10 golden rows
        //
        if ( log.isDebug() ) {
          for ( int i = 0; i < 10 && i < goldenRows.size(); i++ ) {
            log.logDetailed( "Golden row #" + ( i + 1 ) + " : " + goldenRowMeta.getString( goldenRows.get( i ) ) );
          }
        }

        if ( nrLocationErrors == 0 ) {
          final int[] stepFieldIndices = new int[ location.getFieldMappings().size() ];
          final int[] goldenIndices = new int[ location.getFieldMappings().size() ];
          for ( int i = 0; i < location.getFieldMappings().size(); i++ ) {
            TransUnitTestFieldMapping fieldMapping = location.getFieldMappings().get( i );

            stepFieldIndices[ i ] = resultRowMeta.indexOfValue( fieldMapping.getStepFieldName() );
            goldenIndices[ i ] = goldenRowMeta.indexOfValue( fieldMapping.getDataSetFieldName() );
            log.logDetailed( "Field to compare #" + i + " found on step index : " + stepFieldIndices[ i ] + ", golden index : " + goldenIndices[ i ] );
          }

          for ( int rowNumber = 0; rowNumber < resultRows.size(); rowNumber++ ) {
            Object[] resultRow = resultRows.get( rowNumber );
            Object[] goldenRow = goldenRows.get( rowNumber );

            // Now compare the input to the golden row
            //
            for ( int i = 0; i < location.getFieldMappings().size(); i++ ) {
              ValueMetaInterface stepValueMeta = resultCollection.getRowMeta().getValueMeta( stepFieldIndices[ i ] );
              Object stepValue = resultRow[ stepFieldIndices[ i ] ];

              ValueMetaInterface goldenValueMeta = goldenRowMeta.getValueMeta( goldenIndices[ i ] );
              Object goldenValue = goldenRow[ goldenIndices[ i ] ];

              if ( log.isDetailed() ) {
                log.logDebug( "Comparing Meta '" + stepValueMeta.toString() + "' with '" + goldenValueMeta.toString() + "'" );
                log.logDebug( "Comparing Value '" + stepValue + "' with '" + goldenValue + "'" );
              }

              Object goldenValueConverted;

              // sometimes there are data conversion issues because of the the database...
              //
              if ( goldenValueMeta.getType() == stepValueMeta.getType() ) {
                goldenValueConverted = goldenValue;
              } else {
                goldenValueConverted = stepValueMeta.convertData( goldenValueMeta, goldenValue );
              }

              try {
                int cmp = stepValueMeta.compare( stepValue, stepValueMeta, goldenValueConverted );
                if ( cmp != 0 ) {
                  if ( log.isDebug() ) {
                    log.logDebug( "Unit test failure: '" + stepValue + "' <> '" + goldenValue + "'" );
                  }
                  String comment = "Validation againt golden data failed for row number " + ( rowNumber + 1 )
                    + ": step value [" + stepValueMeta.getString( stepValue )
                    + "] does not correspond to data set value [" + goldenValueMeta.getString( goldenValue ) + "]";
                  results.add( new UnitTestResult(
                    trans.getName(), unitTest.getName(), location.getDataSetName(), location.getStepname(),
                    true, comment ) );
                  nrLocationErrors++;
                }
              } catch ( KettleValueException e ) {
                throw new KettleException( "Unable to compare step data against golden data set '" + location.getDataSetName() + "'", e );
              }
            }
          }
        }

        if ( nrLocationErrors == 0 ) {
          String comment = "Test passed succesfully against golden data set";
          results.add( new UnitTestResult(
            trans.getName(), unitTest.getName(), location.getDataSetName(), location.getStepname(),
            false, comment ) );
        } else {
          nrErrors += nrLocationErrors;
        }
      }
    }

    if ( nrErrors == 0 ) {
      String comment = "Test passed succesfully against unit test";
      results.add( new UnitTestResult(
        trans.getName(), unitTest.getName(), null, null,
        false, comment ) );

    }
    return nrErrors;
  }

  public static final String getDirectoryFromPath( String path ) {
    int lastSlashIndex = path.lastIndexOf( '/' );
    if ( lastSlashIndex >= 0 ) {
      return path.substring( 0, lastSlashIndex );
    } else {
      return "/";
    }
  }

  public static final String getNameFromPath( String path ) {
    int lastSlashIndex = path.lastIndexOf( '/' );
    if ( lastSlashIndex >= 0 ) {
      return path.substring( lastSlashIndex + 1 );
    } else {
      return path;
    }
  }

  public static RowMetaInterface getStepOutputFields( LogChannelInterface log, DataSet dataSet, TransUnitTestSetLocation inputLocation ) throws KettleException {
    RowMetaInterface dataSetRowMeta = dataSet.getSetRowMeta( false );
    RowMetaInterface outputRowMeta = new RowMeta();

    for ( int i = 0; i < inputLocation.getFieldMappings().size(); i++ ) {
      TransUnitTestFieldMapping fieldMapping = inputLocation.getFieldMappings().get( i );
      ValueMetaInterface injectValueMeta = dataSetRowMeta.searchValueMeta( fieldMapping.getDataSetFieldName() );
      if ( injectValueMeta == null ) {
        throw new KettleException( "Unable to find mapped field '" + fieldMapping.getDataSetFieldName() + "' in data set '" + dataSet.getName() + "'" );
      }
      // Rename to the step output names though...
      //
      injectValueMeta.setName( fieldMapping.getStepFieldName() );
      outputRowMeta.addValueMeta( injectValueMeta );
    }

    return outputRowMeta;
  }

  /**
   * Get the TransTweak for a tweak description (from the dialog)
   *
   * @param tweakDescription The description to look for
   * @return the tweak or NONE if nothing matched
   */
  public TransTweak getTweakForDescription( String tweakDescription ) {
    if ( StringUtils.isEmpty( tweakDescription ) ) {
      return TransTweak.NONE;
    }
    int index = Const.indexOfString( tweakDescription, tweakDesc );
    if ( index < 0 ) {
      return TransTweak.NONE;
    }
    return TransTweak.values()[ index ];
  }

  public static final String getTestTypeDescription( TestType testType ) {
    int index = 0; // NONE
    if ( testType != null ) {
      TestType[] testTypes = TestType.values();
      for ( int i = 0; i < testTypes.length; i++ ) {
        if ( testTypes[ i ] == testType ) {
          index = i;
          break;
        }
      }
    }

    return testTypeDesc[ index ];
  }

  /**
   * Get the TestType for a tweak description (from the dialog)
   *
   * @param testTypeDescription The description to look for
   * @return the test type or NONE if nothing matched
   */
  public static final TestType getTestTypeForDescription( String testTypeDescription ) {
    if ( StringUtils.isEmpty( testTypeDescription ) ) {
      return TestType.NONE;
    }
    int index = Const.indexOfString( testTypeDescription, testTypeDesc );
    if ( index < 0 ) {
      return TestType.NONE;
    }
    return TestType.values()[ index ];
  }

  public static final String[] getTestTypeDescriptions() {
    return testTypeDesc;
  }

} public class FactoriesHierarchy {
  private IMetaStore metaStore;
  private List<DatabaseMeta> databases;
  private MetaStoreFactory<DataSetGroup> groupFactory;
  private MetaStoreFactory<DataSet> setFactory;
  private MetaStoreFactory<TransUnitTest> testFactory;

  public FactoriesHierarchy( IMetaStore metaStore, List<DatabaseMeta> databases ) throws MetaStoreException {
    this.metaStore = metaStore;
    this.databases = databases;
    groupFactory = new MetaStoreFactory<DataSetGroup>( DataSetGroup.class, metaStore, PentahoDefaults.NAMESPACE );
    groupFactory.addNameList( DataSetConst.DATABASE_LIST_KEY, databases );
    List<DataSetGroup> groups = groupFactory.getElements();

    setFactory = new MetaStoreFactory<DataSet>( DataSet.class, metaStore, PentahoDefaults.NAMESPACE );
    setFactory.addNameList( DataSetConst.GROUP_LIST_KEY, groups );
    List<DataSet> sets = setFactory.getElements();

    setFactory.addNameFactory( DataSetConst.GROUP_LIST_KEY, groupFactory );

    testFactory = new MetaStoreFactory<TransUnitTest>( TransUnitTest.class, metaStore, PentahoDefaults.NAMESPACE );
    testFactory.addNameList( DataSetConst.SET_LIST_KEY, sets );

  }

  public IMetaStore getMetaStore() {
    return metaStore;
  }

  public void setMetaStore( IMetaStore metaStore ) {
    this.metaStore = metaStore;
  }

  public List<DatabaseMeta> getDatabases() {
    return databases;
  }

  public void setDatabases( List<DatabaseMeta> databases ) {
    this.databases = databases;
  }

  public MetaStoreFactory<DataSetGroup> getGroupFactory() {
    return groupFactory;
  }

  public void setGroupFactory( MetaStoreFactory<DataSetGroup> groupFactory ) {
    this.groupFactory = groupFactory;
  }

  public MetaStoreFactory<DataSet> getSetFactory() {
    return setFactory;
  }

  public void setSetFactory( MetaStoreFactory<DataSet> setFactory ) {
    this.setFactory = setFactory;
  }

  public MetaStoreFactory<TransUnitTest> getTestFactory() {
    return testFactory;
  }

  public void setTestFactory( MetaStoreFactory<TransUnitTest> testFactory ) {
    this.testFactory = testFactory;
  }
}
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.eclipse.swt.SWT;
import org.eclipse.swt.widgets.MessageBox;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.RowMetaAndData;
import org.pentaho.di.core.SourceToTargetMapping;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.gui.SpoonFactory;
import org.pentaho.di.core.logging.LogChannel;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaString;
import org.pentaho.di.core.util.StringUtil;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.dataset.DataSet;
import org.pentaho.di.dataset.DataSetField;
import org.pentaho.di.dataset.DataSetGroup;
import org.pentaho.di.dataset.TransTweak;
import org.pentaho.di.dataset.TransUnitTest;
import org.pentaho.di.dataset.TransUnitTestFieldMapping;
import org.pentaho.di.dataset.TransUnitTestSetLocation;
import org.pentaho.di.dataset.TransUnitTestTweak;
import org.pentaho.di.dataset.spoon.dialog.DataSetDialog;
import org.pentaho.di.dataset.spoon.dialog.DataSetGroupDialog;
import org.pentaho.di.dataset.spoon.dialog.EditRowsDialog;
import org.pentaho.di.dataset.spoon.dialog.TransUnitTestDialog;
import org.pentaho.di.dataset.spoon.xtpoint.TransMetaModifier;
import org.pentaho.di.dataset.spoon.xtpoint.WriteToDataSetExtensionPoint;
import org.pentaho.di.dataset.util.DataSetConst;
import org.pentaho.di.dataset.util.FactoriesHierarchy;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.shared.SharedObjectInterface;
import org.pentaho.di.shared.SharedObjects;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.ui.core.dialog.EnterMappingDialog;
import org.pentaho.di.ui.core.dialog.EnterSelectionDialog;
import org.pentaho.di.ui.core.dialog.ErrorDialog;
import org.pentaho.di.ui.core.dialog.SelectRowDialog;
import org.pentaho.di.ui.spoon.ISpoonMenuController;
import org.pentaho.di.ui.spoon.Spoon;
import org.pentaho.di.ui.spoon.trans.TransGraph;
import org.pentaho.metastore.api.IMetaStore;
import org.pentaho.metastore.api.exceptions.MetaStoreException;
import org.pentaho.metastore.persist.MetaStoreFactory;
import org.pentaho.metastore.stores.delegate.DelegatingMetaStore;
import org.pentaho.metastore.util.PentahoDefaults;
import org.pentaho.ui.xul.dom.Document;
import org.pentaho.ui.xul.impl.AbstractXulEventHandler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
    }
  }

  public void editDataSetGroup( String groupName ) throws MetaStoreException, KettleException {

    Spoon spoon = Spoon.getInstance();
    DelegatingMetaStore metaStore = spoon.getMetaStore();
    List<DatabaseMeta> databases = getAvailableDatabases( spoon.getRepository() );
    MetaStoreFactory<DataSetGroup> groupFactory = new MetaStoreFactory<DataSetGroup>( DataSetGroup.class, metaStore, PentahoDefaults.NAMESPACE );

    groupFactory.addNameList( DataSetConst.DATABASE_LIST_KEY, databases );
    DataSetGroup dataSetGroup = groupFactory.loadElement( groupName );

    DataSetGroupDialog groupDialog = new DataSetGroupDialog( spoon.getShell(), dataSetGroup, databases );
    while ( groupDialog.open() ) {
      String message = validateDataSetGroup( dataSetGroup, groupName, groupFactory.getElementNames() );

      // Save the group ...
      //
      if ( message == null ) {
        groupFactory.saveElement( dataSetGroup );
        break;
      } else {
        MessageBox box = new MessageBox( spoon.getShell(), SWT.OK );
        box.setText( "Error" );
        box.setMessage( message );
        box.open();
      }
    }

  }

  public void addDataSetGroup() {

    Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );

    IMetaStore metaStore = spoon.getMetaStore();
    MetaStoreFactory<DataSetGroup> groupFactory = new MetaStoreFactory<DataSetGroup>( DataSetGroup.class, metaStore, PentahoDefaults.NAMESPACE );

    try {

      DataSetGroup dataSetGroup = new DataSetGroup();
      List<DatabaseMeta> databases = getAvailableDatabases( spoon.getRepository() );

      DataSetGroupDialog groupDialog = new DataSetGroupDialog( spoon.getShell(), dataSetGroup, databases );
      while ( groupDialog.open() ) {
        // Verify empty name, existing name...
        //
        String message = validateDataSetGroup( dataSetGroup, null, groupFactory.getElementNames() );

        // Save the group again...
        //
        if ( message == null ) {
          groupFactory.saveElement( dataSetGroup );
          break;
        } else {
          MessageBox box = new MessageBox( spoon.getShell(), SWT.OK );
          box.setText( "Error" );
          box.setMessage( message );
          box.open();
        }
      }
    } catch ( Exception e ) {
      new ErrorDialog( spoon.getShell(), "Error", "Error creating a new data set group", e );
    }
  }

  public void removeDataSetGroup() {

    Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );

    IMetaStore metaStore = spoon.getMetaStore();
    MetaStoreFactory<DataSetGroup> groupFactory = new MetaStoreFactory<DataSetGroup>( DataSetGroup.class, metaStore, PentahoDefaults.NAMESPACE );

    try {

      List<String> groupNames = groupFactory.getElementNames();
      Collections.sort( groupNames );
      EnterSelectionDialog esd = new EnterSelectionDialog( spoon.getShell(), groupNames.toArray( new String[ groupNames.size() ] ), "Select the group", "Select the group to edit..." );
      String groupName = esd.open();
      if ( groupName != null ) {

        // TODO: Find the unit tests for this group, if there are any, we can't remove the group
        //
        groupFactory.deleteElement( groupName );
      }
    } catch ( Exception e ) {
      new ErrorDialog( spoon.getShell(), "Error", "Error retrieving the list of data set groups or deleting a group", e );
    }
  }


  private String validateDataSetGroup( DataSetGroup dataSetGroup, String previousName, List<String> groupNames ) {

    String message = null;

    String newName = dataSetGroup.getName();
    if ( StringUtil.isEmpty( newName ) ) {
      message = BaseMessages.getString( PKG, "DataSetHelper.DataSetGroup.NoNameSpecified.Message" );
    } else if ( !StringUtil.isEmpty( previousName ) && !previousName.equals( newName ) ) {
      message = BaseMessages.getString( PKG, "DataSetHelper.DataSetGroup.RenamingOfADataSetNotSupported.Message" );
    } else if ( StringUtil.isEmpty( previousName ) && Const.indexOfString( newName, groupNames ) >= 0 ) {
      message = BaseMessages.getString( PKG, "DataSetHelper.DataSetGroup.AGroupWithNameExists.Message", newName );
    } else if ( dataSetGroup.getType() == null ) {
      message = BaseMessages.getString( PKG, "DataSetHelper.DataSetGroup.NoGroupTypeSpecified.Message" );
    } else {
      switch ( dataSetGroup.getType() ) {
        case Database:
          if ( dataSetGroup.getDatabaseMeta() == null ) {
            message = BaseMessages.getString( PKG, "DataSetHelper.DataSetGroup.NoDatabaseSpecified.Message" );
          }
          break;
        case CSV:
          break;
      }
    }

    return message;
  }

  public static String validateDataSet( DataSet dataSet, String previousName, List<String> setNames ) {