public class TestUtils
{


    /** logger for this class */
    private static Logger myLog = LogManager.getLogger( TestUtils.class );

    private static final String TEST_PROPERTY_SOURCE_NAME = "PropertySourceForTest";

    /**
     * Removes all key/value pairs from environment
     * @param aEnv
     */
    public static void removeAllFromEnvironment( ConfigurableEnvironment aEnv )
    {
        MutablePropertySources propertySources = aEnv.getPropertySources();
        List<PropertySource<?>> toRemove = new ArrayList<PropertySource<?>>( );
        propertySources.forEach( p -> toRemove.add( p ) );
        toRemove.forEach( p -> propertySources.remove( p.getName() ) );
    }

    /**
     * Adds the given key/value pair to the environment into the PropertySourceForTest collection
     *
     * @param aEnv the environment
     * @param aKey the key
     * @param aValue the value
     */
    public static void addToEnvironment( ConfigurableEnvironment aEnv, String aKey, String aValue )
            throws SdiException
    {
        Map<String, Object> map = ConfigUtils.getOrCreatePropertySource( aEnv, TEST_PROPERTY_SOURCE_NAME );

        myLog.debug( "setting property " + aKey + " = " + aValue + " into the environment" );
        map.remove( aKey );
        map.put( aKey, aValue );
    }

    /**
     * Removes the given key from the environments collection PropertySourceForTest
     *
     * @param aEnv the environment
     * @param aKey the key
     */
    public static void removeFromEnvironment( ConfigurableEnvironment aEnv, String aKey )
            throws SdiException
    {
        Map<String, Object> map = ConfigUtils.getOrCreatePropertySource( aEnv, TEST_PROPERTY_SOURCE_NAME );

        myLog.debug( "removing property " + aKey + " from the environment" );
        map.remove( aKey );
    }

    /**
     * Replaces the value of given key in the environments collection PropertySourceForTest
     *
     * @param aEnv the environment
     * @param aKey the key
     * @param aValue the new value
     */
    public static void replaceInEnvironment( ConfigurableEnvironment aEnv, String aKey, String aValue )
            throws SdiException
    {
        Map<String, Object> map = ConfigUtils.getOrCreatePropertySource( aEnv, TEST_PROPERTY_SOURCE_NAME );

        myLog.debug( "replacing property " + aKey + " in the environment. New value: " + aValue  );
        map.remove( aKey );
        map.put( aKey, aValue );
    }

    /**
     * Outputs the environment to log output
     * @param aEnv
     */
    public static void debugPropertySources( ConfigurableEnvironment aEnv )
    {
        MutablePropertySources propertySources = aEnv.getPropertySources();
        for ( PropertySource<?> propertySource : propertySources )
        {
            myLog.debug( "PropertySource: " + propertySource );
        }
    }

} public class SdiException extends Exception
{
    private static final long serialVersionUID = 1L;

    public static final int EXIT_CODE_NO_ERROR = 0;
    public static final int EXIT_CODE_UNKNOWN_ERROR = 1;
    public static final int EXIT_CODE_PARSE_ERROR = 2;
    public static final int EXIT_CODE_CONFIG_ERROR = 3;
    public static final int EXIT_CODE_MAIL_ERROR = 4;
    public static final int EXIT_CODE_IMPORT_ERROR = 5;
    public static final int EXIT_CODE_FTP_ERROR = 6;
    public static final int EXIT_CODE_SSH_ERROR = 7;

    private int myExitCode;

    /**
     * Constructor
     *
     */
    public SdiException( int aExitCode )
    {
        super();
    }

    /**
     * Constructor
     *
     * @param aMessage
     */
    public SdiException( String aMessage, int aExitCode )
    {
        super( aMessage );
        myExitCode = aExitCode;
    }

    /**
     * Constructor
     *
     * @param aCause
     */
    public SdiException( Throwable aCause, int aExitCode )
    {
        super( aCause );
        myExitCode = aExitCode;
    }

    /**
     * Constructor
     *
     * @param aMessage
     * @param aCause
     */
    public SdiException( String aMessage, Throwable aCause, int aExitCode )
    {
        super( aMessage, aCause );
        myExitCode = aExitCode;
    }


    /**
     * @return exitCode
     */
    public int getExitCode()
    {
        return myExitCode;
    }

    /**
     * @param aThrowable
     * @return
     */
    public static SdiException toSdiException( Throwable aThrowable )
    {
        return toSdiException( aThrowable, SdiException.EXIT_CODE_UNKNOWN_ERROR );
    }

    /**
     * @param aThrowable
     * @return
     */
    public static SdiException toSdiException( Throwable aThrowable, int aErrorCode )
    {
        SdiException newEx;
        if ( aThrowable instanceof SdiException )
        {
            newEx = (SdiException) aThrowable;
        }
        else
        {
            newEx = new SdiException( "Throwable caught ", aThrowable, aErrorCode );
        } // if..else t instanceof SdiException

        return newEx;
    }

    @Override
    public String toString()
    {
        String result = super.toString();

        result += "; ExitCode: " + myExitCode;

        return result;
    }


} @Component
public class ConfigUtils
{

    /** logger for this class */
    private static Logger myLog = LogManager.getLogger( ConfigUtils.class );
    public static final String PROP_SOURCE_NAME_CMD_LINE = "CommandLineArguments";
    private static final String PROP_SOURCE_NAME_DYNAMIC = "DynamicProperties";
    public static final String KEY_PROP_OUTPUT_DIR = "dynamic.outputDir.file";

    // must be manually injected since static!
    private static ConversionService  myConversionService;

    /**
     * Converts the given classname into a properties file name according following rules:
     *
     * <ul>
     *     <li> if class is annotated with SdiProps the value of the annotation is used, suffixed by
     *          ".properties" (unless already present).</li>
     *     <li>if annotation is missing or no value configured:</li>
     *     <ul>
     *         <li> if class name ends with "Properties" this suffix will be truncated and replaced by
     *         ".properties"</li>
     *         <li> any other class name is used as is and suffixed with ".properties"</li>
     *     </ul>
     * </ul>
     *
     */
    public static String makePropertyResourceName( Class<?> aClass )
    {
        String found = null;

        SdiProps ann = aClass.getAnnotation( SdiProps.class );

        if ( ann != null )
        {
            found = ann.value();
        } // if ann != null

        if ( StringUtils.hasText( found ) )
        {
            if ( found.endsWith( ".properties" ) )
            {
                return found;
            } // if condition

            return found + ".properties";
        }

        String classname = aClass.getSimpleName();

        if ( classname.endsWith( "Properties" ) || classname.endsWith( "properties" ) )
        {
            return classname.substring( 0, classname.length() - "Properties".length() ) + ".properties";
        } // if condition

        return classname + ".properties";

    }

    /**
     * Tries to read the value from the given environment and to convert it to an int.
     * <p>
     * If the converstion fails, an excption is thrown.
     * <p>
     *
     * @param aEnv
     * @param aKey
     * @param aDefault
     * @return
     * @throws SdiException
     */
    public static int getIntProperty( Environment aEnv, String aKey ) throws SdiException
    {

        try
        {
            checkConversionServiceInitialized();
            return myConversionService.convert( aEnv.getRequiredProperty( aKey ), Integer.class );
        }
        catch ( Throwable t )
        {
            throw new SdiException( "No integer value found for property "+ aKey,
                                    t,
                                    SdiException.EXIT_CODE_CONFIG_ERROR );
        }
    }

    /**
     * @throws SdiException
     *
     */
    private static void checkConversionServiceInitialized() throws SdiException
    {
        if ( myConversionService == null )
        {
            myLog.warn( "ConversionService not initialized" );
            // Maybe forgotten in a unit test? Note: Without this check a forgotten injection in a unit
            // test would hardly be noticed if you get - e.g. - a int property with a default value
            throw new SdiException( "ConversionService not initialized", SdiException.EXIT_CODE_CONFIG_ERROR );
        } // if myConversionService == null

    }

    /**
     * Tries to read the value from the given environment and to convert it to an int.
     * <p>
     * If the converstion fails, the default value is returned.
     * <p>
     *
     * @param aEnv
     * @param aKey
     * @param aDefault
     * @return
     */
    public static int getIntProperty( Environment aEnv, String aKey, int aDefault )
    {
        try
        {
            checkConversionServiceInitialized();
            return myConversionService.convert( aEnv.getRequiredProperty( aKey ), Integer.class );
        }
        catch ( Throwable t )
        {
            return aDefault;
        }
    }

    /**
     * Tries to read the value from the given environment.
     * <p>
     * If the converstion fails, an excption is thrown.
     * <p>
     *
     * @param aEnv
     * @param aKey
     * @return
     * @throws SdiException
     */
    public static String getStringProperty( Environment aEnv,
                                            String aKey ) throws SdiException
    {
        try
        {
            return aEnv.getRequiredProperty( aKey );
        }
        catch ( Throwable t )
        {
            throw new SdiException( "No value found for property "+ aKey,
                                    t,
                                    SdiException.EXIT_CODE_CONFIG_ERROR );
        }
    }

    /**
     * Tries to read the value from the given environment and to convert it to a boolean.
     * <p>
     * If the conversion fails, an excption is thrown.
     * <p>
     *
     * @param aEnv
     * @param aKey
     * @return
     * @throws SdiException
     */
    public static boolean getBooleanProperty( Environment aEnv,
                                              String aKey ) throws SdiException
    {
        try
        {
            checkConversionServiceInitialized();
            return myConversionService.convert( aEnv.getRequiredProperty( aKey ), Boolean.class );
        }
        catch ( Throwable t )
        {
            throw new SdiException( "No boolean value found for property "+ aKey,
                                    SdiException.EXIT_CODE_CONFIG_ERROR );
        }
    }

    /**
     * Tries to read the value from the given environment and to convert it to a boolean.
     * <p>
     * If the converstion fails, the default value is returned.
     * <p>
     *
     * @param aEnv
     * @param aKey
     * @param aDefault
     * @return
     */
    public static boolean getBooleanProperty( Environment aEnv,
                                              String aKey,
                                              boolean aDefault )
    {
        try
        {
            return getBooleanProperty( aEnv, aKey );
        }
        catch ( Throwable t )
        {
            return aDefault;
        }
    }

    /**
     * Adds the given value to the environment, using the property source "DynamicProperties"
     * <p>
     * @param aEnv
     * @param aKey
     * @param aValue
     * @throws SdiException
     */
    public static void addToEnvironment( ConfigurableEnvironment aEnv,
                                         String aKey,
                                         Object aValue )
            throws SdiException
    {
        Map<String, Object> map = getOrCreatePropertySource( aEnv, PROP_SOURCE_NAME_DYNAMIC );

        myLog.debug( "setting property " + aKey + " = " + aValue + " into the environment" );
        map.remove( aKey );
        map.put( aKey, aValue );
    }

    /**
     * Removes the given key from the environment (property source "DynamicProperties")
     * <p>
     * @param aEnv
     * @param aKey
     * @throws SdiException
     */
    public static void removeFromEnvironment( ConfigurableEnvironment aEnv,
                                              String aKey )
            throws SdiException
    {
        Map<String, Object> map = getOrCreatePropertySource( aEnv, PROP_SOURCE_NAME_DYNAMIC );

        myLog.debug( "removing property " + aKey +  " from the environment" );
        map.remove( aKey );
    }

    /**
     * Tries to retrieve the named PropertySource from the environment. If not found it creates a
     * new one.
     * <p>
     *
     * @param aEnv
     * @param aPropertySourceName
     * @return the embedded property map
     * @throws SdiException if the found property source is not instance of PropertiesPropertySource
     */
    public static Map<String, Object> getOrCreatePropertySource( ConfigurableEnvironment aEnv,
                                                                 String aPropertySourceName ) throws SdiException
    {
        MutablePropertySources mps = aEnv.getPropertySources();
        PropertySource<?> ps = mps.get( aPropertySourceName );
        PropertiesPropertySource pps = null;

        if ( ps == null )
        {
            Properties props = new Properties();
            pps = new PropertiesPropertySource( aPropertySourceName, props );
            mps.addFirst( pps );
        }
        else
        {
            if ( !( ps instanceof PropertiesPropertySource ) )
            {
                throw new SdiException( "Found property source is not instance of PropertiesPropertySource "
                                        + " but: " + ps.getClass().getName(),
                                        SdiException.EXIT_CODE_CONFIG_ERROR );
            } // if !( ps instanceof PropertiesPropertySource )

            pps = (PropertiesPropertySource) ps;
        }

        Map<String,Object> map = pps.getSource();
        return map;
    }


    /**
     * Injects springs conversion service
     * <p>
     * @param  aConversionService
     *         myConversionService to set
     */
    public static void setConversionService( ConversionService aConversionService )
    {
        myConversionService = aConversionService;
    }

    /**
     * Assembles a set of property names starting with the given prefix.
     * <p>
     * The resulting set is sorted according the natural ordering of the full property name.
     * <p>
     * @param aEnv
     * @param aKeyPrefix
     * @return
     */
    public static Collection<String> getPropertyNamesStartingWith( ConfigurableEnvironment aEnv,
                                                                   String aKeyPrefix )
    {
        return getAllPropertyNames( aEnv ).stream()
            .filter( key -> key.startsWith( aKeyPrefix ) )
            .collect( Collectors.toCollection( (Supplier<Set<String>>) TreeSet::new ) );
    }

    /**
     * Returns a collection of all property names in the environment.
     * <p>
     * @param aEnv
     * @return
     */
    public static Collection<String> getAllPropertyNames( ConfigurableEnvironment aEnv )
    {
        Collection<String> result = new HashSet<>();
        aEnv.getPropertySources().forEach( ps -> result.addAll( getAllPropertyNames( ps ) ) );
        return result;
    }

    /**
     * Returns a collection of all property names in the given PropertySource.
     * <p>
     * @param aPropSource
     * @return
     */
    public static Collection<String> getAllPropertyNames( PropertySource<?> aPropSource )
    {
        Collection<String> result = new HashSet<>();

        if ( aPropSource instanceof CompositePropertySource)
        {
            CompositePropertySource cps = (CompositePropertySource) aPropSource;
            cps.getPropertySources().forEach( ps -> result.addAll( getAllPropertyNames( ps ) ) );
            return result;
        }

        if ( aPropSource instanceof EnumerablePropertySource<?> )
        {
            EnumerablePropertySource<?> ps = (EnumerablePropertySource<?>) aPropSource;
            Arrays.asList( ps.getPropertyNames() ).forEach( key -> result.add( key ) );
            return result;
        }

        // note: Most descendants of PropertySource are EnumerablePropertySource. There are some
        // few others like JndiPropertySource or StubPropertySource
        myLog.debug( "Given PropertySource is instanceof " + aPropSource.getClass().getName()
                     + " and cannot be iterated" );

        return result;
    }

}
 @Configuration
public class ConversionServiceProvider
{

    @Bean
    public ConversionService getConversionService()
    {
        ConversionServiceFactoryBean bean = new ConversionServiceFactoryBean();
        bean.setConverters( getConverters() );
        bean.afterPropertiesSet();
        ConversionService object = bean.getObject();
        return object;
    }

    private Set<Converter<?, ?>> getConverters()
    {
        Set<Converter<?, ?>> converters = new HashSet<Converter<?, ?>>();

        // here is the place to register custom converters

        return converters;
    }

} @SdiConverter( ConverterDate.CONVERTER_NAME )
@Component
@Scope(BeanDefinition.SCOPE_PROTOTYPE)
public class ConverterDate implements FieldConverter<Date>
{

    /** logger for this class */
    private Logger myLog = LogManager.getLogger( ConverterDate.class );

    public static final String CONVERTER_NAME = "toDate";
    public static final String PATTERN_SUFFIX = ".pattern";

    private String myDatePattern;

    /**
     * Constructor
     *
     */
    public ConverterDate()
    {
        super();
    }

    /**
     * @see ch.sdi.core.intf.FieldConverter#init(org.springframework.core.env.Environment, java.lang.String)
     */
    @Override
    public ConverterDate init( Environment aEnv, String aFieldname ) throws SdiException
    {
        String pattern = aEnv.getProperty( SdiMainProperties.KEY_PREFIX_CONVERTER
                                          + CONVERTER_NAME + "." + aFieldname + PATTERN_SUFFIX );
        if ( !StringUtils.hasText( pattern ) )
        {
            myLog.trace( "No pattern found for field " + aFieldname + ". Looking for a generic pattern" );
            pattern = aEnv.getProperty( SdiMainProperties.KEY_PREFIX_CONVERTER
                                       + CONVERTER_NAME + PATTERN_SUFFIX  );
        } // if !StringUtils.hasText( pattern )

        pattern = pattern.trim();

        myLog.debug( "found pattern for ConverterDate: " + pattern );
        setDatePattern( pattern );
        return this;
    }

    /**
     * @see ch.sdi.core.intf.FieldConverter#convert(java.lang.String)
     */
    @Override
    public Date convert( String aValue ) throws SdiException
    {
        if ( !StringUtils.hasText( aValue ) )
        {
            myLog.debug( "Given value is null" );
            return null;
        }

        DateFormat df;
        if ( StringUtils.hasText( myDatePattern ) )
        {
            df = new SimpleDateFormat( myDatePattern );
        }
        else
        {
            df = new SimpleDateFormat();
        } // if..else StringUtils.hasText( myPattern )

        Date result;

        try
        {
            result = df.parse( aValue );
        }
        catch ( ParseException t )
        {
            throw new SdiException( "Problems converting date: " + aValue
                                    + "; myDatePattern: " + myDatePattern,
                                    SdiException.EXIT_CODE_PARSE_ERROR );
        }

        return result;
    }


    /**
     * @return datePattern
     */
    public String getDatePattern()
    {
        return myDatePattern;
    }


    /**
     * @param  aDatePattern
     *         datePattern to set
     */
    public void setDatePattern( String aDatePattern )
    {
        myDatePattern = aDatePattern;
    }

} @Component
public class ConverterFactory
{

    /** logger for this class */
    private Logger myLog = LogManager.getLogger( ConverterFactory.class );

    @Autowired
    private ApplicationContext myAppCtxt;

    /**
     * Looks up the configured converters for the given collector fieldnames.
     * <p>
     * If for a particular field no configured is configured the standard ConverterString() (which
     * does nothing) is returned.
     * <p>
     *
     * @param aFieldnames
     * @return a list of appropriate converters, ordered in the same order as the given fieldnames
     * @throws SdiException if one of the configured converters cannot be found
     */
    public List<FieldConverter<?>> getFieldConverters( Collection<String> aFieldnames ) throws SdiException
    {
        List<FieldConverter<?>> result = new ArrayList<FieldConverter<?>>();

        for ( String fieldname : aFieldnames )
        {
            FieldConverter<?> converter = getConverterFor( fieldname );
            myLog.trace( "Converter for field " + fieldname + ": " + converter.getClass().getSimpleName() );
            result.add( converter );
        }

        return result;
    }

    /**
     * Looks up the configured converter for a collector fieldname.
     * <p>
     * If the fieldname has no configured converter the standard ConverterString() (which does nothing) is
     * returned.
     * <p>
     *
     * @param aFieldname must not be empty
     * @return the appropriate converter
     * @throws SdiException if the configured converter cannot be found
     */
    public FieldConverter<?> getConverterFor( String aFieldname ) throws SdiException
    {
        if ( !StringUtils.hasText( aFieldname ) )
        {
            throw new SdiException( "Given fieldname is empty!",
                                    SdiException.EXIT_CODE_UNKNOWN_ERROR );

        } // if !StringUtils.hasText( aFieldname )

        myLog.debug( "Looking up converter for field " + aFieldname );

        Environment env = myAppCtxt.getEnvironment();
        String converterName = env.getProperty( SdiMainProperties.KEY_PREFIX_CONVERTER + aFieldname );

        if ( !StringUtils.hasText( converterName ) )
        {
            return new ConverterString().init( env, aFieldname );
        } // if !StringUtils.hasText( converterName )

        myLog.debug( "Looking up converter with name " + converterName );

        Map<String, Object>  beans = myAppCtxt.getBeansWithAnnotation( SdiConverter.class );

        myLog.trace( "Found candidates for converter: " + beans.keySet() );

        for ( Object bean : beans.values() )
        {
            if ( !( bean instanceof FieldConverter ) )
            {
                throw new SdiException( "Bean annotated with @SdiConverter which is not a FieldConverter, but: "
                        + bean.getClass().getName(),
                        SdiException.EXIT_CODE_CONFIG_ERROR );
            } // if !bean instanceof FieldConverter

            FieldConverter<?> converter = (FieldConverter<?>) bean;

            SdiConverter annotation = bean.getClass().getAnnotation( SdiConverter.class );
            String value = annotation.value();

            if ( converterName.equals( value ) )
            {
                return converter.init( env, aFieldname );
            } // if converterName.equals( value )
        }

        throw new SdiException( "Converter " + converterName + " for field '" + aFieldname + "' not found",
                                SdiException.EXIT_CODE_CONFIG_ERROR );
    }



} @SdiConverter( ConverterJpgFromHexDump.CONVERTER_NAME )
@Component
//@Scope(BeanDefinition.SCOPE_PROTOTYPE)
public class ConverterJpgFromHexDump extends ConverterImage
{

    /** logger for this class */
    private Logger myLog = LogManager.getLogger( ConverterJpgFromHexDump.class );
    public static final String CONVERTER_NAME = "toJpgFromHexDump";

    /**
     * @see ch.sdi.core.intf.FieldConverter#init(org.springframework.core.env.Environment, java.lang.String)
     */
    @Override
    public FieldConverter<BufferedImage> init( Environment aEnv, String aFieldname ) throws SdiException
    {
        return this;
    }


    /**
     * @see ch.sdi.core.intf.FieldConverter#convert(java.lang.String)
     */
    @Override
    public BufferedImage convert( String aValue ) throws SdiException
    {
        if ( !StringUtils.hasText( aValue ) )
        {
            myLog.debug( "Given value is null" );
            return null;
        }

        byte[] binary;

        try
        {
            binary = Hex.decodeHex( aValue.toCharArray() );
        }
        catch ( DecoderException t )
        {
            throw new SdiException( "Cannot convert input value to binary array: " + aValue, t,
                                    SdiException.EXIT_CODE_PARSE_ERROR );
        }

        ByteArrayInputStream bais = new ByteArrayInputStream( binary );

        return super.convert( bais );
    }

} @Component
public class FilterFactory
{

    /** logger for this class */
    private Logger myLog = LogManager.getLogger( FilterFactory.class );


    @Autowired
    private ApplicationContext myAppCtxt;

    /**
     * Instantiates the filter described by the given param.
     * <p>
     * The param is expected to be:
     * <pre>
     *     &lt;filterName>:&lt;fieldname>[:additionalParams]
     * </pre>
     * The number of additional parameters is dependant of the particular filter type.
     * <p>
     * @param aParams
     * @return the initialized filter
     * @throws SdiException on any problem
     */
    public CollectFilter<?> getFilter( String aParams ) throws SdiException
    {
        if ( !StringUtils.hasText( aParams ) )
        {
            throw new SdiException( "Given param is empty!",
                                    SdiException.EXIT_CODE_CONFIG_ERROR );

        }

        String[] values = aParams.split( ":", 3 );

        if ( values.length < 2 )
        {
            throw new SdiException( "Given param does not contain at least 2 parts: " + aParams,
                                    SdiException.EXIT_CODE_CONFIG_ERROR );
        }

        String filterName = values[0];
        String fieldName = values[1];
        String params = values.length == 3 ? values[2] : null;

        myLog.debug( "Looking up filter " + filterName + " for field " + fieldName );

        Map<String, Object>  beans = myAppCtxt.getBeansWithAnnotation( SdiFilter.class );

        myLog.trace( "Found candidates for filters: " + beans.keySet() );

        for ( Object bean : beans.values() )
        {
            SdiFilter annotation = bean.getClass().getAnnotation( SdiFilter.class );
            String value = annotation.value();

            if ( filterName.equals( value ) )
            {
                if ( !( bean instanceof CollectFilter ) )
                {
                    throw new SdiException( "Found Bean annotated with @SdiFilter which is not a CollectFilter, but: "
                            + bean.getClass().getName(),
                            SdiException.EXIT_CODE_CONFIG_ERROR );
                }

                CollectFilter<?> filter = (CollectFilter<?>) bean;

                return filter.init( fieldName, params );
            }
        }

        throw new SdiException( "No filter found for filtername " + filterName,
                                SdiException.EXIT_CODE_CONFIG_ERROR );
    }


} @Component
public class CsvParser
{


    /** logger for this class */
    private Logger myLog = LogManager.getLogger( CsvParser.class );

    /**
     * Parses the given input stream.
     * <p>
     *
     * @param aInputStream
     *        must not be null
     * @param aDelimiter
     *        must not be null
     * @param aEncoding
     *        The encoding to be used. If null or empty, the systems default encoding is used.
     * @return a list which contains a list for each found person. The inner list contains the found
     *         values for this person. The number and the order must correspond to the configured field
     *         name list (see
     *         in each line.
     * @throws SdiException
     */
    public List<List<String>> parse( InputStream aInputStream,
                                     String aDelimiter,
                                     String aEncoding ) throws SdiException
    {
        return parse( aInputStream, aDelimiter, aEncoding, null );
    }

    /**
     * Parses the given input stream.
     * <p>
     *
     * @param aInputStream
     *        must not be null
     * @param aDelimiter
     *        must not be null
     * @param aEncoding
     *        The encoding to be used. If null or empty, the systems default encoding is used.
     * @return a list which contains a list for each found person. The inner list contains the found
     *         values for this person. The number and the order must correspond to the configured field
     *         name list (see
     *         in each line.
     * @throws SdiException
     */
    public List<List<String>> parse( InputStream aInputStream,
                                     String aDelimiter,
                                     String aEncoding,
                                     List<RawDataFilterString> aFilters ) throws SdiException
    {
        if ( !StringUtils.hasLength( aDelimiter ) )
        {
            throw new SdiException( "Delimiter not set", SdiException.EXIT_CODE_CONFIG_ERROR );
        } // if myDelimiter == null

        try
        {
            myLog.debug( "Using encoding " + aEncoding );

            BufferedReader br = new BufferedReader( !StringUtils.hasText( aEncoding )
                                                    ? new InputStreamReader( aInputStream )
                                                    : new InputStreamReader( aInputStream, aEncoding ) );
            List<List<String>> result = new ArrayList<>();
            Collection<String> myLinesFiltered = new ArrayList<>();

            int lineNo = 0;
            String line;
            LineLoop:
            while ( ( line = br.readLine() ) != null )
            {
                lineNo++;

                if ( aFilters != null )
                {
                    for ( RawDataFilterString filter : aFilters )
                    {
                        if ( filter.isFiltered( line ) )
                        {
                            myLog.debug( "Skipping commented line: " + line );
                            myLinesFiltered.add( line );
                            continue LineLoop;
                        }
                    }
                }

                myLog.debug( "Parsing line " + lineNo + ": " + line );

                List<String> list = new ArrayList<String>();
                Scanner sc = new Scanner( line );
                try
                {
                    sc.useDelimiter( aDelimiter );
                    while ( sc.hasNext() )
                    {
                        list.add( sc.next() );
                    }

                    // Note: if the line is terminated by the delimiter (last entry not present, the last entry
                    // will not appear in the scanned enumeration. Check for this special case:
                    if ( line.endsWith( aDelimiter ) )
                    {
                        list.add( "" );
                    } // if line.endsWith( aDelimiter )
                }
                finally
                {
                    sc.close();
                }

                result.add( list );
            }

            myLog.info( new ReportMsg( ReportMsg.ReportType.PREPARSE_FILTER, "Filtered lines",
                                       myLinesFiltered ) );

            return result;
        }
        catch ( Throwable t )
        {
            throw new SdiException( "Problems while parsing CSV file",
                                    t,
                                    SdiException.EXIT_CODE_PARSE_ERROR );
        }
    }

}
 public interface CollectorResult
{
    public Collection<String> getFieldnames();
    public Collection<Dataset> getRows();

} @SdiProps( "sdimain" )
public interface SdiMainProperties extends SdiProperties
{
    public static final String USER_OVERRIDE_PREFIX = "user.";
    public static final String SDI_PROPERTIES_RESOURCE_NAME = "Sdi.properties";

    public static final String KEY_DRYRUN = "sdi.dry-run";
    public static final String KEY_CHECK_DUPLICATE_ON_DRYRUN = "sdi.checkDuplicateOnDryRun";
    public static final String KEY_PROPERTIESOVERRIDE_INCLUDEROOT =
            "sdi.propertiesoverride.includeroot";
    public static final String KEY_OUTPUT_DIR = "sdi.outputDir";

    public static final String KEY_COLLECT_TYPE = "sdi.collect.type";

    public static final String KEY_COLLECT_CSV_FILENAME = "sdi.collect.csv.filename";
    public static final String KEY_COLLECT_CSV_DELIMITER = "sdi.collect.csv.delimiter";
    public static final String KEY_COLLECT_CSV_ENCODING = "sdi.collect.csv.encoding";
    public static final String KEY_COLLECT_CSV_HEADER_ROW = "sdi.collect.csv.headerrow";
    public static final String KEY_COLLECT_CSV_SKIP_AFTER_HEADER = "sdi.collect.csv.skipafterheader";
    public static final String KEY_COLLECT_CSV_FIELD_NAMES = "sdi.collect.csv.fieldnames";
    public static final String KEY_COLLECT_CSV_COMMENT_CHARS_PREFIX = "sdi.collect.csv.comment.";

    public static final String KEY_PREFIX_CONVERTER = "sdi.converter.";

    public static final String KEY_TARGET_IGNORE_FAILED_PERSON = "sdi.target.ignoreFailedPerson";

    public static final String KEY_PREFIX_TARGET_JOB = "sdi.targetjob.";

    public static final String KEY_MAIL_JOB = KEY_PREFIX_TARGET_JOB + "mail";
    public static final String KEY_SQL_JOB = KEY_PREFIX_TARGET_JOB + "sql";
    public static final String KEY_FTP_JOB = KEY_PREFIX_TARGET_JOB + "ftp";

    public static final String KEY_SQL_CONNECT_STRING = KEY_SQL_JOB + ".connectString";
    public static final String KEY_SQL_USER = KEY_SQL_JOB + ".user";
    public static final String KEY_SQL_PASSWORD = KEY_SQL_JOB + ".password";

    public static final String KEY_TARGET_HOST = "sdi.target.host";

    public static final String KEY_FTP_DEST_DIR = KEY_FTP_JOB + ".destinationDir";
    public static final String KEY_FTP_CMD_LINE = KEY_FTP_JOB + ".cmdLine";
    public static final String KEY_FTP_USER = KEY_FTP_JOB + ".user";
    public static final String KEY_FTP_PASSWORD = KEY_FTP_JOB + ".password";

    public static final String KEY_SSH_USER = "sdi.ssh.user";
    public static final String KEY_SSH_PASSWORD = "sdi.ssh.password";
    public static final String KEY_SSH_PORT = "sdi.ssh.port";
    public static final String KEY_SSH_CHECK_CERTIFICATE = "sdi.ssh.checkcertificate";

    public static final String KEY_TARGET_NAME = "sdi.target.name";
    public static final String KEY_NORMALIZE_IGNORE_FIELD = "sdi.normalize.ignoreField";




}
import java.io.File;
import java.net.URL;
import java.util.Collection;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import ch.sdi.core.TestUtils;
import ch.sdi.core.exc.SdiException;
import ch.sdi.core.impl.cfg.ConfigUtils;
import ch.sdi.core.impl.cfg.ConversionServiceProvider;
import ch.sdi.core.impl.data.converter.ConverterDate;
import ch.sdi.core.impl.data.converter.ConverterFactory;
import ch.sdi.core.impl.data.converter.ConverterJpgFromHexDump;
import ch.sdi.core.impl.data.filter.FilterFactory;
import ch.sdi.core.impl.parser.CsvParser;
import ch.sdi.core.intf.CollectorResult;
import ch.sdi.core.intf.SdiMainProperties;
/**
 * Copyright (c) 2014 by the original author or authors.
 *
 * This code is free software; you can redistribute it and/or modify it under the terms of the
 * GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */


package ch.sdi.core.impl.data;





/**
 * Testcase
 *
 * @version 1.0 (09.11.2014)
 * @author  Heri
 */
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes={CsvCollector.class,
                               CsvParser.class,
                               InputCollectorFactory.class,
                               ConverterFactory.class,
                               ConverterDate.class,
                               ConverterJpgFromHexDump.class,
                               ConversionServiceProvider.class,
                               FilterFactory.class })
public class CsvCollectorTest
{

    /** logger for this class */
    private Logger myLog = LogManager.getLogger( CsvCollectorTest.class );
    @Autowired
    private ConfigurableEnvironment myEnv;
    @Autowired
    private CsvCollector myClassUnderTest;
    @Autowired
    private ConversionService  myConversionService;



    /**
     * @throws java.lang.Exception
     */
    @Before
    public void setUp() throws Exception
    {
        ConfigUtils.setConversionService( myConversionService );

        TestUtils.addToEnvironment( myEnv, "sdi.converter.avatar", ConverterJpgFromHexDump.CONVERTER_NAME  );
        TestUtils.addToEnvironment( myEnv, "sdi.converter.birthday", ConverterDate.CONVERTER_NAME  );
        TestUtils.addToEnvironment( myEnv, "sdi.converter." + ConverterDate.CONVERTER_NAME
                                    + ".birthday.pattern", "dd.MM.yy"  );
        TestUtils.addToEnvironment( myEnv, "sdi.converter.entrydate",
                                    ConverterDate.CONVERTER_NAME  );
        TestUtils.addToEnvironment( myEnv, "sdi.converter." + ConverterDate.CONVERTER_NAME
                                    + ".entrydate.pattern", "yyyy-MM-dd mm:ss"  );

    }


    /**
     * @param aFilename
     * @return
     */
    private File toFile( String aFilename )
    {
        URL url = ClassLoader.getSystemResource( aFilename );
        File file = new File( url.getPath() );
        return file;
    }

    /**
     * Test method for {@link ch.sdi.core.impl.data.CsvCollector#execute()}.
     */
    @Test
    public void testExecute() throws Throwable
    {
        myLog.debug( "delimiter not yet set in environment" );
        testException();

        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_DELIMITER, ";" );

        myLog.debug( "filename not yet set in environment" );
        testException();

        myLog.debug( "wrong filename in environment" );
        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME, "blabla" );
        testException();

        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                    toFile( "CSV_testdata_noHeader.csv" ).getCanonicalPath() );

        myLog.debug( "fieldnames not yet set in environment" );
        testException();

        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES,
                                    "Screenname,Name,Middlename,Prename,email,birthday,entrydate" );

        myLog.debug( "fieldnames now set in environment" );
        testSuccess( 7, 2 );

        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "2" );
        myLog.debug( "skip >= rowsize. Ignored because headerRow is false" );
        testSuccess( 7, 2 );


        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" );
        myLog.debug( "skip >= rowsize. HeaderRow=true -> Exception" );
        testException();

        // CSV_testdata_withHeader0.csv:
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" );
        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                    toFile( "CSV_testdata_withHeader0.csv" ).getCanonicalPath() );
        TestUtils.removeFromEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES );

        myLog.debug( "CSV_testdata_withHeader0.csv success" );
        testSuccess( 7, 2 );

        // CSV_testdata_withHeader2.csv:
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "2" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                    toFile( "CSV_testdata_withHeader2.csv" ).getCanonicalPath() );
        myLog.debug( "CSV_testdata_withHeader2.csv success" );
        testSuccess( 7, 2 );

        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "5" );
        myLog.debug( "skip >= rowsize. HeaderRow=true -> Exception" );
        testException();

        // CSV-File with too many entries
        TestUtils.addToEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES,
                "Screenname,Name,Middlename,Prename,email,birthday" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "false" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                        toFile( "CSV_testdata_noHeader.csv" ).getCanonicalPath() );
        myLog.debug( "CSV-File with too many entries -> Exception" );
        testException();

        // empty field content
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FIELD_NAMES,
                "Screenname,Name,Middlename,Prename,email,birthday,entrydate" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_SKIP_AFTER_HEADER, "0" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "false" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                        toFile( "CSV_testdata_noHeader_empty_fields.csv" ).getCanonicalPath() );
        myLog.debug( "loading avatar picture (jpg hex dump)" );
        testSuccess( 7, 2 );

        // avatar
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_HEADER_ROW, "true" );
        TestUtils.replaceInEnvironment( myEnv, SdiMainProperties.KEY_COLLECT_CSV_FILENAME,
                                        toFile( "CSV_testdata_withHeader0_Avatar.csv" ).getCanonicalPath() );
        myLog.debug( "empty field content -> Empty String or null (if date)" );
        testSuccess( 8, 2 );

    }

    /**
     *
     */
    private void testException()
    {
        try
        {
            myClassUnderTest.execute();
            Assert.fail( "Exception expected" );
        }
        catch ( SdiException t )
        {
            myLog.debug( "Expected exception received. Message: " + t.getMessage() );
        }
    }

    /**
    *
    */
   private void testSuccess( int aExpectedFieldNum, int aExpectedRowNum ) throws Throwable
   {
       CollectorResult actual = myClassUnderTest.execute();