public class Alert {
    private String str_PluginName;   
    private String str_GraphName;
    private Integer node_id;
    private Integer alert_id;
    private BigDecimal raise_value = new BigDecimal("0");
    private String condition;
    private Integer num_samples;
    // last alert (unix timestamp)
    private int last_alert;
    // alert limit in minutes, do not send alert again if not at least x time passed between last alert
    private int alert_limit = 0;
    private CopyOnWriteArrayList<AlertValue> v_values = new CopyOnWriteArrayList<>();
    private String alertMsg;
    private String hostname;
    
    
    /** and a new value set and check if alert condition is raised and if we need to send notifications 
     * 
     * @param p_timestamp timestamp
     * @param p_value value of metric
     */
    public void addAndCheckSample(int p_timestamp, BigDecimal p_Strvalue)
    {
        try
        {

            boolean sendAlert = false;
            AlertValue av = new AlertValue();
            av.setTimestamp(p_timestamp);
            av.setValue(p_Strvalue);
            v_values.add(av);
            BigDecimal p_value = av.getValue();
            // if we have higher sample count now, remove first
            if(v_values.size() > num_samples)
            {
                v_values.remove(0);
            }
            
            // now do the check magic
            BigDecimal avg = returnAvgBig();
            // value equal given value? alert
            switch (condition) {
                case "eq":
                    if(p_value.compareTo(raise_value) == 0)
                    {
                        this.setAlertMsg("Plugin: " + str_PluginName + " Graph: " + str_GraphName + " Host: " + hostname + " value is: " + p_value + " with alert condition => equal: "+raise_value);
                        sendAlert = true;
                    }
                    break;
                case "lt":
                    if(p_value.compareTo(raise_value) == -1)
                    {
                        this.setAlertMsg("Plugin: " + str_PluginName + " Graph: " + str_GraphName + " Host: " + hostname + " value is: " + p_value + " with alert condition => less then: "+raise_value);
                        sendAlert = true;
                    }
                    break;     
                case "gt":
                    if(p_value.compareTo(raise_value) == 1)
                    {
                        this.setAlertMsg("Plugin: " + str_PluginName + " Graph: " + str_GraphName + " Host: " + hostname + " value is: " + p_value + " with alert condition => greater then: "+raise_value);
                        sendAlert = true;
                    }
                    break; 
                case "gtavg":
                    if(avg.compareTo(raise_value) == 1)
                    {
                        this.setAlertMsg("Plugin: " + str_PluginName + " Graph: " + str_GraphName + " Host: " + hostname + " avg value is: " + avg + " with alert condition => avg of "+num_samples+" samples greater then: "+raise_value);
                        sendAlert = true;
                    }
                    break;  
                case "ltavg":
                   if(avg.compareTo(raise_value) == -1)
                    {           
                        this.setAlertMsg("Plugin: " + str_PluginName + " Graph: " + str_GraphName + " Host: " + hostname + " avg value is: " + avg + " with alert condition => avg of "+num_samples+" samples less then: "+raise_value);
                        sendAlert = true;
                    }
                    break;                  
            }

            // check if we need to alert
            if(sendAlert)
            {
                if(getAlert_limit() > 0)
                {
                    int curTime = getUnixtime();
                    // convert alert_time to seconds
                    int alim = getAlert_limit() * 60;
                    int lalert = last_alert + alim;
                    if(lalert < curTime )
                    {
                        setLast_alert(getUnixtime());
                        sendNotifications(this);
                        
                    }
                }
                else
                {
                    setLast_alert(getUnixtime());
                    sendNotifications(this);
                    
                }

            }
        } catch (Exception ex)
        {
            com.clavain.muninmxcd.logger.error("Error in Alert/addAndCheckSample (recv value: "+p_Strvalue+"): " + ex.getLocalizedMessage());
        }
    }
    
    private BigDecimal returnAvgBig() {
       BigDecimal numbers = new BigDecimal(v_values.size());     
       BigDecimal retval = new BigDecimal(0);
       for (AlertValue l_av : v_values) {
          // retval += l_av.doubleValue();
          retval = retval.add(l_av.getValue());
       }
       BigDecimal average = retval;
       average = average.divide(new BigDecimal(v_values.size()),2,RoundingMode.HALF_UP);
       return average;
    }
    
    private double returnAvg()
    {
       double[] numbers = new double[v_values.size()];     
       double retval = 0;
       for (AlertValue l_av : v_values) {
           retval += l_av.getValue().doubleValue();
       }
       double average = retval / v_values.size();
       return average;
    }    
    
    /**
     * @return the str_PluginName
     */
    public String getPluginName() {
        return str_PluginName;  
    }

    /**
     * @param str_PluginName the str_PluginName to set
     */
    public void setPluginName(String str_PluginName) {
        this.str_PluginName = str_PluginName;
    }

    /**
     * @return the str_GraphName
     */
    public String getGraphName() {
        return str_GraphName;
    }

    /**
     * @param str_GraphName the str_GraphName to set
     */
    public void setGraphName(String str_GraphName) {
        this.str_GraphName = str_GraphName;
    }

    /**
     * @return the node_id
     */
    public Integer getNode_id() {
        return node_id;
    }

    /**
     * @param node_id the node_id to set
     */
    public void setNode_id(Integer node_id) {
        this.node_id = node_id;
    }

    /**
     * @return the alert_id
     */
    public Integer getAlert_id() {
        return alert_id;
    }

    /**
     * @param alert_id the alert_id to set
     */
    public void setAlert_id(Integer alert_id) {
        this.alert_id = alert_id;
    }

    /**
     * @return the raise_value
     */
    public Double getRaise_value() {
        return raise_value.doubleValue();
    }

    /**
     * @param raise_value the raise_value to set
     */
    public void setRaise_value(BigDecimal raise_value) {
        this.raise_value = raise_value;
    }

    /**
     * @return the condition
     */
    public String getCondition() {
        return condition;
    }

    /**
     * @param condition the condition to set
     */
    public void setCondition(String condition) {
        this.condition = condition;
    }

    /**
     * @return the v_values
     */
    public CopyOnWriteArrayList<AlertValue> getValueList() {
        return v_values;
    }

    /**
     * @param v_values the v_values to set
     */
    public void setValueList(CopyOnWriteArrayList<AlertValue> v_values) {
        this.v_values = v_values;
    }

    /**
     * @return the num_samples
     */
    public Integer getNum_samples() {
        return num_samples;
    }

    /**
     * @param num_samples the num_samples to set
     */
    public void setNum_samples(Integer num_samples) {
        this.num_samples = num_samples;
    }

    /**
     * @return the alertMsg
     */
    public String getAlertMsg() {
        return alertMsg;
    }

    /**
     * @param alertMsg the alertMsg to set
     */
    public void setAlertMsg(String alertMsg) {
        this.alertMsg = alertMsg;
    }

    /**
     * @return the hostname
     */
    public String getHostname() {
        if(hostname == null)
        {
            this.setHostname(getNodeHostNameForMuninNode(this.node_id));
        }
        return hostname;
    }

    /**
     * @param hostname the hostname to set
     */
    public void setHostname(String hostname) {
        this.hostname = hostname;
    }

    /**
     * @return the last_alert
     */
    public int getLast_alert() {
        return last_alert;
    }

    /**
     * @param last_alert the last_alert to set
     */
    public void setLast_alert(int last_alert) {
        this.last_alert = last_alert;
    }

    /**
     * @return the alert_limit
     */
    public int getAlert_limit() {
        return alert_limit;
    }

    /**
     * @param alert_limit the alert_limit to set
     */
    public void setAlert_limit(int alert_limit) {
        this.alert_limit = alert_limit;
    }
} public class PushOverMessage {
    private String userKey;
    private String title;
    private String message;
    
    public PushOverMessage(String p_userKey, String p_title, String p_message)
    {
        userKey = p_userKey;
        title = p_title;
        message = p_message;
    }

    /**
     * @return the userKey
     */
    public String getUserKey() {
        return userKey;
    }

    /**
     * @param userKey the userKey to set
     */
    public void setUserKey(String userKey) {
        this.userKey = userKey;
    }

    /**
     * @return the title
     */
    public String getTitle() {
        return title;
    }

    /**
     * @param title the title to set
     */
    public void setTitle(String title) {
        this.title = title;
    }

    /**
     * @return the message
     */
    public String getMessage() {
        return message;
    }

    /**
     * @param message the message to set
     */
    public void setMessage(String message) {
        this.message = message;
    }
} public class ShortTextMessage {
    private String message;
    private String mobile;
    
    public ShortTextMessage(String p_message, String p_mobile)
    {
        message = p_message;
        mobile = p_mobile;
    }

    /**
     * @return the message
     */
    public String getMessage() {
        return message;
    }

    /**
     * @param message the message to set
     */
    public void setMessage(String message) {
        this.message = message;
    }

    /**
     * @return the mobile
     */
    public String getMobile() {
        return mobile;
    }

    /**
     * @param mobile the mobile to set
     */
    public void setMobile(String mobile) {
        this.mobile = mobile;
    }
} public class TTSMessage {
    private String message;
    private String mobile;
    
    public TTSMessage(String p_message, String p_mobile)
    {
        message = p_message;
        mobile = p_mobile;
    }

    /**
     * @return the message
     */
    public String getMessage() {
        return message;
    }

    /**
     * @param message the message to set
     */
    public void setMessage(String message) {
        this.message = message;
    }

    /**
     * @return the mobile
     */
    public String getMobile() {
        return mobile;
    }

    /**
     * @param mobile the mobile to set
     */
    public void setMobile(String mobile) {
        this.mobile = mobile;
    }
} public class PushOverLimiter implements Runnable {

    @Override
    public void run() {
        logger.info("PushOverLimiter started (API Rate Limiting)");
        while(true)
        {
            try {
                PushOverMessage pom = com.clavain.muninmxcd.notification_pushover_queue.take();
                logger.info("[PushOverLimiter] Processing Message with UserKey: " + pom.getUserKey() + " MSG: " + pom.getMessage());
                sendPushOverMessage(pom.getUserKey(), pom.getTitle(), pom.getMessage());
                Thread.sleep(500);
            } catch (Exception ex)
            {
               logger.error("Error in PushOverLimiter: " + ex.getLocalizedMessage());
            }
        }
    }
    
} public class SMSLimiter implements Runnable {

    @Override
    public void run() {
        logger.info("SMSLimiter started (API Rate Limiting)");
        while(true)
        {
            try {
                ShortTextMessage sms = com.clavain.muninmxcd.notification_sms_queue.take();
                logger.info("[SMSLimiter] Processing Message to: " + sms.getMobile() + " MSG: " + sms.getMessage());
                sendSMSMessage(sms.getMessage(), sms.getMobile());
                Thread.sleep(500);
            } catch (Exception ex)
            {
                logger.error("Error in SMSLimiter: " + ex.getLocalizedMessage());
            }
        }
    }
    
} public class TTSLimiter implements Runnable {

    @Override
    public void run() {
        logger.info("TTSLimiter started (API Rate Limiting)");
        while(true)
        {
            try {
                TTSMessage tts = com.clavain.muninmxcd.notification_tts_queue.take();
                logger.info("[TTSLimiter] Processing Message to: " + tts.getMobile() + " MSG: " + tts.getMessage());
                sendTTSMessage(tts.getMessage(), tts.getMobile());
                Thread.sleep(500);
            } catch (Exception ex)
            {
                logger.error("Error in TTSLimiter: " + ex.getLocalizedMessage());
            }
        }
    }
    
} public class ReturnDebugTrace {
    private String output;
    private int checktime;
    private int retval;
    private int cid;
    private int user_id;

    /**
     * @return the output
     */
    public String getOutput() {
        return output;
    }

    /**
     * @param output the output to set
     */
    public void setOutput(String output) {
        this.output = output;
    }

    /**
     * @return the checktime
     */
    public int getChecktime() {
        return checktime;
    }

    /**
     * @param checktime the checktime to set
     */
    public void setChecktime(int checktime) {
        this.checktime = checktime;
    }

    /**
     * @return the retval
     */
    public int getRetval() {
        return retval;
    }

    /**
     * @param retval the retval to set
     */
    public void setRetval(int retval) {
        this.retval = retval;
    }

    /**
     * @return the cid
     */
    public int getCid() {
        return cid;
    }

    /**
     * @param cid the cid to set
     */
    public void setCid(int cid) {
        this.cid = cid;
    }

    /**
     * @return the user_id
     */
    public int getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(int user_id) {
        this.user_id = user_id;
    }
} public class MongoExecutor implements Runnable {

    @Override
    public void run() {
        try
        {
            while(true)
            {
                Thread.sleep(50);
                ExecutorService executor = Executors.newFixedThreadPool(Integer.parseInt(p.getProperty("mongo.threads")));
                
                executor.execute(new MongoWorker());

                executor.shutdown();
                while (!executor.isTerminated()) {
                    Thread.sleep(50);
                }
                logger.info("MongoWorker Set finished. Restarting MongoWorkers.");
                System.gc();
            }
        } catch (Exception ex)
        {
            logger.fatal("Error in MongoExecutor: " + ex.getLocalizedMessage());
        }
    }
    
} public class MongoEssentialExecutor implements Runnable {

    @Override
    public void run() {
        try
        {
            while(true)
            {
                Thread.sleep(50);
                ExecutorService executor = Executors.newFixedThreadPool(2);
                
                executor.execute(new MongoEssentialWorker());

                executor.shutdown();
                while (!executor.isTerminated()) {
                    Thread.sleep(50);
                }
                logger.info("MongoPkgWorker Set finished. Restarting MongoPkgWorkers.");
                System.gc();
            }
        } catch (Exception ex)
        {
            logger.fatal("Error in MongoPkgExecutor: " + ex.getLocalizedMessage());
        }
    }
    
} public class JettyLauncher implements Runnable {

    @Override
    public void run() {
            // Jetty
        try
        {
            Server server = new Server(Integer.parseInt(p.getProperty("api.port")));
            server.setHandler(new JettyHandler());
            logger.info("Starting API Server on port " + p.getProperty("api.port"));
            server.start();
            server.join();
        } catch (Exception ex)
        {
            logger.fatal("Api Server crashed: " + ex.getLocalizedMessage());  
        }
    }
    
} public class ServiceCheck implements Serializable {
    private static final long serialVersionUID = -4991349865504351287L;
    private String  checkname;
    private Integer interval;
    private Integer checktype;
    private String nonearg;
    private String command;
    private String[] param;
    private Integer user_id;
    private Integer cid;
    private Integer iterations = 0;
    private boolean is_active = true;


    /**
     * @return the checkname
     */
    public String getCheckname() {
        return checkname;
    }

    /**
     * @param checkname the checkname to set
     */
    public void setCheckname(String checkname) {
        this.checkname = checkname;
    }

    /**
     * @return the interval
     */
    public Integer getInterval() {
        return interval;
    }

    /**
     * @param interval the interval to set
     */
    public void setInterval(Integer interval) {
        this.interval = interval;
    }

    /**
     * @return the checktype
     */
    public Integer getChecktype() {
        return checktype;
    }

    /**
     * @param checktype the checktype to set
     */
    public void setChecktype(Integer checktype) {
        this.checktype = checktype;
    }

    /**
     * @return the nonearg
     */
    public String getNonearg() {
        return nonearg;
    }

    /**
     * @param nonearg the nonearg to set
     */
    public void setNonearg(String nonearg) {
        this.nonearg = nonearg;
    }

    /**
     * @return the command
     */
    public String getCommand() {
        return command;
    }

    /**
     * @param command the command to set
     */
    public void setCommand(String command) {
        this.command = command;
    }

    /**
     * @return the param
     */
    public String[] getParam() {
        return param;
    }

    /**
     * @param param the param to set
     */
    public void setParam(String[] p_param) {
        param = p_param;
    }

    /**
     * @return the user_id
     */
    public Integer getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(Integer user_id) {
        this.user_id = user_id;
    }

    /**
     * @return the cid
     */
    public Integer getCid() {
        return cid;
    }

    /**
     * @param cid the cid to set
     */
    public void setCid(Integer cid) {
        this.cid = cid;
    }


    /**
     * @return the iterations
     */
    public Integer getIterations() {
        return iterations;
    }

    /**
     * @param iterations the iterations to set
     */
    public void setIterations(Integer iterations) {
        this.iterations = iterations;
    }

    /**
     * @return the is_active
     */
    public boolean isIs_active() {
        return is_active;
    }

    /**
     * @param is_active the is_active to set
     */
    public void setIs_active(boolean is_active) {
        this.is_active = is_active;
    }


} public class MuninNode 
{
  private String    str_nodename;
  private String    str_hostname    = "unset";  
  private int       i_port          = 4949;
  private String    str_group;
  private boolean   b_isRunning     = false;
  private transient long  l_lastFrontendQuery;   
  private String    str_muninVersion = "";
  private transient CopyOnWriteArrayList<MuninPlugin> v_loaded_plugins;
  private transient int       i_GraphCount    = 0;
  private int       i_lastRun        = 0;
  private Integer   node_id          = 0;
  private Integer   user_id          = 0;
  private int       queryInterval    = 0;
  private int       last_plugin_load = 0;
  private String    authpw           = "";
  private boolean   is_init = false;
  private boolean   track_pkg       = false;
  private boolean   essentials      = false;
  private int       last_pkg_update = 0;
  private int       last_essentials_update = 0;
  private transient Socket lastSocket;
  private String    str_via = "unset";

  
    public void setQueryInterval(Integer p_int)
    {
        queryInterval = p_int;
    }
    
    public int getQueryInterval()
    {
        return queryInterval;
    }
  
    public int getGraphCount()
    {
        return i_GraphCount;
    }

    /**
     * @return the str_hostname
     */
    public String getHostname()
    {
        return str_hostname;
    }

    /**
     * @param str_hostname the str_hostname to set
     */
    public void setHostname(String str_hostname)
    {
        this.str_hostname = str_hostname;
    }

    /**
     * @return the i_port
     */
    public int getPort()
    {
        return i_port;
    }

    /**
     * @param i_port the i_port to set
     */
    public void setPort(int i_port)
    {
        this.i_port = i_port;
    }

    /**
     * @return the str_nodename
     */
    public String getNodename()
    {
        return str_nodename;
    }

    /**
     * @param str_nodename the str_nodename to set
     */
    public void setNodename(String str_nodename)
    {
        this.str_nodename = str_nodename;
    }

    /**
     * @return the str_group
     */
    public String getGroup()
    {
        return str_group;
    }

    /**
     * @param str_group the str_group to set
     */
    public void setGroup(String str_group)
    {
        this.str_group = str_group;
    }
    
    /**
     * 
     * @return if the Thread is active or not 
     */
    public boolean isRunning()
    {
        return b_isRunning;
    }
    
    public void setIsRunning(boolean p_isRunning)
    {
        b_isRunning = p_isRunning;
    }

    
    /** 
     * will retrieve a list of loaded munin plugins for this host. If the node was never
     * contacted before the list is downloaded from munin-node
     * 
     * @return list of plugins
     */
    public CopyOnWriteArrayList<MuninPlugin> getPluginList()
    {
        if(getLoadedPlugins() == null)
        {
            // using logger from main class because of gson throwing stackoverflow otherwise
            logger.info("Empty Pluginlist for " + this.getHostname() + ". Loading from munin-node...");
            loadPlugins();
        }
        return getLoadedPlugins();
    }
    

    /**
     * Will load the plugin list from munin-node
     */
    public boolean loadPlugins()
    {
        setLoadedPlugins(new CopyOnWriteArrayList<MuninPlugin>());
        String l_lastProceeded = "";
        
        
        try 
        {
            Socket cs = new Socket();
            cs.setKeepAlive(false);
            cs.setSoLinger(true, 0);
            cs.setReuseAddress(true);  
            cs.setSoTimeout(com.clavain.muninmxcd.socketTimeout);
            if(!str_via.equals("unset"))
            {
                cs.connect(new InetSocketAddress(this.getStr_via(), this.getPort()),com.clavain.muninmxcd.socketTimeout);    
            }
            else
            {
                cs.connect(new InetSocketAddress(this.getHostname(), this.getPort()),com.clavain.muninmxcd.socketTimeout);
            }
          
            if(p.getProperty("kill.sockets").equals("true"))
            {
                SocketCheck sc = new SocketCheck(cs,getUnixtime());
                sc.setHostname(this.getHostname());
                com.clavain.muninmxcd.v_sockets.add(sc);
            }
            PrintStream os = new PrintStream( cs.getOutputStream() );
            BufferedReader in = new BufferedReader(new InputStreamReader( cs.getInputStream()) );  
            
            String s = in.readLine();

            if(s != null)
            {
                // Set version
                os.println("version");
                Thread.sleep(150);
                s = in.readLine();
                
                String version = s.substring(s.indexOf(":")+1,s.length()).trim();
                this.str_muninVersion = version;
                
                if(authpw != null)
                {
                    // if authpw is set, verify
                    if(!authpw.trim().equals(""))
                    {
                        os.println("config muninmxauth");
                        Thread.sleep(150);
                        String apw = in.readLine();
                        s = in.readLine();
                        if(!apw.trim().equals(this.getAuthpw()))
                        {
                            logger.error("Invalid muninmxauth password for host: " + this.getHostname());
                            cs.close();
                            return false;
                        }
                    }
                }
                // check anyway if muninmxauth plugin is present
                else
                {
                    os.println("config muninmxauth");
                    Thread.sleep(100);
                    String apw = in.readLine();
                    if(!apw.trim().equals("# Unknown service"))
                    {
                        logger.error("no auth password given, but muninmxauth plugin present on " + this.getHostname());
                        cs.close();
                        return false;                        
                    }
                    s = in.readLine();
                }
                
                // get list of available plugins
                if(str_via.equals("unset"))
                {
                    os.println("list");
                }
                else
                {
                    os.println("list " + str_hostname);
                }

                Thread.sleep(250);
                s = in.readLine();
 
                // if response is empty and host is not via, do a list $hostname
                if(s.trim().equals("") && str_via.equals("unset"))
                {
                    logger.info("Plugin Response Empty on " + this.getHostname() + " trying to load with list $hostname");
                    os.println("list " + this.getHostname());
                    Thread.sleep(250);
                    s = in.readLine();
                }
                
                String l_tmp;
                StringTokenizer l_st = new StringTokenizer(s, " ");
                
                // create plugin
                MuninPlugin l_mp = new MuninPlugin();
                // negative support
                ArrayList<String> tmp_negatives = new ArrayList<String>();
                
                while(l_st.hasMoreTokens())
                {
                     
                    String l_strPlugin = l_st.nextToken();
                    
                    // check for track_pkg and muninmx essentials
                    if(l_strPlugin.equals("muninmx_trackpkg"))
                    {
                        this.setTrack_pkg(true);
                        continue;
                    }
                    
                    // got essentials?
                    if(l_strPlugin.equals("muninmx_essentials"))
                    {
                        this.setEssentials(true);
                        continue;
                    }                    
                    
                    if(isPluginIgnored(l_strPlugin.toUpperCase()))
                    {
                        continue;
                    }
                                      
                    l_mp.setPluginName(l_strPlugin);

                    os.println("config " + l_strPlugin);
                    
                    // create graphs for plugin
                    int l_iGraphsFound = 0;  
                    int l_iTmp         = 0;
                    MuninGraph  l_mg = new MuninGraph();                    
                    l_mg.setQueryInterval(this.getQueryInterval());
                    while ((l_tmp = in.readLine()) != null) 
                    {
                      if(l_tmp.startsWith("."))
                      {
                        break;
                      }
                      // collect graphs only for plugin
                      String l_strName;
                      String l_strType;
                      String l_strValue;

                      
                      if(!l_tmp.contains("graph_") && !l_tmp.trim().equals("") && !l_tmp.contains("host_name") && !l_tmp.contains("multigraph") && !l_tmp.trim().equals("graph no") && !l_tmp.trim().equals("# Bad exit") && !l_tmp.trim().contains("info Currently our peer") && !l_tmp.trim().startsWith("#") && !l_tmp.trim().contains("Bonding interface errors"))
                      {
                        l_lastProceeded = l_tmp;
                        l_strName  = l_tmp.substring(0,l_tmp.indexOf("."));
                        l_strType  = l_tmp.substring(l_tmp.indexOf(".")+1,l_tmp.indexOf(" "));
                        l_strValue = l_tmp.substring(l_tmp.indexOf(" ")+1,l_tmp.length());
                        //System.err.println("Name: " + l_strName + " Type: " + l_strType + " Value: " + l_strValue);
                        
                        if(l_strType.equals("label"))
                        {
                           l_iTmp++;
                          
                           if(l_iTmp > 1)
                           {
                               l_mp.addGraph(l_mg);
                               l_mg = new MuninGraph();
                               l_mg.setQueryInterval(this.getQueryInterval());
                           }
                           l_mg.setGraphName(l_strName);
                           l_mg.setGraphLabel(l_strValue);
                        }
                        else if(l_strType.equals("draw"))
                        {
                            l_mg.setGraphDraw(l_strValue);
                        }
                        else if(l_strType.equals("type"))
                        {
                            l_mg.setGraphType(l_strValue);
                        }
                        else if(l_strType.equals("info"))
                        {
                            l_mg.setGraphInfo(l_strValue);
                        }
                        else if(l_strType.equals("negative"))
                        {
                            // add to temporary negative list to set negatives later
                            tmp_negatives.add(l_strValue);
                        }

                        //System.out.println(l_strName); 
                        //System.out.println(l_strType);
                        //System.out.println(l_strValue);
                      }
                      else
                      {
                          // set plugin title
                          if(l_tmp.contains("graph_title"))
                          {
                              l_mp.setPluginTitle(l_tmp.substring(12,l_tmp.length()));
                          }
                          // set plugin info, if any
                          if(l_tmp.contains("graph_info"))
                          {
                              l_mp.setPluginInfo(l_tmp.substring(11,l_tmp.length()));
                          }
                          // set graph category
                          if(l_tmp.contains("graph_category"))
                          {
                              l_mp.setPluginCategory(l_tmp.substring(15,l_tmp.length()));
                          }
                          // set graph vlabel
                          if(l_tmp.contains("graph_vlabel"))
                          {
                              l_mp.setPluginLabel(l_tmp.substring(13,l_tmp.length()));
                          }
                          // set plugin title
                          if(l_tmp.contains("graph_mxdraw"))
                          {
                              l_mp.setStr_LineMode(l_tmp.substring(13,l_tmp.length()));
                          }                          
                      }
                      
                    }
                  
                    
                    // add to pluginlist
                    l_mp.addGraph(l_mg);
                    
                   Iterator it = l_mp.getGraphs().iterator();
                   while(it.hasNext())
                   {
                       MuninGraph l_mpNg = (MuninGraph) it.next();
                       if(tmp_negatives.contains(l_mpNg.getGraphName()))
                       {
                           l_mpNg.setNegative(true);
                       }
                   }
                    
                    // add plugin if it got valid graphs and add nodeid (req. for alerts)
                    if(l_mp.getGraphs().size() > 0)
                    {
                        l_mp.set_NodeId(this.getNode_id());
                        getLoadedPlugins().add(l_mp);
                    }
                    // flush temporary negatives
                    tmp_negatives.clear();
                    l_mp = null;
                    l_mp = new MuninPlugin();
                    //String l_strGraphTitle = s.substring(s.indexOf("graph_title") + 11,s.length());
                    //System.out.println(" - " + l_strGraphTitle);
                }
                cs.close();
                in.close();
                os.close();
                last_plugin_load = getUnixtime();
                //System.out.println(s);
            }
            else
            {
                cs.close();
                in.close();
                os.close();
                logger.warn("Error loading plugins on " + str_hostname + " ("+this.getNode_id()+"). Check connectivity or munin-node");
            }
            /*
            for (MuninPlugin l_mn : getLoadedPlugins()) {
                i_GraphCount = i_GraphCount + l_mn.getGraphs().size();
                logger.debug(l_mn.getGraphs().size() + " graphs found for plugin: " + l_mn.getPluginName().toUpperCase() + " on node: " + this.getNodename());
            }*/
        } catch (Exception ex) {
            logger.error("Error loading plugins on " + str_hostname + " ("+this.getNode_id()+") : " + ex.getMessage());
            ex.printStackTrace();
            return false;
        } 

        return true;
    }
            

    public void run() {
        b_isRunning = true;

        if(this.str_via.equals("unset"))
        {
            logger.info(getHostname() + " Monitoring job started");
        }
        else
        {
            logger.info(getHostname() + " (VIA: "+this.str_via+") Monitoring job started");
        }
        
        int iCurTime = getUnixtime();
        int iPluginRefreshTime = last_plugin_load + Integer.parseInt(p.getProperty("plugin.refreshtime"));
        try {
            // update plugins, maybe we have some new :)
            // double try to load plugins if fail
       
            if(getPluginList().size() > 0)
            {
                if(!is_init)
                {
                    logger.info("[Job: " + getHostname() + "] Updating Database");
                    // update graphs in database too
                    for(MuninPlugin it_pl : getPluginList()) {
                        if(it_pl.getGraphs().size() > 0)
                        {
                            //logger.info(it_pl.getPluginName());
                            dbUpdatePluginForNode(getNode_id(),it_pl);
                        }
                     }
                    // delete now missing plugins
                    dbDeleteMissingPlugins(getNode_id(),getPluginList());
                    logger.info("[Job: " + getHostname() + "] Databaseupdate Done");
                    is_init = true;
                }
                else
                {
                    if(iCurTime > iPluginRefreshTime )
                    {
                        logger.info("Refreshing Plugins on " + this.getHostname());
                        this.loadPlugins();
                        dbUpdateAllPluginsForNode(this);
                    }
                }
            }
            else
            {
                this.loadPlugins();
            }
                  
            
            Socket clientSocket = new Socket();
            clientSocket.setSoTimeout(com.clavain.muninmxcd.socketTimeout);
            clientSocket.setKeepAlive(false);
            clientSocket.setReuseAddress(true);   
            if(this.str_via.equals("unset"))
            {            
                clientSocket.connect(new InetSocketAddress(this.getHostname(), this.getPort()),com.clavain.muninmxcd.socketTimeout);
            }
            else
            {
                clientSocket.connect(new InetSocketAddress(this.getStr_via(), this.getPort()),com.clavain.muninmxcd.socketTimeout);
            }
            lastSocket = clientSocket;
            SocketCheck sc = new SocketCheck(clientSocket,getUnixtime());
            if(p.getProperty("kill.sockets").equals("true"))
            {  
                sc.setHostname(this.getHostname());
                com.clavain.muninmxcd.v_sockets.add(sc);
            }
            this.i_lastRun = getUnixtime();

            // track packages?
            if(this.track_pkg)
            {
                updateTrackPackages(clientSocket);
            }
            
            // gather essentials?
            if(this.essentials)
            {
                updateEssentials(clientSocket);
            }
            
            // update graphs for all plugins
            Iterator it = this.getLoadedPlugins().iterator();
            while(it.hasNext())
            {
                MuninPlugin l_mp = (MuninPlugin) it.next();
                if(logMore)
                {
                    logger.info(getHostname() + " fetching graphs for " + l_mp.getPluginName().toUpperCase());
                }
                // snmp support
                if(!str_via.equals("unset"))
                { 
                    l_mp.updateAllGraps(this.getStr_via(), this.getPort(), clientSocket, getQueryInterval());
                }  
                else
                {
                    l_mp.updateAllGraps(this.getHostname(), this.getPort(), clientSocket, getQueryInterval());
                }
                // add all graphs to insertion queue for mongodb
                queuePluginFetch(l_mp.returnAllGraphs(), l_mp.getPluginName());
            }
            clientSocket.close();
            
            if(p.getProperty("kill.sockets").equals("true"))
            {
                com.clavain.muninmxcd.v_sockets.remove(sc);
            }
            sc = null;
        } catch (Exception ex) {
           logger.fatal("Error in thread for host: " + getHostname() + " : " + ex.getLocalizedMessage());
           ex.printStackTrace();
        }
        int iRunTime = getUnixtime() - iCurTime;
        dbUpdateLastContact(this.getNode_id());
        logger.info(getHostname() + " Monitoring job stopped - runtime: " + iRunTime);
        
    }

    
    /*
     * update essentials
     */
    private void updateEssentials(Socket p_socket)
    {
        String decodestr = "";
        try {
            PrintStream os = new PrintStream( p_socket.getOutputStream() );
            BufferedReader in = new BufferedReader(new InputStreamReader( p_socket.getInputStream()) );
            os.println("config muninmx_essentials");
            
            // skip first line if starts with #
            decodestr = in.readLine();
            if(decodestr.startsWith("#"))
            {
                decodestr = in.readLine();    
            }

            if(decodestr.equals("."))
            {
                decodestr = in.readLine();    
            }             
            BasicDBObject doc = new BasicDBObject();
            doc.put("data", decodestr);
            doc.put("time",getUnixtime());
            doc.put("node", this.node_id);
            doc.put("type","essential");
            com.clavain.muninmxcd.mongo_essential_queue.add(doc);  
            logger.info("Essentials Updated for Node: " + this.getHostname() + ": received base64 (length): " + decodestr.length());
            last_essentials_update = getUnixtime();
            // read following .
            in.readLine();
        } catch (Exception ex)
        {
            logger.error("Error in updateEssentials for Node " + this.getHostname() + " : " + ex.getLocalizedMessage());
            logger.error("updateEssentials for Node " + this.getHostname() + " received: " + decodestr);
            ex.printStackTrace();     
        }
    }
    
    /*
     * update track package log
     */
    private void updateTrackPackages(Socket p_socket)
    {
        // only try to update this once per hour
        int curTime = getUnixtime();
        int lalert = this.last_pkg_update + 3600;
        if(lalert > curTime )
        {  
            return;
        } 
        
        String decodestr = "";
        try 
        {
            logger.info("TrackPackages - fetching " + this.str_hostname);
            PrintStream os = new PrintStream( p_socket.getOutputStream() );
            BufferedReader in = new BufferedReader(new InputStreamReader( p_socket.getInputStream()) );
            os.println("config muninmx_trackpkg");
            // skip first line if starts with #
            decodestr = in.readLine();
            if(decodestr.startsWith("#"))
            {
                decodestr = in.readLine();    
            }

            if(decodestr.equals("."))
            {
                decodestr = in.readLine();    
            }   
            byte[] decode = Base64.decodeBase64(decodestr);
            ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(decode); 
            GZIPInputStream gzipInputStream;
            gzipInputStream = new GZIPInputStream(byteArrayInputStream);

            InputStreamReader inputStreamReader = new InputStreamReader(gzipInputStream);
            BufferedReader bufferedReader = new BufferedReader(inputStreamReader, 4);

            String read;
            String sum = bufferedReader.readLine();
            if(sum == null) 
            {  
                logger.error("TrackPackages - sum is null for: " + this.str_hostname);
                return; 
            }  
            String dist = bufferedReader.readLine();
            String ver  = bufferedReader.readLine();
            String kernel = bufferedReader.readLine();
            if(dbTrackLogChangedForNode(sum, this.node_id))
            {
                logger.info("TrackPackages - packages changed, updating " + this.str_hostname);
                // purge old logs
                removeOldPackageTrack(this.node_id);
                
                dbUpdateNodeDistVerKernel(sum,dist, ver, kernel, this.node_id);      
                int i = 0;
                while ((read = bufferedReader.readLine()) != null) 
                {
                     BasicDBObject doc = new BasicDBObject();
                     doc.put("package", read);
                     doc.put("time",getUnixtime());
                     doc.put("node", this.node_id);
                     doc.put("type","trackpkg");
                     com.clavain.muninmxcd.mongo_essential_queue.add(doc);
                     i++;
                } 
                logger.info("TrackPackages Updated for Node: " + this.getHostname() + " ("+dist+" " + ver + " " + kernel + "). tracking " + i + " packages"); 
            }
            else
            {
                logger.info("TrackPackages - sum not changed since last run for Node: " + this.getHostname());
            }
            this.last_pkg_update = getUnixtime();

        } catch (Exception ex)
        {
            logger.error("Error in updateTrackPackages for Node " + this.getHostname() + " : " + ex.getLocalizedMessage());
            logger.error("updateTrackPackages for Node " + this.getHostname() + " received: " + decodestr);
            ex.printStackTrace();
        }
            
    }
    
    public void queuePluginFetch(ArrayList<MuninGraph> p_almg, String p_strPluginName)
    {
        queuePluginFetch(p_almg, p_strPluginName, 0);
    }
    
    /**
     * fill insertion queue with current graph values for each plugin
     */
    public void queuePluginFetch(ArrayList<MuninGraph> p_almg, String p_strPluginName, int p_iCustomId)
    {
        Iterator<MuninGraph> it = p_almg.iterator();
        while(it.hasNext())
        {
            MuninGraph mg = it.next();
            // prepare document object
            BasicDBObject doc = new BasicDBObject();
            doc.put("hostname", this.getHostname());
            doc.put("plugin", p_strPluginName);
            doc.put("graph", mg.getGraphName());
            doc.put("value", mg.getGraphValue().toString());
            doc.put("recv", mg.getLastGraphTime());
            doc.put("user_id", this.getUser_id());
            doc.put("nodeid", this.getNode_id());
            
            if(p_iCustomId > 0)
            {
                doc.put("customId",p_iCustomId);
            }
            
            // only queue if plugin is initialized or it is a if_err plugin
            if(mg.isInit() || p_strPluginName.startsWith("if_err") || p_strPluginName.equals("swap"))
            {
                com.clavain.muninmxcd.mongo_queue.add(doc);
                mg.setLastQueued(getUnixtime());
                
                logger.debug("Queued: " + this.getHostname() + " (" + p_strPluginName + " / " + mg.getGraphName() + ") Value: " + mg.getGraphValue());
                if(logMore)
                {
                    logger.info("Queued: " + this.getHostname() + " (" + p_strPluginName + " / " + mg.getGraphName() + ") Value: " + mg.getGraphValue());   
                }                
            }

        }
    }
    
    /**
     * @return the v_loaded_plugins
     */
    public CopyOnWriteArrayList<MuninPlugin> getLoadedPlugins() {
        return v_loaded_plugins;
    }

    /**
     * @param v_loaded_plugins the v_loaded_plugins to set
     */
    public void setLoadedPlugins(CopyOnWriteArrayList<MuninPlugin> v_loaded_plugins) {
        this.v_loaded_plugins = v_loaded_plugins;
    }

    /**
     * @return the l_lastFrontendQuery
     */
    public long getLastFrontendQuery() {
        return l_lastFrontendQuery;
    }

    /**
     * @param l_lastFrontendQuery the l_lastFrontendQuery to set
     */
    public void setLastFrontendQuery(long l_lastFrontendQuery) {
        this.l_lastFrontendQuery = l_lastFrontendQuery;
    }
    
    /**
     * Sets lastFrontendQuery to current unixtime
     */
    public void setLastFrontendQuery()
    {
        l_lastFrontendQuery = System.currentTimeMillis() / 1000L;
    }

    private void cleanUpGraphs() 
    {
        Iterator it = this.getLoadedPlugins().iterator();
        while(it.hasNext())
        {
            MuninPlugin l_mp = (MuninPlugin) it.next();
            Iterator itg = l_mp.getGraphs().iterator();
            while(itg.hasNext())
            {
                MuninGraph l_mg = (MuninGraph) itg.next();
                l_mg.setGraphValue("0");
                l_mg.setLastGraphValue("0");
                l_mg.setLastGraphValueCounter("0");
            }
        }
    }

    /**
     * @return the node_id
     */
    public Integer getNode_id() {
        return node_id;
    }

    /**
     * @param node_id the node_id to set
     */
    public void setNode_id(Integer node_id) {
        this.node_id = node_id;
    }

    /**
     * @return the user_id
     */
    public Integer getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(Integer user_id) {
        this.user_id = user_id;
    }

    /**
     * @return the lastSocket
     */
    public Socket getLastSocket() {
        return lastSocket;
    }

    /**
     * @return the str_via
     */
    public String getStr_via() {
        return str_via;
    }

    /**
     * @param str_via the str_via to set
     */
    public void setStr_via(String str_via) {
        this.str_via = str_via;
    }

    /**
     * @return the authpw
     */
    public String getAuthpw() {
        return authpw;
    }

    /**
     * @param authpw the authpw to set
     */
    public void setAuthpw(String authpw) {
        this.authpw = authpw;
    }

    /**
     * @return the track_pkg
     */
    public boolean isTrack_pkg() {
        return track_pkg;
    }

    /**
     * @param track_pkg the track_pkg to set
     */
    public void setTrack_pkg(boolean track_pkg) {
        this.track_pkg = track_pkg;
    }

    /**
     * @return the essentials
     */
    public boolean isEssentials() {
        return essentials;
    }

    /**
     * @param essentials the essentials to set
     */
    public void setEssentials(boolean essentials) {
        this.essentials = essentials;
    }
}
 public class MuninPlugin {
    private String str_PluginName;
    private String str_PluginTitle;
    private String str_PluginInfo;
    private String str_PluginCategory;
    private String str_PluginLabel;
    private String str_LineMode = "default";
    private long   l_lastFrontendQuery;
    private transient long   l_lastMuninQuery;
    private transient Socket csMuninSocket;
    private transient boolean b_IntervalIsSeconds = false;
    private int i_nodeId;
    private transient Integer from_time;
    private transient Integer to_time;
    private transient String timezone;
    private transient Integer customId;
    private transient Integer query_interval;
    private transient Integer user_id;
    private transient String crontab = "false";
    private ArrayList<MuninGraph> v_graphs = new ArrayList<>();

    
    public void addGraph(MuninGraph p_graph)
    {
        getGraphs().add(p_graph);
    }
    
    /**
     * @return the str_PluginName
     */
    public String getPluginName() {
        return str_PluginName;
    }

    /**
     * @param str_PluginName the str_PluginName to set
     */
    public void setPluginName(String str_PluginName) {
        this.str_PluginName = str_PluginName;
    }

    /**
     * @return the str_GraphTitle
     */
    public String getPluginTitle() {
        return str_PluginTitle;
    }

    /**
     * @param str_GraphTitle the str_GraphTitle to set
     */
    public void setPluginTitle(String str_GraphTitle) {
        this.str_PluginTitle = str_GraphTitle;
    }

    /**
     * @return the str_GraphInfo
     */
    public String getPluginInfo() {
        return str_PluginInfo;
    }

    /**
     * @param str_PluginInfo the str_GraphInfo to set
     */
    public void setPluginInfo(String str_GraphInfo) {
        this.str_PluginInfo = str_GraphInfo;
    }

    /**
     * @return the v_graphs
     */
    public ArrayList<MuninGraph> getGraphs() {
        return getV_graphs();
    }

    /**
     * @param v_graphs the v_graphs to set
     */
    public void setGraphs(ArrayList<MuninGraph> v_graphs) {
        this.setV_graphs(v_graphs);
    }

    /**
     * @return the l_lastFrontendQuery
     */
    public long getLastFrontendQuery() {
        return l_lastFrontendQuery;
    }

    /**
     * @param l_lastFrontendQuery the l_lastFrontendQuery to set
     */
    public void setLastFrontendQuery(long l_lastFrontendQuery) {
        this.l_lastFrontendQuery = l_lastFrontendQuery;
    }
    
    /**
     * Sets lastFrontendQuery to current unixtime
     */
    public void setLastFrontendQuery()
    {
        l_lastFrontendQuery = System.currentTimeMillis() / 1000L;
    }
    
    
    /*
     * will connect to munin if no connections exists and will update
     * values for all graphs, then return all graphs with updated stats
     */
    public void updateAllGraps(String p_strHostname, int p_iPort, Socket p_socket, int p_queryInterval)
    {

        try
        {
            setCsMuninSocket(p_socket);
            // connection available?
            if(getCsMuninSocket() != null)
            {
                if(!csMuninSocket.isConnected() || getCsMuninSocket().isClosed())
                {
                    setCsMuninSocket(null);
                    setCsMuninSocket(new Socket());
                    getCsMuninSocket().setSoTimeout(5000);
                    getCsMuninSocket().setKeepAlive(false);
                    getCsMuninSocket().setSoLinger(true, 0);
                    getCsMuninSocket().setReuseAddress(true);                     
                    getCsMuninSocket().connect(new InetSocketAddress(p_strHostname, p_iPort),5000);
                    logger.info("Reconnecting to " + p_strHostname);
                }
            }
            else
            {
                setCsMuninSocket(new Socket());
                getCsMuninSocket().setSoTimeout(5000);
                getCsMuninSocket().setKeepAlive(false);
                getCsMuninSocket().setSoLinger(true, 0);
                getCsMuninSocket().setReuseAddress(true);                  
                getCsMuninSocket().connect(new InetSocketAddress(p_strHostname, p_iPort),5000);
            }
            PrintStream os = new PrintStream( getCsMuninSocket().getOutputStream() );
            BufferedReader in = new BufferedReader(new InputStreamReader( getCsMuninSocket().getInputStream()) );
            os.println("fetch " + this.getPluginName());
            if(logMore)
            {
                logger.info(p_strHostname + " Executed:  fetch " + this.getPluginName());
            }
            String line;    
            while((line = in.readLine()) != null) {
                if(line.startsWith("."))
                {
                    if(logMore)
                    {
                        logger.info(p_strHostname + " Executed:  fetch " + this.getPluginName() + " BUT INSTEAD GOT . AS RESULT");
                    }                    
                    // close wait fix for single mode
                    if(customId != null)
                    {
                        os.println("quit");
                        os.close();
                        in.close();
                        csMuninSocket.close();
                        csMuninSocket = null;
                    }                        
                    return;
                }
                if(logMore)
                {                
                    logger.info(p_strHostname + " Received from (fetch " + this.getPluginName()+") Result: " + line);
                }
                //System.out.println(line);
                if(line.contains("value") && !line.contains("#"))
                {
                    String l_graphName = line.substring(0,line.indexOf("."));
                    String l_value      = line.substring(line.indexOf(" ")+1,line.length());
                    if(logMore)
                    {
                        logger.info(p_strHostname + " - " + l_graphName + " - " + l_value);
                    }
                    Iterator it = this.getV_graphs().iterator();
                    while (it.hasNext())
                    {
                        MuninGraph l_mg = (MuninGraph) it.next();
                        l_mg.setQueryInterval(p_queryInterval);
                        // required for customer plugin intervals
                        if(IntervalIsSeconds() != false)
                        {
                           l_mg.setIntervalIsSeconds(true); 
                        }
                        
                        if(logMore)
                        {
                            logger.info(p_strHostname + " - equalcheck - Given: " + l_mg.getGraphName() + " required: " + l_graphName);
                        }                          
                        if(l_mg.getGraphName().equals(l_graphName))
                        {    
                            if(logMore)
                            {
                                logger.info(p_strHostname + " - equalcheck passed - Given: " + l_mg.getGraphName() + " found: " + l_graphName);
                            }                                
                            if(l_value.trim().length() < 1)
                            {
                                l_mg.setGraphValue("0");
                            }
                            else
                            {
                                // check....
                                try
                                {
                                    if(logMore)
                                    {
                                        logger.info(p_strHostname + " - graph: " + l_mg.getGraphName() + " calling setGraphValue with value: " + l_value.trim());
                                    }                                    
                                    l_mg.setGraphValue(l_value.trim());
                                    // check if we need to add alert value as well
                                    if(l_mg.isInit())
                                    {
                                        ArrayList<Alert> av = getAlertByNidPluginAndGraph(this.get_NodeId(),this.getPluginName(),l_mg.getGraphName());
                                        if(!av.isEmpty())
                                        {
                                            for (Alert l_av : av) {
                                                l_av.addAndCheckSample(getUnixtime(), l_mg.getGraphValue());
                                            }
                                        }
                                    }
                                } catch (Exception ex)
                                {
                                    l_mg.setGraphValue("U");
                                    System.err.println(p_strHostname + " setvalue error on "+this.str_PluginName+" with " + l_value + " details: " + ex.getLocalizedMessage());
                                    ex.printStackTrace();
                                }
                            }
                        }
                        
                    }
                }
            }

            if(customId != null)
            {
                os.close();
                in.close();
                csMuninSocket.shutdownInput();
                csMuninSocket.shutdownOutput();
                csMuninSocket.close();
                
                csMuninSocket = null;
            }

        } catch (Exception ex)
        {
            //this.getCsMuninSocket().close();
            //try { csMuninSocket.close(); } catch (Exception e) {};
            logger.error(p_strHostname + " Unable to connect/process: " + ex.getMessage());
            ex.printStackTrace();
        }
        
    }
    
    public ArrayList<MuninGraph> returnAllGraphs()
    {
        return getV_graphs();   
    }

    void setPluginCategory(String p_strCategory) {
        setStr_PluginCategory(p_strCategory);
    }

    /**
     * @return the str_PluginLabel
     */
    public String getPluginLabel() {
        return str_PluginLabel;
    }

    /**
     * @param str_PluginLabel the str_PluginLabel to set
     */
    public void setPluginLabel(String str_PluginLabel) {
        this.str_PluginLabel = str_PluginLabel;
    }

    /**
     * @return the str_PluginCategory
     */
    public String getStr_PluginCategory() {
        return str_PluginCategory;
    }

    /**
     * @param str_PluginCategory the str_PluginCategory to set
     */
    public void setStr_PluginCategory(String str_PluginCategory) {
        this.str_PluginCategory = str_PluginCategory;
    }

    /**
     * @return the b_IntervalIsSeconds
     */
    public boolean IntervalIsSeconds() {
        return b_IntervalIsSeconds;
    }

    /**
     * @param b_IntervalIsSeconds the b_IntervalIsSeconds to set
     */
    public void set_IntervalIsSeconds(boolean b_IntervalIsSeconds) {
        this.b_IntervalIsSeconds = b_IntervalIsSeconds;
    }

    /**
     * @return the i_nodeId
     */
    public int get_NodeId() {
        return i_nodeId;
    }

    /**
     * @param i_nodeId the i_nodeId to set
     */
    public void set_NodeId(int i_nodeId) {
        this.i_nodeId = i_nodeId;
    }

    /**
     * @return the to_time
     */
    public Integer getTo_time() {
        return to_time;
    }

    /**
     * @param to_time the to_time to set
     */
    public void setTo_time(Integer to_time) {
        this.to_time = to_time;
    }

    /**
     * @return the timezone
     */
    public String getTimezone() {
        return timezone;
    }

    /**
     * @param timezone the timezone to set
     */
    public void setTimezone(String timezone) {
        this.timezone = timezone;
    }

    /**
     * @return the v_graphs
     */
    public ArrayList<MuninGraph> getV_graphs() {
        return v_graphs;
    }

    /**
     * @param v_graphs the v_graphs to set
     */
    public void setV_graphs(ArrayList<MuninGraph> v_graphs) {
        this.v_graphs = v_graphs;
    }

    /**
     * @return the from_time
     */
    public Integer getFrom_time() {
        return from_time;
    }

    /**
     * @param from_time the from_time to set
     */
    public void setFrom_time(Integer from_time) {
        this.from_time = from_time;
    }

    /**
     * @return the customId
     */
    public Integer getCustomId() {
        return customId;
    }

    /**
     * @param customId the customId to set
     */
    public void setCustomId(Integer customId) {
        this.customId = customId;
    }

    /**
     * @return the user_id
     */
    public Integer getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(Integer user_id) {
        this.user_id = user_id;
    }

    /**
     * @return the query_interval
     */
    public Integer getQuery_interval() {
        return query_interval;
    }

    /**
     * @param query_interval the query_interval to set
     */
    public void setQuery_interval(Integer query_interval) {
        this.query_interval = query_interval;
    }

    /**
     * @return the csMuninSocket
     */
    public Socket getCsMuninSocket() {
        return csMuninSocket;
    }

    /**
     * @param csMuninSocket the csMuninSocket to set
     */
    public void setCsMuninSocket(Socket csMuninSocket) {
        this.csMuninSocket = csMuninSocket;
    }

    /**
     * @return the crontab
     */
    public String getCrontab() {
        return crontab;
    }

    /**
     * @param crontab the crontab to set
     */
    public void setCrontab(String crontab) {
        this.crontab = crontab;
    }

    /**
     * @return the str_LineMode
     */
    public String getStr_LineMode() {
        return str_LineMode;
    }

    /**
     * @param str_LineMode the str_LineMode to set
     */
    public void setStr_LineMode(String str_LineMode) {
        this.str_LineMode = str_LineMode;
    }
}
 public class Analyzer implements Runnable {

    private String rcaId;
    private String groupname;
    private String category;
    private Integer user_id = 0;
    private String status = "initialized";
    private int singlehost = 0;
    private boolean running = false;
    private int nodes_affected = 0;
    private int nodes_processed = 0;
    private boolean finished = false;
    private int matchcount = 0;
    private transient ArrayList<MuninNode> rca_nodes = new ArrayList<>();
    private ArrayList<RcaResult> results = new ArrayList<>();
    private int starttime   = 0;
    private BigDecimal percentage = new BigDecimal("10");
    private int querydays = 4;
    private int start = 0;
    private int end = 0;
    private int day = 86400;
    private BigDecimal threshold = new BigDecimal(10.00);
    
    public Analyzer(String p_rcaId) {
        rcaId = p_rcaId;
    }

    public boolean configureFromDatabase() {
        boolean retval = false;
        try {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * from rca WHERE rcaId = '" + getRcaId() + "'");
            while (rs.next()) {
                setGroupname(rs.getString("groupname"));
                setCategory(rs.getString("categoryfilter"));
                setUser_id((Integer) rs.getInt("user_id"));
                setSinglehost(rs.getInt("singlehost"));
                percentage = rs.getBigDecimal("percentage");
                querydays = rs.getInt("querydays") + 1;
                start = rs.getInt("start_time");
                end = rs.getInt("end_time");
                threshold = rs.getBigDecimal("threshold");
                retval = true;
                setStatus("Analyzer configured. Waiting for open slot...");
            }
            conn.close();
        } catch (Exception ex) {
            logger.error("[RCA] Error in configureFromDatabase for rcaId: " + getRcaId() + " - " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }
        return retval;
    }

    @Override
    public void run() {
        setStarttime(getUnixtime());
        setRunning(true);
        com.clavain.muninmxcd.v_analyzer.add(this);
        try {
            int maxjobs = Integer.parseInt(p.getProperty("rca.maxjobs"));

            while (com.clavain.muninmxcd.rcajobs_running > maxjobs) {
                logger.info("[RCA] Cannot start job: " + getRcaId() + " because already " + com.clavain.muninmxcd.rcajobs_running + " of " + maxjobs + " running. Trying again in 30s");
                Thread.sleep(30000);
            }

        } catch (Exception ex) {
            logger.error("[RCA] Error in sleep phase (waiting for slot) for rcaId: " + getRcaId() + " - " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }

        // ok lets go
        try {
            com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running + 1;
            setStatus("Analyzer job preparing - collecting matching nodes");
            User aUser = getUserFromDatabase(getUser_id());
            if (aUser == null) {
                com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running - 1;
                logger.fatal("[RCA] " + getRcaId() + " cannot proceed, user is null");
                setStatus(" cannot proceed, user is null");
                setRunning(false);
                return;
            }
            
            // single host? only add one node
            if(getSinglehost() > 0)
            {
                logger.info("[RCA] " + getRcaId() + " Filtering with Single Host ID: " + getSinglehost());
                MuninNode mn = getMuninNode(getSinglehost());
                if(mn == null)
                {
                    com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running - 1;
                    logger.fatal("[RCA] " + getRcaId() + " cannot proceed, single node ("+getSinglehost()+") is null");  
                    setStatus(" cannot proceed, single node (" + getSinglehost() + ") is null");
                    setRunning(false);
                    return;
                }    
                
                if(aUser.getUserrole().equals("admin"))
                {
                    getRca_nodes().add(mn);
                    logger.info("[RCA] " + getRcaId() + " Added single node: " + mn.getNodename());
                }
                else if(aUser.getUserrole().equals("userext"))
                {
                    if(mn.getUser_id().equals(getUser_id()))
                    {
                        getRca_nodes().add(mn);
                        logger.info("[RCA] " + getRcaId() + " Added single node: " + mn.getNodename());
                    }
                }
                else
                {
                    List<String> accessgroups = Arrays.asList(aUser.getAccessgroup().split(","));
                    if (accessgroups.contains(mn.getGroup()) && mn.getGroup().equals(getGroupname())) {
                        getRca_nodes().add(mn);
                        logger.info("[RCA] " + getRcaId() + " Added single node: " + mn.getNodename());
                     }                    
                }
            }
            // check if we filter on a group base and add node if the user got the permission todo that
            else if (getGroupname() != null) {
                logger.info("[RCA] " + getRcaId() + " Filtering with group: " + getGroupname());
                for (MuninNode l_mn : com.clavain.muninmxcd.v_munin_nodes) {
                    if(l_mn.getGroup() != null)
                    {
                        if (l_mn.getGroup().equals(getGroupname())) {
                            if (aUser.getUserrole().equals("admin")) {
                                getRca_nodes().add(l_mn);
                                logger.info("[RCA] " + getRcaId() + " Added group (" + getGroupname() + ") filtered node: " + l_mn.getNodename());
                            } else if (aUser.getUserrole().equals("userext")) {
                                if (l_mn.getUser_id().equals(aUser.getUser_id()) && l_mn.getGroup().equals(getGroupname())) {
                                    getRca_nodes().add(l_mn);
                                    logger.info("[RCA] " + getRcaId() + " Added group (" + getGroupname() + ") filtered node: " + l_mn.getNodename());
                                }
                            } else if (aUser.getUserrole().equals("user")) {
                                // check if group is in accessgroups
                                List<String> accessgroups = Arrays.asList(aUser.getAccessgroup().split(","));
                                if (accessgroups.contains(l_mn.getGroup()) && l_mn.getGroup().equals(getGroupname())) {
                                    getRca_nodes().add(l_mn);
                                    logger.info("[RCA] " + getRcaId() + " Added group (" + getGroupname() + ") filtered node: " + l_mn.getNodename());
                                }
                            }
                        }
                    }
                }
            } else {
                // fuck add all nodes...
                for (MuninNode l_mn : com.clavain.muninmxcd.v_munin_nodes) {
                    if (aUser.getUserrole().equals("admin")) {
                        getRca_nodes().add(l_mn);
                        logger.info("[RCA] " + getRcaId() + " Added node: " + l_mn.getNodename());
                    } else if (aUser.getUserrole().equals("userext")) {
                        if (l_mn.getUser_id().equals(aUser.getUser_id())) {
                            getRca_nodes().add(l_mn);
                            logger.info("[RCA] " + getRcaId() + " Added node: " + l_mn.getNodename());
                        }
                    } else if (aUser.getUserrole().equals("user")) {
                        // check if group is in accessgroups
                        List<String> accessgroups = Arrays.asList(aUser.getAccessgroup().split(","));
                        if (accessgroups.contains(l_mn.getGroup())) {
                            getRca_nodes().add(l_mn);
                            logger.info("[RCA] " + getRcaId() + " Added node: " + l_mn.getNodename());
                        }
                    }
                }
            }
            setNodes_affected(getRca_nodes().size());

            if(getNodes_affected() < 1)
            {
                setStatus("no nodes found for analyzing matching search query");
                logger.info("[RCA] " + getRcaId() + "no nodes found for analyzing matching search query");
                com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running - 1;
                setFinished(true);
                dbSetRcaFinished(getRcaId());
                setRunning(false);
                //com.clavain.muninmxcd.v_analyzer.remove(this);
                return;
            }
            logger.info("[RCA] " + getRcaId() +"Analyzer job started - processing " + getRca_nodes().size() + " nodes..." );
            // analyze this shit
            setStatus("Analyzer job started - processing " + getRca_nodes().size() + " nodes...");
            
            // BRAIN DAMAGE INCOMING, FOR FOR FOR FOR WHOOOOHOOO!
            for (MuninNode mn : rca_nodes) {
                pluginLoop:
                for (MuninPlugin mp : mn.getPluginList())
                {
                    if(category != null)
                    {
                        if(mp.getStr_PluginCategory() != null)
                        {
                            if(!mp.getStr_PluginCategory().equals(category))
                            {
                                continue pluginLoop;
                            }
                        }
                    }
                    graphLoop:
                    for (MuninGraph mg : mp.getGraphs())
                    {
                        status = "Analyzing " + mn.getNodename() + " - " + mp.getPluginName().toUpperCase() + "/" + mg.getGraphName();
                        //BigDecimal t = getTotalForPluginAndGraph(mp.getPluginName(), mg.getGraphName(), start, end, mn.getUser_id(), mn.getNode_id());
                        BigDecimal t = getAverageForPluginAndGraph(mp.getPluginName(), mg.getGraphName(), start, end, mn.getUser_id(), mn.getNode_id());
                        if(t == null)
                        {
                            continue graphLoop;
                        }
                        // skip if average is below threeshold
                        if(t.compareTo(threshold) == 0 || t.compareTo(threshold) == -1)                        
                        {
                            continue graphLoop;
                        }
                        int iterations = 1;
                        int p_start = start;
                        int p_end = end;
                        ArrayList<BigDecimal> values = new ArrayList<>();
                        iterationLoop:
                        while(iterations < this.querydays)
                        {
                            p_start = start-(day*iterations);
                            p_end = end-(day*iterations); 
                            BigDecimal adding = getAverageForPluginAndGraph(mp.getPluginName(), mg.getGraphName(), p_start, p_end, mn.getUser_id(), mn.getNode_id());
                            if(adding == null)
                            {
                                //
                            }
                            else
                            {
                                values.add(adding);    
                            }
                            //logger.info("[RCA] iteration found: " + adding + " Total: "+t+" query: " + mp.getPluginName() +" / " + mg.getGraphName() + " start: " + p_start+ " end: " + p_end) ;
                            iterations++;
                        }
                        BigDecimal avg = returnAvgBig(values);
                        if(avg == null)
                        {
                            continue graphLoop;
                        }
                        
                        BigDecimal foundPercentage;
                        if(avg.compareTo(t) == 1)
                        {
                            foundPercentage = ReversePercentageFromValues(t,avg);
                        }
                        else
                        {
                             foundPercentage = ReversePercentageFromValues(avg,t);
                        }
                        //logger.info("[RCA] avg: " + avg + " total: " + t + " foundPercentage " + foundPercentage + " query: " + mp.getPluginName() +" / " + mg.getGraphName());
                        // did we receive negative value, then convert to positive?
                        if(foundPercentage.signum() == -1)
                        {
                            foundPercentage = foundPercentage.abs();
                        }
                        
                        // match
                        if(foundPercentage.compareTo(percentage) == 0 || foundPercentage.compareTo(percentage) == 1)
                        {
                            RcaResult result = new RcaResult();
                            result.setNodeId(mn.getNode_id());
                            result.setNodeName(mn.getNodename());
                            result.setGraphName(mg.getGraphName());
                            result.setPluginName(mp.getPluginName());
                            result.setPercentage(foundPercentage);
                            result.setGraphLabel(mg.getGraphLabel());
                            result.setDaysAverage(avg);
                            result.setInputAverage(t);
                            results.add(result);
                            matchcount++;
                            status = "Added Result for node "+mn.getNodename()+" with match of "+foundPercentage+"% on " + mp.getPluginName().toUpperCase()+"/"+mg.getGraphLabel();
                            //logger.info("[RCA] " + getRcaId() + " Added Result for node "+mn.getNodename()+" with match of "+foundPercentage+"% on " + mp.getPluginName().toUpperCase()+"/"+mg.getGraphLabel());
                        }
                    }
                }
                nodes_processed++;
            }

            setFinished(true);
            
            logger.info("[RCA] " + getRcaId() + " job completed");
            
        } catch (Exception ex) {
            com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running - 1;
            setStatus("Error Occured - " + ex.getLocalizedMessage());
            logger.error("[RCA] Error in running thread for rcaId: " + getRcaId() + " - " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }
        
        com.clavain.muninmxcd.rcajobs_running = com.clavain.muninmxcd.rcajobs_running - 1;
        dbSetRcaFinished(getRcaId());
        status = "Analysis complete";
        setFinished(true);
        setRunning(false);
    }

    /**
     * @return the rcaId
     */
    public String getRcaId() {
        return rcaId;
    }

    /**
     * @param rcaId the rcaId to set
     */
    public void setRcaId(String rcaId) {
        this.rcaId = rcaId;
    }

    /**
     * @return the groupname
     */
    public String getGroupname() {
        return groupname;
    }

    /**
     * @param groupname the groupname to set
     */
    public void setGroupname(String groupname) {
        this.groupname = groupname;
    }

    /**
     * @return the category
     */
    public String getCategory() {
        return category;
    }

    /**
     * @param category the category to set
     */
    public void setCategory(String category) {
        this.category = category;
    }

    /**
     * @return the user_id
     */
    public Integer getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(Integer user_id) {
        this.user_id = user_id;
    }

    /**
     * @return the status
     */
    public String getStatus() {
        return status;
    }

    /**
     * @param status the status to set
     */
    public void setStatus(String status) {
        this.status = status;
    }

    /**
     * @return the singlehost
     */
    public int getSinglehost() {
        return singlehost;
    }

    /**
     * @param singlehost the singlehost to set
     */
    public void setSinglehost(int singlehost) {
        this.singlehost = singlehost;
    }

    /**
     * @return the running
     */
    public boolean isRunning() {
        return running;
    }

    /**
     * @param running the running to set
     */
    public void setRunning(boolean running) {
        this.running = running;
    }

    /**
     * @return the nodes_affected
     */
    public int getNodes_affected() {
        return nodes_affected;
    }

    /**
     * @param nodes_affected the nodes_affected to set
     */
    public void setNodes_affected(int nodes_affected) {
        this.nodes_affected = nodes_affected;
    }

    /**
     * @return the finished
     */
    public boolean isFinished() {
        return finished;
    }

    /**
     * @param finished the finished to set
     */
    public void setFinished(boolean finished) {
        this.finished = finished;
        // upload to database
        dbSetRcaOutput(this);
    }

    /**
     * @return the rca_nodes
     */
    public ArrayList<MuninNode> getRca_nodes() {
        return rca_nodes;
    }

    /**
     * @param rca_nodes the rca_nodes to set
     */
    public void setRca_nodes(ArrayList<MuninNode> rca_nodes) {
        this.rca_nodes = rca_nodes;
    }

    /**
     * @return the results
     */
    public ArrayList<RcaResult> getResults() {
        return results;
    }

    /**
     * @param results the results to set
     */
    public void setResults(ArrayList<RcaResult> results) {
        this.results = results;
    }

    /**
     * @return the starttime
     */
    public int getStarttime() {
        return starttime;
    }

    /**
     * @param starttime the starttime to set
     */
    public void setStarttime(int starttime) {
        this.starttime = starttime;
    }
} public class PluginUpdater implements Runnable {

    private MuninNode p_mn_node;
    
    public PluginUpdater(MuninNode it_mn) {
        p_mn_node = it_mn;
    }

    @Override
    public void run() {
        com.clavain.muninmxcd.logger.info("* " + p_mn_node.getHostname()+ " Loading Plugins");
        if(!p_mn_node.loadPlugins())
        {
            com.clavain.muninmxcd.v_munin_nodes.remove(p_mn_node);
            com.clavain.muninmxcd.logger.warn("** Removed " + p_mn_node.getHostname() + " from nodelist because unable to fetch plugins, retrying later");      
        }
        else
        {
            com.clavain.muninmxcd.logger.info("* " + p_mn_node.getHostname()+ " got " + p_mn_node.getGraphCount() + " graphs of " + p_mn_node.getLoadedPlugins().size() + " plugins");
        }
        
    }
    
} public class Database {

    public static String clearStringForSQL(String p_str)
    {
        if(p_str == null)
        {
            return p_str;
        }
        String retval = p_str;
        retval = retval.replaceAll("'","");  
        retval = retval.replaceAll("<",""); 
        retval = retval.replaceAll("`", "");
        retval = retval.replaceAll("´", "");
        retval = retval.replaceAll(";", "");
        return retval;
    }    
    
    // Establish a connection to the database
    public static Connection connectToDatabase(Properties p)
    {
        Connection conn;
        try {
            logger.debug("Connecting to MySQL");
            conn =
               DriverManager.getConnection("jdbc:mysql://"+p.getProperty("mysql.host")+":"+p.getProperty("mysql.port")+"/"+p.getProperty("mysql.db")+"?" +
                                           "user="+p.getProperty("mysql.user")+"&password="+p.getProperty("mysql.pass")+"&autoReconnect=true&failOverReadOnly=false&maxReconnects=10");

            return(conn);

        } catch (Exception ex) {
            // handle any errors
            logger.fatal("Error connecting to database: " + ex.getMessage());
            return(null);
        }
    }  

  
    public static void dbSetRcaFinished(String p_rcaId)
    {
        try {
         Connection conn = connectToDatabase(p);   
         java.sql.Statement stmt = conn.createStatement();
         stmt.executeUpdate("UPDATE rca SET is_finished = 1 WHERE rcaId = '" + p_rcaId+"'");
         conn.close();
        } catch (Exception ex)
        {
            logger.error("[RCA] Error in dbSetRcaFinished: " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }        
    }
    
    public static void dbSetRcaOutput(Analyzer p_analyzer)
    {
        try {
         Gson gson = new GsonBuilder().excludeFieldsWithModifiers(Modifier.TRANSIENT).create();
         String json = gson.toJson(p_analyzer);
         Connection conn = connectToDatabase(p);   
         java.sql.Statement stmt = conn.createStatement();
         stmt.executeUpdate("UPDATE rca SET `output` = '"+json+"' WHERE rcaId = '" + p_analyzer.getRcaId()+"'");
         conn.close();
        } catch (Exception ex)
        {
            logger.error("[RCA] Error in dbSetRcaOutput: " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }          
    }
    
    public static void dbUpdatePluginForNode(Integer nodeId, MuninPlugin mp)
    {
        try {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * from node_plugins WHERE node_id = "+nodeId+" AND pluginname = '"+clearStringForSQL(mp.getPluginName())+"'"); 
            if(rowCount(rs) < 1)
            {
                logger.info("[Node " + nodeId + "] Adding Plugin: " + mp.getPluginName() + " to database");
                stmt.executeUpdate("INSERT INTO node_plugins (node_id,pluginname,plugintitle,plugininfo,plugincategory) VALUES ("+nodeId+",'"+clearStringForSQL(mp.getPluginName())+"','"+clearStringForSQL(mp.getPluginTitle())+"','"+clearStringForSQL(mp.getPluginInfo())+"','"+clearStringForSQL(mp.getStr_PluginCategory())+"')");
            }
            else
            {
                rs = stmt.executeQuery("SELECT * from node_plugins WHERE node_id = "+nodeId+" AND pluginname = '"+clearStringForSQL(mp.getPluginName())+"'");
                while(rs.next())
                {
                    // plugin title or/and category change?
                    if(!clearStringForSQL(mp.getPluginTitle()).equals(rs.getString("plugintitle")))
                    {
                        logger.info("[Node " + nodeId + "] Plugin: " + mp.getPluginName() + " got new Title. Updating Title");
                        stmt.executeUpdate("UPDATE node_plugins SET plugintitle = '" + clearStringForSQL(mp.getPluginTitle()) + "' WHERE node_id = "+nodeId+" AND pluginname = '"+clearStringForSQL(mp.getPluginName())+"'");                 
                    }
                    if(!clearStringForSQL(mp.getStr_PluginCategory()).equals(rs.getString("plugincategory")))
                    {
                        logger.info("[Node " + nodeId + "] Plugin: " + mp.getPluginName() + " got new category. Updating Category (New: "+mp.getStr_PluginCategory()+" Old: "+rs.getString("plugincategory")+" )");
                        stmt.executeUpdate("UPDATE node_plugins SET plugincategory = '" + clearStringForSQL(mp.getStr_PluginCategory()) + "' WHERE node_id = "+nodeId+" AND pluginname = '"+clearStringForSQL(mp.getPluginName())+"'"); 
                    }     
                    if(!clearStringForSQL(mp.getStr_LineMode()).equals(rs.getString("linemode")))
                    {
                        logger.info("[Node " + nodeId + "] Plugin: " + mp.getPluginName() + " got new linemode. Updating linemode (New: "+mp.getStr_LineMode()+" Old: "+rs.getString("linemode")+" )");
                        stmt.executeUpdate("UPDATE node_plugins SET linemode = '" + clearStringForSQL(mp.getStr_LineMode()) + "' WHERE node_id = "+nodeId+" AND pluginname = '"+clearStringForSQL(mp.getPluginName())+"'"); 
                    }                        
                }
            }
            conn.close();
        } catch (Exception ex)
        {
            logger.error("Error in dbUpdatePlugin: " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }
    }
    
    public static void dbUpdateLastContact(Integer nodeId)
    {
        try {
         Connection conn = connectToDatabase(p);   
         java.sql.Statement stmt = conn.createStatement();
         stmt.executeUpdate("UPDATE nodes SET last_contact = NOW() WHERE id = " + nodeId);
         conn.close();
        } catch (Exception ex)
        {
            logger.error("Error in dbUpdateLastContact: " + ex.getLocalizedMessage());
            ex.printStackTrace();
        }
    }
    
    public static void dbUpdateAllPluginsForNode(MuninNode p_mn)
    {
        if(p_mn.getPluginList().size() > 0)
        {
            logger.info("[Job: " + p_mn.getHostname() + "] Updating Database");
            // update graphs in database too
            for(MuninPlugin it_pl : p_mn.getPluginList()) {
                if(it_pl.getGraphs().size() > 0)
                {
                    dbUpdatePluginForNode(p_mn.getNode_id(),it_pl);
                }
            }
            // delete now missing plugins
            dbDeleteMissingPlugins(p_mn.getNode_id(),p_mn.getPluginList());
            logger.info("[Job: " + p_mn.getHostname() + "] Databaseupdate Done");
        }
        else
        {
            logger.warn("[Job: " + p_mn.getHostname() + "] Databaseupdate skipped. Pluginsize is 0");
        }
    }
    
    public static User getUserFromDatabase(Integer user_id)
    {
        User luser = null;
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM users WHERE id = " + user_id);
            while(rs.next())
            {   
                luser = new User();
                luser.setAccessgroup(rs.getString("accessgroup"));
                luser.setUsername(rs.getString("username"));
                luser.setUserrole(rs.getString("userrole"));
                luser.setUser_id(rs.getInt("id"));
            }
        } catch (Exception ex)
        {
            return null;
        }
        
        return luser;
    }
    
    
    public static ServiceCheck getServiceCheckFromDatabase(Integer cid)
    {
        ServiceCheck sc = null;
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM service_checks WHERE id = " + cid);

            while(rs.next())
            {
                Gson gson = new Gson();
                sc = gson.fromJson(rs.getString("json"), ServiceCheck.class);
                sc.setCid(rs.getInt("id"));
                sc.setUser_id(rs.getInt("user_id"));
                v_serviceChecks.add(sc);
                logger.info("* " + sc.getCheckname() + " Service Check added from database");                
            }   
            conn.close();
        } catch (Exception ex)
        {
            logger.error("getServiceCheckFromDatabase Error: " + ex.getLocalizedMessage());
            ex.printStackTrace();
            return null;
        }        
        return sc;
    }
    
    public static MuninNode getMuninNodeFromDatabase(Integer nodeId)
    {
        MuninNode l_mn = new MuninNode();
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM nodes WHERE id = " + nodeId);

            while(rs.next())
            {
                l_mn.setHostname(rs.getString("hostname"));
                l_mn.setNodename(rs.getString("hostname"));
                l_mn.setNode_id(rs.getInt("id"));
                l_mn.setPort(rs.getInt("port"));
                l_mn.setUser_id(rs.getInt("user_id"));
                l_mn.setQueryInterval(rs.getInt("query_interval"));  
                l_mn.setStr_via(rs.getString("via_host"));
                l_mn.setAuthpw(rs.getString("authpw"));
                l_mn.setGroup(rs.getString("groupname"));
            }  
            conn.close();
        } catch (Exception ex)
        {
            logger.error("getMuninNodeFromDatabase Error: " + ex.getLocalizedMessage());
            ex.printStackTrace();
            return null;
        }
        if(l_mn.getHostname().trim().equals("unset"))
        {
            return null;
        }
        else
        {
            return l_mn;
        }
    }
    
    public static void dbDeleteMissingPlugins(Integer nodeId,CopyOnWriteArrayList<MuninPlugin> mps)
    {
        try {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * from node_plugins WHERE node_id = "+nodeId); 
            boolean found = false;
            while(rs.next())
            {
                found = false;
                for(MuninPlugin mp : mps) 
                {
                    if(mp.getPluginName().equals(rs.getString("pluginname")))
                    {
                        found = true;
                    }
                }
                if(found == false)
                {
                    logger.info("[Node " + nodeId + "] Removing Plugin: " + rs.getString("pluginname") + " from Database. Not found on munin node anymore");
                    java.sql.Statement stmtt = conn.createStatement();
                    stmtt.executeUpdate("DELETE FROM node_plugins WHERE id = "+rs.getInt("id"));
                }
            }
            conn.close();
        } catch (Exception ex)
        {
             logger.error("Error in dbDeleteMissingPlugins: " + ex.getLocalizedMessage());
        }
    }
    
    public static void dbScheduleAllCustomJobs()
    {
        try 
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();  
            ResultSet rs = stmt.executeQuery("SELECT * FROM plugins_custom_interval");
            while(rs.next())
            {
                scheduleCustomIntervalJob(rs.getInt("id"));
            }
        } catch (Exception ex)
        {
            logger.error("Startup Schedule for Custom Jobs failed." + ex.getLocalizedMessage());
            ex.printStackTrace();
        }
    }
    
    public static void dbAddAllAlerts()
    {
        try 
        {
            Connection conn = connectToDatabase(p);
            conn.setReadOnly(true);
            //
            java.sql.Statement stmt = conn.createStatement();  
            ResultSet rs = stmt.executeQuery("SELECT alerts.*,nodes.hostname FROM alerts LEFT JOIN nodes ON alerts.node_id = nodes.id");
            while(rs.next())
            {
                Alert av = new Alert();
                av.setAlert_id(rs.getInt("id"));
                av.setCondition(rs.getString("condition"));
                av.setGraphName(rs.getString("graphname"));
                av.setPluginName(rs.getString("pluginname"));
                av.setRaise_value(rs.getBigDecimal("raise_value"));
                av.setNum_samples(rs.getInt("num_samples"));
                av.setAlert_limit(rs.getInt("alert_limit"));
                av.setHostname(rs.getString("hostname"));
                av.setNode_id(rs.getInt("node_id"));
                com.clavain.muninmxcd.v_alerts.add(av);
            }
            logger.info("Startup for Alerts Done");
        } catch (Exception ex)
        {
            logger.error("Startup for Alerts failed. retrying in 60 seconds" + ex.getLocalizedMessage());
            try {
                Thread.sleep(60000);
                dbAddAllAlerts();
            } catch (InterruptedException ex1) {
                logger.error("Startup for Alerts restart failed");
            }
            ex.printStackTrace();
        }     
    }
    
    public static boolean dbAddAllAlertWithId(Integer p_aid)
    {
        boolean retval = false;
        try 
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();  
            ResultSet rs = stmt.executeQuery("SELECT alerts.*,nodes.hostname FROM alerts LEFT JOIN nodes ON alerts.node_id = nodes.id WHERE alerts.id = " + p_aid);
            while(rs.next())
            {
                Alert av = new Alert();
                av.setAlert_id(rs.getInt("id"));
                av.setCondition(rs.getString("condition"));
                av.setGraphName(rs.getString("graphname"));
                av.setPluginName(rs.getString("pluginname"));
                av.setRaise_value(rs.getBigDecimal("raise_value"));
                av.setNum_samples(rs.getInt("num_samples"));
                av.setAlert_limit(rs.getInt("alert_limit"));
                av.setHostname(rs.getString("hostname"));
                av.setNode_id(rs.getInt("node_id"));
                com.clavain.muninmxcd.v_alerts.add(av);
                retval = true;
            }
        } catch (Exception ex)
        {
            logger.error("Add Alert "+p_aid+" failed." + ex.getLocalizedMessage());
            ex.printStackTrace();
        }   
        return retval;
    }    
    
    public static MuninPlugin getMuninPluginForCustomJobFromDb(Integer p_id)
    {
        MuninPlugin retval = new MuninPlugin();
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();  
            ResultSet rs = stmt.executeQuery("SELECT plugins_custom_interval.*,plugins_custom_interval.query_interval AS second_interval , nodes.*, nodes.id AS nodeid, nodes.query_interval AS node_query_interval FROM  `plugins_custom_interval` LEFT JOIN nodes ON plugins_custom_interval.node_id = nodes.id WHERE plugins_custom_interval.id = "+p_id); 
            while(rs.next())
            {
                MuninNode l_node = getMuninNode(rs.getInt("nodeid"));
                if(l_node == null)
                {
                    logger.error("getMuninPluginFromCustomInterval: cannot find MuninNode with id " + rs.getInt("nodeid") + " for custom interval: " + p_id);
                    return null;
                }
                retval.set_IntervalIsSeconds(true);
                retval.set_NodeId(l_node.getNode_id());
                retval.setTo_time(rs.getInt("to_time"));
                retval.setFrom_time(rs.getInt("from_time"));
                retval.setTimezone(rs.getString("timezone"));
                String str_PluginName = rs.getString("pluginname").trim();
                retval.setUser_id(l_node.getUser_id());
                retval.setQuery_interval(rs.getInt("second_interval"));
                retval.setCrontab(rs.getString("crontab"));
                retval.setCustomId(p_id);
                // find plugin for custom interval and copy graphs and plugin informations
                Iterator it = l_node.getPluginList().iterator();
                while(it.hasNext())
                {
                    MuninPlugin l_mp = (MuninPlugin) it.next();
                    if(l_mp.getPluginName().equals(str_PluginName))
                    {
                        retval.setPluginInfo(l_mp.getPluginInfo());
                        retval.setPluginLabel((l_mp.getPluginLabel()));
                        retval.setPluginName(l_mp.getPluginName());
                        retval.setPluginTitle(l_mp.getPluginTitle());
                        retval.setStr_PluginCategory(l_mp.getStr_PluginCategory());
                        
                        // copy graph base informations
                        Iterator git = l_mp.getGraphs().iterator();
                        while(git.hasNext())
                        {
                            MuninGraph old_mg = (MuninGraph) git.next();
                            MuninGraph new_mg = new MuninGraph();
                            new_mg.setGraphDraw(old_mg.getGraphDraw());
                            new_mg.setGraphInfo(old_mg.getGraphInfo());
                            new_mg.setGraphLabel(old_mg.getGraphLabel());
                            new_mg.setGraphName(old_mg.getGraphName());
                            new_mg.setGraphType(old_mg.getGraphType());
                            new_mg.setNegative(old_mg.isNegative());
                            new_mg.setQueryInterval(rs.getInt("second_interval"));
                            new_mg.setIntervalIsSeconds(true);
                            retval.addGraph(new_mg);
                            logger.info("getMuninPluginFromCustomInterval: added graph " + new_mg.getGraphName() + " for custom interval: " + p_id);
                        }
                    }
                }
               return retval;
            }
            
        } catch (Exception ex)
        {
            logger.error("Error in getMuninPluginFromCustomInterval: " + ex.getLocalizedMessage());
            ex.printStackTrace();
            retval = null;
        }
        return retval;
    }
    
    
    public static void dbUpdateNodeDistVerKernel(String p_sum,String p_dist, String p_ver, String p_kernel, int p_nodeid)
    {
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();  
            if(p_dist.contains("/etc/SuSE-release"))
            {
                p_dist = "SuSE";
                p_ver = "unknown";
                p_kernel = "unknown";
            }
            stmt.executeUpdate("UPDATE nodes SET trackpkg_sum = '"+clearStringForSQL(p_sum)+"', track_dist = '"+clearStringForSQL(p_dist)+"', track_ver = '"+clearStringForSQL(p_ver)+"', track_kernel = '"+clearStringForSQL(p_kernel)+"', track_update = NOW() WHERE id = " + p_nodeid);
            
        } catch (Exception ex)
        {
            logger.error("Error in dbUpdateNodeDistVerKernel (Node: "+p_nodeid+") - " + ex.getLocalizedMessage());
        }        
    }
    
    public static boolean dbTrackLogChangedForNode(String p_sum, int p_nodeid)
    {
        boolean retval = false;
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement();  
            ResultSet rs = stmt.executeQuery("SELECT id FROM nodes WHERE id = " + p_nodeid + " AND trackpkg_sum = '"+p_sum+"'"); 
            if(rowCount(rs) < 1)
            {
                retval = true;
            }
        } catch (Exception ex)
        {
            logger.error("Error in dbTrackLogChangedForNode (Node: "+p_nodeid+" Sum: "+p_sum+" ) - " + ex.getLocalizedMessage());
        }
        return retval;
    }
    
    public static void removeOldPackageTrack(int p_nodeid)
    {
        try {
            
            logger.info("Purging Package Logs for NodeID: " + p_nodeid);
            DB db;
            String dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbessentials");
            db = m.getDB(dbName);
            db.setWriteConcern(WriteConcern.SAFE);
            DBCollection col = db.getCollection("trackpkg");   
            BasicDBObject query = new BasicDBObject();
            query.append("node", p_nodeid);
            col.remove(query);
            db.setWriteConcern(WriteConcern.NONE);
        } catch (Exception ex)
        {
            logger.error("Error in removeOldPackageTrack: " + ex.getLocalizedMessage());
        }
    }
    
    public static int rowCount(ResultSet rs) throws SQLException
    {
        int rsCount = 0;
        while(rs.next())
        {
            //do your other per row stuff 
            rsCount = rsCount + 1;
        }//end while
        return rsCount;
    } 
    
    public static boolean serviceCheckGotDowntime(Integer cid, int timestamp)
    {
        boolean retval = false;
        try
        {
            Connection conn = connectToDatabase(p);
            java.sql.Statement stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,ResultSet.CONCUR_READ_ONLY);
            ResultSet rs = stmt.executeQuery("SELECT * FROM downtimes WHERE check_id = " + cid);
            if(rowCount(rs) > 0)
            {
                Integer ftime;
                Integer ttime;
                rs.beforeFirst();
                while(rs.next())
                {
                    if(rs.getInt("repeating") == 1)
                    {
                        ftime =  (int) getStampFromTimeAndZone(rs.getString("from_time"),rs.getString("timezone"));
                        ttime =  (int) getStampFromTimeAndZone(rs.getString("to_time"),rs.getString("timezone"));
                    }
                    else
                    {
                        ftime = Integer.parseInt(rs.getString("from_time"));
                        ttime = Integer.parseInt(rs.getString("to_time"));
                    }
                    if(ftime < timestamp && timestamp < ttime)
                    {
                        return true;
                    }
                }
            }
        } catch (Exception ex)
        {
            com.clavain.muninmxcd.logger.error("serviceCheckGotDowntime Error: " + ex.getLocalizedMessage());
        }
        return retval;
    }    
} public class Quartz {
    
    public static boolean scheduleServiceCheck(Integer p_cid)
    {
        ServiceCheck sc = getServiceCheckFromDatabase(p_cid);
        if(sc == null)
        {
            return false;
        }
        return scheduleServiceCheck(sc);
    }
    
    // schedule a new check
    public static boolean scheduleServiceCheck(ServiceCheck p_sc)
    {
        boolean l_retVal = false;   
        Trigger trigger = newTrigger().withIdentity("checktrigger", p_sc.getUser_id().toString() + p_sc.getCid() + System.currentTimeMillis()).startNow().withSchedule(simpleSchedule().withIntervalInMinutes(p_sc.getInterval()) .repeatForever().withMisfireHandlingInstructionFireNow()).build(); 
        JobDetail job = newJob(CheckJob.class).withIdentity(p_sc.getCid().toString(), p_sc.getUser_id().toString()).usingJobData("cid", p_sc.getCid()).build();
        try
        {
            com.clavain.muninmxcd.sched_checks.scheduleJob(job, trigger);
            l_retVal = true;
        } catch (Exception ex)
        {
            com.clavain.muninmxcd.logger.error(ex);
        }
        return l_retVal;
    }    
    
    // delete a job
    public static boolean unscheduleServiceCheck(String p_cid, String p_uid)
    {
        boolean l_retVal = false;
        JobKey jk = new JobKey(p_cid.toString(),p_uid.toString());
        try {
            com.clavain.muninmxcd.sched_checks.deleteJob(jk);
            l_retVal = true;
        } catch (SchedulerException ex) {
            com.clavain.muninmxcd.logger.error(ex.getLocalizedMessage());
        }
        
        return l_retVal;
    }    
    
    public static boolean isServiceCheckScheduled(int p_cid)
    {
        boolean retval = false;
        String match = p_cid+"";
        
        for(ScheduledJob sj : getScheduledServiceChecks())
        {
            if(sj.getJobName().equals(match))
            {
                return true;
            }
        }
        
        return retval;
    }  
    
    public static ArrayList<ScheduledJob> getScheduledServiceChecks()
    {
        ArrayList<ScheduledJob> retval = new ArrayList<>();
        try {
            for (String groupName : com.clavain.muninmxcd.sched_checks.getJobGroupNames()) {

                for (JobKey jobKey : com.clavain.muninmxcd.sched_checks.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) {

                     String jobName = jobKey.getName();
                     String jobGroup = jobKey.getGroup();

                     //get job's trigger
                     List<Trigger> triggers = (List<Trigger>) com.clavain.muninmxcd.sched_checks.getTriggersOfJob(jobKey);
                     Date nextFireTime = triggers.get(0).getNextFireTime(); 
                     ScheduledJob sj = new ScheduledJob();
                     sj.setJobName(jobName);
                     sj.setGroupName(jobGroup);
                     sj.setNextFireTime(nextFireTime.toString());
                     retval.add(sj);
                     }

               }
        } catch (SchedulerException ex) {
            com.clavain.muninmxcd.logger.error(ex.getLocalizedMessage());
            ex.printStackTrace();
        }
        return retval;
    }        
    
    // schedule a new check
    public static boolean scheduleJob(MuninNode mn)
    {
        boolean l_retVal = false;   
        String uid = mn.getUser_id().toString();
        Trigger trigger = newTrigger().withIdentity("trigger", uid + mn.getNode_id() + System.currentTimeMillis()).startNow().withSchedule(simpleSchedule().withIntervalInMinutes(mn.getQueryInterval()) .repeatForever().withMisfireHandlingInstructionFireNow()).build(); 
        JobDetail job = newJob(MuninJob.class).withIdentity(mn.getNode_id().toString(), mn.getUser_id().toString()).usingJobData("nodeId", mn.getNode_id()).build();

        try
        {
            sched.scheduleJob(job, trigger);
            logger.info("Scheduled Job for Node: " + mn.getHostname() + " with interval: " + mn.getQueryInterval() + " minutes");
            l_retVal = true;
        } catch (Exception ex)
        {
            logger.error("Unable to Schedule Job for Node: " + mn.getHostname());
            logger.error(ex);
        }
        return l_retVal;
    }

    public static boolean isScheduled(Integer p_nodeid, Integer p_user_id)
    {
        boolean l_retVal = false;
        String jobSearch = p_user_id + "." + p_nodeid; 
        try {
            // enumerate each job group
            for(String group: com.clavain.muninmxcd.sched.getJobGroupNames()) {
                // enumerate each job in group
                for(JobKey jobKey : com.clavain.muninmxcd.sched.getJobKeys((GroupMatcher<JobKey>) groupEquals(group))) {
                    if(jobKey.toString().equals(jobSearch))
                    {
                        l_retVal = true;
                    }
                }
            }
        } catch (SchedulerException ex) {
            logger.error("Error in isScheduled: " + ex.getLocalizedMessage());
        }
        return l_retVal;
    }
  
    public static boolean isJobScheduled(int p_nodeid)
    {
        boolean retval = false;
        String match = p_nodeid+"";
        for(ScheduledJob sj : getScheduledJobs())
        {
            if(sj.getJobName().equals(match))
            {
                return true;
            }
        }
        
        return retval;
    }    
    
    
    public static boolean unscheduleCheck(String p_nodeid, String p_uid)
    {
        boolean l_retVal = false;
        JobKey jk = new JobKey(p_nodeid,p_uid);
        try {
            com.clavain.muninmxcd.sched.deleteJob(jk);
            l_retVal = true;
        } catch (SchedulerException ex) {
            logger.error("Error in unscheduleCheck: " + ex.getLocalizedMessage());
        }
        
        return l_retVal;
    }    
    
   public static boolean isCustomJobScheduled(int p_cid)
    {
        boolean retval = false;
        String match = p_cid+"";
        for(ScheduledJob sj : getScheduledCustomJobs())
        {
            if(sj.getJobName().equals(match))
            {
                return true;
            }
        }
        
        return retval;
    }    
    
    
    public static boolean unscheduleCustomJob(String p_cid, String p_uid)
    {
        boolean l_retVal = false;
        JobKey jk = new JobKey(p_cid,p_uid);
        try {
            com.clavain.muninmxcd.sched_custom.deleteJob(jk);
            com.clavain.muninmxcd.v_cinterval_plugins.remove(getMuninPluginForCustomJob(Integer.parseInt(p_cid)));
            l_retVal = true;
        } catch (SchedulerException ex) {
            logger.error("Error in unscheduleCustomCheck: " + ex.getLocalizedMessage());
        }
        
        return l_retVal;
    }   
    
    // schedule a custom interval check
    public static boolean scheduleCustomIntervalJob(Integer p_cid)
    {
        boolean retval = false;
        MuninPlugin l_mp = getMuninPluginForCustomJobFromDb(p_cid);
        if(l_mp != null)
        {
           String uid = l_mp.getUser_id().toString();
           String cinterval = "";
           // build trigger
           Trigger trigger;
           // crontab trigger
           if(!l_mp.getCrontab().equals("false"))
           {
             // fixed start/end ?
             if(l_mp.getFrom_time() == 0)
             {
                trigger = newTrigger().withIdentity("trigger", uid + l_mp.getCustomId() + System.currentTimeMillis()).withSchedule(CronScheduleBuilder.cronSchedule(l_mp.getCrontab()).withMisfireHandlingInstructionFireAndProceed().inTimeZone(TimeZone.getTimeZone(l_mp.getTimezone()))).build();        
                cinterval = " every " + l_mp.getQuery_interval() + " seconds with crontab: " + l_mp.getCrontab();
             }
             else
             {
                long a = l_mp.getFrom_time();
                long b = l_mp.getTo_time();

                Date startDate = new Date(a*1000L);
                Date endDate = new Date(b*1000L);  
                trigger = newTrigger().withIdentity("trigger", uid + l_mp.getCustomId() + System.currentTimeMillis()).startAt(startDate).withSchedule(CronScheduleBuilder.cronSchedule(l_mp.getCrontab()).withMisfireHandlingInstructionFireAndProceed().inTimeZone(TimeZone.getTimeZone(l_mp.getTimezone()))).endAt(endDate).build();        
                cinterval = " every " + l_mp.getQuery_interval() + " seconds with crontab: " + l_mp.getCrontab() + " from " + startDate + " to " + endDate;
               
             }
           }
           else
           {
                // standard repeat forever trigger
                if(l_mp.getFrom_time() == 0)
                {
                    trigger = newTrigger().withIdentity("trigger", uid + l_mp.getCustomId() + System.currentTimeMillis()).startNow().withSchedule(simpleSchedule().withIntervalInSeconds(l_mp.getQuery_interval()).repeatForever().withMisfireHandlingInstructionFireNow()).build(); 
                    cinterval = " every " + l_mp.getQuery_interval() + " seconds";
                }
                // daterange trigger, ignore missfire
                else
                {
                    long a = l_mp.getFrom_time();
                    long b = l_mp.getTo_time();

                    Date startDate = new Date(a*1000L);
                    Date endDate = new Date(b*1000L);
                    long cur = (System.currentTimeMillis() / 1000L);
                    trigger = newTrigger().withIdentity("trigger", uid + l_mp.getCustomId() + System.currentTimeMillis()).startAt(startDate).withSchedule(simpleSchedule().withIntervalInSeconds(l_mp.getQuery_interval()).repeatForever().withMisfireHandlingInstructionIgnoreMisfires()).endAt(endDate).build(); 
                    cinterval = " every " + l_mp.getQuery_interval() + " seconds from " + startDate + " to " + endDate;
                }
           }
           //Trigger trigger = newTrigger().withIdentity("trigger", uid + l_mp.get_NodeId() + System.currentTimeMillis()).startNow().withSchedule(simpleSchedule().withIntervalInMinutes(l_mp.get .repeatForever().withMisfireHandlingInstructionFireNow()).build(); 
           JobDetail job = newJob(CustomJob.class).withIdentity(l_mp.getCustomId() + "", l_mp.getUser_id().toString()).usingJobData("customId", l_mp.getCustomId()).build();

           try
           {
               com.clavain.muninmxcd.sched_custom.scheduleJob(job, trigger);
               logger.info("Scheduled CustomJob for custom interval: " + l_mp.getCustomId() + " with interval " + cinterval);
               retval = true;
               com.clavain.muninmxcd.v_cinterval_plugins.add(l_mp);
           } catch (Exception ex)
           {
               logger.error("Unable to Schedule Job for custom interval:" + l_mp.getCustomId() + " with interval " + cinterval);
               logger.error(ex.getLocalizedMessage());
           }           

        }
        return retval;
    }
    
    public static ArrayList<ScheduledJob> getScheduledJobs()
    {
        ArrayList<ScheduledJob> retval = new ArrayList<>();
        try {
            for (String groupName : com.clavain.muninmxcd.sched.getJobGroupNames()) {

                for (JobKey jobKey : com.clavain.muninmxcd.sched.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) {

                     String jobName = jobKey.getName();
                     String jobGroup = jobKey.getGroup();

                     //get job's trigger

                     List<Trigger> triggers = (List<Trigger>) com.clavain.muninmxcd.sched.getTriggersOfJob(jobKey);
                     Date nextFireTime = triggers.get(0).getNextFireTime(); 
                     ScheduledJob sj = new ScheduledJob();
                     sj.setJobName(jobName);
                     sj.setGroupName(jobGroup);
                     sj.setNextFireTime(nextFireTime.toString());
                     retval.add(sj);
                     }

               }
        } catch (SchedulerException ex) {
            logger.error("Error in getScheduledJobs(): " + ex.getLocalizedMessage());
        }
        return retval;
    } 
 
    public static ArrayList<ScheduledJob> getScheduledCheckJobs()
    {
        ArrayList<ScheduledJob> retval = new ArrayList<>();
        try {
            for (String groupName : com.clavain.muninmxcd.sched_checks.getJobGroupNames()) {

                for (JobKey jobKey : com.clavain.muninmxcd.sched_checks.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) {

                     String jobName = jobKey.getName();
                     String jobGroup = jobKey.getGroup();

                     //get job's trigger
                     List<Trigger> triggers = (List<Trigger>) com.clavain.muninmxcd.sched_checks.getTriggersOfJob(jobKey);
                     Date nextFireTime = triggers.get(0).getNextFireTime(); 
                     ScheduledJob sj = new ScheduledJob();
                     sj.setJobName(jobName);
                     sj.setGroupName(jobGroup);
                     sj.setNextFireTime(nextFireTime.toString());
                     retval.add(sj);
                     }

               }
        } catch (SchedulerException ex) {
            logger.error("Error in getScheduledJobs(): " + ex.getLocalizedMessage());
        }
        return retval;
    }       
    
    public static ArrayList<ScheduledJob> getScheduledCustomJobs()
    {
        ArrayList<ScheduledJob> retval = new ArrayList<>();
        try {
            for (String groupName : com.clavain.muninmxcd.sched_custom.getJobGroupNames()) {

                for (JobKey jobKey : com.clavain.muninmxcd.sched_custom.getJobKeys(GroupMatcher.jobGroupEquals(groupName))) {

                     String jobName = jobKey.getName();
                     String jobGroup = jobKey.getGroup();

                     //get job's trigger
                     List<Trigger> triggers = (List<Trigger>) com.clavain.muninmxcd.sched_custom.getTriggersOfJob(jobKey);
                     Date nextFireTime = triggers.get(0).getNextFireTime(); 
                     ScheduledJob sj = new ScheduledJob();
                     sj.setJobName(jobName);
                     sj.setGroupName(jobGroup);
                     sj.setNextFireTime(nextFireTime.toString());
                     retval.add(sj);
                     }

               }
        } catch (SchedulerException ex) {
            logger.error("Error in getScheduledJobs(): " + ex.getLocalizedMessage());
        }
        return retval;
    }      
} public class SocketCheck {
    private Socket cs;
    private int socketCreated;
    private String hostname;
    
    public SocketCheck(Socket p_cs, int p_sockettime)
    {
        setSocket(p_cs);
        setSocketCreated(p_sockettime);
    }

    public void closeSocket()
    {
        try {
            cs.close();
        } catch (IOException ex) {
            //Logger.getLogger(SocketCheck.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    
    /**
     * @return the cs
     */
    public Socket getSocket() {
        return cs;
    }

    /**
     * @param cs the cs to set
     */
    public void setSocket(Socket cs) {
        this.cs = cs;
    }

    /**
     * @return the socketCreated
     */
    public int getSocketCreated() {
        return socketCreated;
    }

    /**
     * @param socketCreated the socketCreated to set
     */
    public void setSocketCreated(int socketCreated) {
        this.socketCreated = socketCreated;
    }

    public void setHostname(String p_hostname) {
       hostname = p_hostname;
    }
    
    public String getHostname() {
       return hostname;
    }    
   
} public class MongoWorker implements Runnable {
    private DB db;
    private DBCollection col;
    
    @Override
    public void run() {
        
        String dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbname");
        db = m.getDB(dbName);
        
        logger.info("Started MongoWorker");
        String plugin = "";
        while(true)
        { 
            try
            {
                BasicDBObject doc = mongo_queue.take();
                if(doc != null)
                {
                    plugin = doc.getString("plugin");
                    // each hostname got its own collection
                    col = db.getCollection(doc.getString("user_id") + "_" + doc.getString("nodeid"));
                    doc.removeField("hostname");
                    doc.removeField("nodeid");
                    doc.removeField("user_id");
                    //doc.removeField("plugin");
                    //db.requestStart();
                    col.insert(doc);
                    if(logMore)
                    {
                        logger.info("Mongo: Wrote " + plugin + " / " + doc.getString("graph") + " / " + doc.getString("value"));
                    }
                    //db.requestDone();
                }
                else
                {
                    Thread.sleep(50);    
                }
                
            } catch (Exception ex)
            {
                logger.fatal("Error in MongoWorker: " + ex.getLocalizedMessage());
                ex.printStackTrace();
            }
            
        }
    }
    
} public static int getUnixtime() {
    return (int) (System.currentTimeMillis() / 1000L);
} public class NewNodeWatcher implements Runnable {

    @Override
    public void run() {
       logger.info("NewNodeWatcher (database) started");
       Connection conn;
       while(true)
       {
           try {
               Thread.sleep(60000);
               conn = connectToDatabase(p);
               java.sql.Statement stmt = conn.createStatement();
               ResultSet rs = stmt.executeQuery("SELECT * from nodes");
               while(rs.next())
               {
                   Integer nodeID = rs.getInt("id");
                   if(getMuninNode(nodeID) == null)
                   {
                       MuninNode mn = new MuninNode();
                       mn.setHostname(rs.getString("hostname"));
                       mn.setNodename(rs.getString("hostname"));
                       mn.setNode_id(rs.getInt("id"));
                       mn.setPort(rs.getInt("port"));
                       mn.setUser_id(rs.getInt("user_id"));
                       mn.setQueryInterval(rs.getInt("query_interval"));
                       mn.setStr_via(rs.getString("via_host"));
                       mn.setAuthpw(rs.getString("authpw"));
                       v_munin_nodes.add(mn); 
                       scheduleJob(mn);
                       logger.info("NewNodeWatcher found new node in database: " + mn.getHostname() + " added and job scheduled");
                   }
               }
               
               conn.close();
           } catch (Exception ex)
           {
               logger.error("Error in NewNodeWatcher: " + ex.getLocalizedMessage());
           }
       }
    }
    
} public static void dbScheduleAllCustomJobs()
{
    try 
    {
        Connection conn = connectToDatabase(p);
        java.sql.Statement stmt = conn.createStatement();  
        ResultSet rs = stmt.executeQuery("SELECT * FROM plugins_custom_interval");
        while(rs.next())
        {
            scheduleCustomIntervalJob(rs.getInt("id"));
        }
    } catch (Exception ex)
    {
        logger.error("Startup Schedule for Custom Jobs failed." + ex.getLocalizedMessage());
        ex.printStackTrace();
    }
} public class DataRetentionWorker implements Runnable {
    private DB db;
    private DBCollection col;
    
    @Override
    public void run() {
          int month = 2629743;
          logger.info("Started DataRetentionWorker");        
          try
          {
              int sleepTime = Integer.parseInt(p.getProperty("dataretention.period")) * 3600000;
              while(true)
              {
                   Thread.sleep(sleepTime);
                   logger.info("[DataRetentionWorker] Starting Retention Run");
                   Connection conn = connectToDatabase(p);
                   java.sql.Statement stmt = conn.createStatement();
                   ResultSet rs = stmt.executeQuery("SELECT * FROM users WHERE retention > 0 AND userrole != 'user'");
                   while(rs.next())
                   {
                       logger.info("[DataRetentionWorker - User Mode] Processing User: " + rs.getString("username"));
                       // get nodes from this user
                       Iterator it = v_munin_nodes.iterator();
                       List l_nodes = new ArrayList();
                       while (it.hasNext()) 
                       {
                        MuninNode l_mn = (MuninNode) it.next();
                        if(l_mn.getUser_id().equals(rs.getInt("id")))
                        {
                            logger.info("[DataRetentionWorker User Mode] probing " + l_mn.getHostname() + " from user: " + rs.getString("username"));
                            String colname = l_mn.getUser_id()+"_"+l_mn.getNode_id(); // recv
                            String colnamees = l_mn.getNode_id()+"_ess";  // time
                            int matchtime = rs.getInt("retention") * month;
                            matchtime = getUnixtime() - matchtime;
                            BasicDBObject query = new BasicDBObject("recv", new BasicDBObject("$lt", matchtime));
                            String dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbname");
                            db = m.getDB(dbName);
                            col = db.getCollection(colname);
                            DBCursor cursor = col.find(query);
                            if(cursor.count() > 0)
                            {
                                logger.info("[DataRetentionWorker User Mode] result for " + l_mn.getHostname() + " from user: " + rs.getString("username") + " affected for deletion: " + cursor.count() + " matchtime: lt " + matchtime);
                            }
                            col.remove(query);
                            
                            // now ESSENTIALS
                            query = new BasicDBObject("time", new BasicDBObject("$lt", matchtime));
                            dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbessentials");
                            db = m.getDB(dbName);
                            col = db.getCollection(colnamees);
                            cursor = col.find(query);
                            if(cursor.count() > 0)
                            {
                                logger.info("[DataRetentionWorker User Mode] ESSENTIAL result for " + l_mn.getHostname() + " from user: " + rs.getString("username") + " affected for deletion: " + cursor.count() + " matchtime: lt " + matchtime);
                            }                            
                            col.remove(query);
                            
                          }
                       }
                   }
                   //conn.close();
                   
                   logger.info("[DataRetentionWorker User Mode] Finished Retention Run");
                   
                   logger.info("[DataRetentionWorker Custom Mode] Starting Retention Run");
                   stmt = conn.createStatement();
                   rs.close();
                   rs = stmt.executeQuery("SELECT * FROM plugins_custom_interval WHERE retention > 0");
                   while(rs.next())
                   {  
                        logger.info("[DataRetentionWorker - Custom Mode] Processing Custom ID: " + rs.getString("id") + " Node: " + rs.getString("node_id") + " Plugin: " + rs.getString("pluginname")); 
                        MuninNode l_mn = getMuninNode(rs.getInt("node_id"));
                        if(l_mn != null)
                        {
                            String colname = l_mn.getUser_id()+"_"+l_mn.getNode_id(); // recv
                            int matchtime = rs.getInt("retention") * 86400;
                            matchtime = getUnixtime() - matchtime;
                            BasicDBObject query = new BasicDBObject("recv", new BasicDBObject("$lt", matchtime));
                            query.append("customId", rs.getInt("id"));
                            String dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbname");
                            db = m.getDB(dbName);
                            col = db.getCollection(colname);
                            DBCursor cursor = col.find(query);
                            if(cursor.count() > 0)
                            {
                                logger.info("[DataRetentionWorker Custom Mode] Custom Interval (CID: "+rs.getInt("id")+") POSITIVE RESULTS for " + l_mn.getHostname() + " affected for deletion: " + cursor.count() + " collection: " + colname + " matchtime: lt " + matchtime);
                            }
                            else
                            {
                                logger.info("[DataRetentionWorker Custom Mode] Custom Interval (CID: "+rs.getInt("id")+") NEGATIVE RESULTS for " + l_mn.getHostname() + "count:  " + cursor.count() + " collection: " + colname +" matchtime: lt " + matchtime);
                            }
                            col.remove(query);
                           
                        }
                        else
                        {
                            logger.warn("[DataRetentionWorker Custom Mode] getMuninNode returned null for node_id " + rs.getInt("node_id"));
                        }
                   }                 
                   logger.info("[DataRetentionWorker Custom Mode] Finished Retention Run");
                   
                   // Service Checks
                   logger.info("[DataRetentionWorker ServiceCheck Mode] Starting Retention Run");
                   stmt = conn.createStatement();
                   rs.close();
                   rs = stmt.executeQuery("SELECT service_checks.*,users.retention,users.username FROM service_checks LEFT JOIN users on service_checks.user_id = users.id WHERE users.retention > 0");
                   while(rs.next())
                   {  
                        logger.info("[DataRetentionWorker - ServiceCheck Mode] Processing ServiceCheck ID: " + rs.getString("id") + " Name: " + rs.getString("check_name") + " for User: " + rs.getString("username") + " Retention: " + rs.getString("retention")); 
                        
                        String colname = rs.getInt("user_id") + "cid"+rs.getInt("id");
                        int matchtime = rs.getInt("retention") * month;
                        matchtime = getUnixtime() - matchtime;
                        BasicDBObject query = new BasicDBObject("time", new BasicDBObject("$lt", matchtime));
                        query.append("cid", rs.getInt("id"));
                        query.append("status", 0);
                        String dbName = com.clavain.muninmxcd.p.getProperty("mongo.dbchecks");
                        db = m.getDB(dbName);
                        col = db.getCollection(colname);
                        DBCursor cursor = col.find(query);
                        if(cursor.count() > 0)
                        {
                            logger.info("[DataRetentionWorker ServiceCheck Mode] ServiceCheck ID: " + rs.getString("id") + " Name: " + rs.getString("check_name") + " for User: " + rs.getString("username") + " affected for deletion: " + cursor.count() + " matchtime: lt " + matchtime);
                        }
                        col.remove(query);                            
                        
                   }                                    
                   logger.info("[DataRetentionWorker ServiceCheck Mode] Finished Retention Run");
                   conn.close();
              }
             
          } catch (Exception ex)
          {
              logger.error("Error in DataRetentionWorker: " + ex.getLocalizedMessage());
          }
    }
    
} public class ReturnServiceCheck {
    private Integer returnValue;
    private List<String> output;
    private Integer user_id;
    private Integer cid;
    private Integer checktime;
    private Integer downTimeConfirmedAt;
    private Integer lastDownTimeConfirm;
    private String checktype;
    private String checkname;
    private Integer interval;
    private Integer notifydown = 0;
    private Integer notifyagain = 0;
    private Integer notifyifup = 0;
    private Integer notifyflap = 0;
    private String json;
    
    /**
     * @return the returnValue
     */
    public Integer getReturnValue() {
        return returnValue;
    }

    /**
     * @param aReturnValue the returnValue to set
     */
    public void setReturnValue(Integer aReturnValue) {
        returnValue = aReturnValue;
    }

    /**
     * @return the output
     */
    public List<String> getOutput() {
        return output;
    }

    /**
     * @param aOutput the output to set
     */
    public void setOutput(List<String> aOutput) {
        output = aOutput;
    }
    
    public ReturnServiceCheck(Integer p_iReturnValue,List<String> p_returnOutput)
    {
        returnValue = p_iReturnValue;
        output = p_returnOutput;
    }

    /**
     * @return the user_id
     */
    public Integer getUser_id() {
        return user_id;
    }

    /**
     * @param user_id the user_id to set
     */
    public void setUser_id(Integer user_id) {
        this.user_id = user_id;
    }

    /**
     * @return the cid
     */
    public Integer getCid() {
        return cid;
    }

    /**
     * @param cid the cid to set
     */
    public void setCid(Integer cid) {
        this.cid = cid;
    }

    /**
     * @return the checktime
     */
    public Integer getChecktime() {
        return checktime;
    }

    /**
     * @param checktime the checktime to set
     */
    public void setChecktime(Integer checktime) {
        this.checktime = checktime;
    }

    /**
     * @return the downTimeConfirmedAt
     */
    public Integer getDownTimeConfirmedAt() {
        return downTimeConfirmedAt;
    }

    /**
     * @param downTimeConfirmedAt the downTimeConfirmedAt to set
     */
    public void setDownTimeConfirmedAt(Integer downTimeConfirmedAt) {
        this.downTimeConfirmedAt = downTimeConfirmedAt;
    }

    /**
     * @return the lastDownTimeConfirm
     */
    public Integer getLastDownTimeConfirm() {
        return lastDownTimeConfirm;
    }

    /**
     * @param lastDownTimeConfirm the lastDownTimeConfirm to set
     */
    public void setLastDownTimeConfirm(Integer lastDownTimeConfirm) {
        this.lastDownTimeConfirm = lastDownTimeConfirm;
    }

    /**
     * @return the checktype
     */
    public String getChecktype() {
        return checktype;
    }

    /**
     * @param checktype the checktype to set
     */
    public void setChecktype(String checktype) {
        this.checktype = checktype;
    }

    /**
     * @return the checkname
     */
    public String getCheckname() {
        return checkname;
    }

    /**
     * @param checkname the checkname to set
     */
    public void setCheckname(String checkname) {
        this.checkname = checkname;
    }

    /**
     * @return the interval
     */
    public Integer getInterval() {
        return interval;
    }

    /**
     * @param interval the interval to set
     */
    public void setInterval(Integer interval) {
        this.interval = interval;
    }

    /**
     * @return the notifydown
     */
    public Integer getNotifydown() {
        return notifydown;
    }

    /**
     * @param notifydown the notifydown to set
     */
    public void setNotifydown(Integer notifydown) {
        this.notifydown = notifydown;
    }

    /**
     * @return the notifyagain
     */
    public Integer getNotifyagain() {
        return notifyagain;
    }

    /**
     * @param notifyagain the notifyagain to set
     */
    public void setNotifyagain(Integer notifyagain) {
        this.notifyagain = notifyagain;
    }

    /**
     * @return the notifyifup
     */
    public Integer getNotifyifup() {
        return notifyifup;
    }

    /**
     * @param notifyifup the notifyifup to set
     */
    public void setNotifyifup(Integer notifyifup) {
        this.notifyifup = notifyifup;
    }

    /**
     * @return the notifyflap
     */
    public Integer getNotifyflap() {
        return notifyflap;
    }

    /**
     * @param notifyflap the notifyflap to set
     */
    public void setNotifyflap(Integer notifyflap) {
        this.notifyflap = notifyflap;
    }

    /**
     * @return the json
     */
    public String getJson() {
        return json;
    }

    /**
     * @param json the json to set
     */
    public void setJson(String json) {
        this.json = json;
    }
    
    
} public class MongoCheckExecutor implements Runnable {

    @Override
    public void run() {
        try
        {
            while(true)
            {
                Thread.sleep(50);
                ExecutorService executor = Executors.newFixedThreadPool(2);
                
                executor.execute(new MongoCheckWorker());

                executor.shutdown();
                while (!executor.isTerminated()) {
                    Thread.sleep(50);
                }
                logger.info("MongoCheckWorker Set finished. Restarting MongoCheckWorkers.");
                System.gc();
            }
        } catch (Exception ex)
        {
            logger.fatal("Error in MongoCheckExecutor: " + ex.getLocalizedMessage());
        }
    }
    
} public class ErrorNotifyExecutor implements Runnable {

    @Override
    public void run() {
        try
        {
            while(true)
            {
                Thread.sleep(500);
                ExecutorService executor = Executors.newFixedThreadPool(Integer.parseInt(p.getProperty("errornotify.threads")));
                
                executor.execute(new ErrorNotifyWorker());

                executor.shutdown();
                while (!executor.isTerminated()) {
                    Thread.sleep(100);
                }
                logger.info("ErrorNotifyWorker Set finished. Restarting.");
                System.gc();
            }
        } catch (Exception ex)
        {
            logger.fatal("Error in ErrorNotifyWorker " + ex.getLocalizedMessage());
        }
    }
    
}
import com.clavain.alerts.Alert;
import com.clavain.alerts.msg.PushOverMessage;
import com.clavain.alerts.msg.ShortTextMessage;
import com.clavain.alerts.msg.TTSMessage;
import com.clavain.alerts.ratelimiter.PushOverLimiter;
import com.clavain.alerts.ratelimiter.SMSLimiter;
import com.clavain.alerts.ratelimiter.TTSLimiter;
import com.clavain.checks.ReturnDebugTrace;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import com.mongodb.WriteConcern;
import com.clavain.executors.MongoExecutor;
import com.clavain.executors.MongoEssentialExecutor;
import com.clavain.handlers.JettyLauncher;
import com.clavain.json.ServiceCheck;
import com.clavain.munin.MuninNode;
import com.clavain.munin.MuninPlugin;
import com.clavain.rca.Analyzer;
import com.clavain.workers.PluginUpdater;
import java.io.FileInputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import static com.clavain.utils.Database.*;
import static com.clavain.utils.Quartz.*;
import com.clavain.utils.SocketCheck;
import com.clavain.workers.MongoWorker;
import java.util.Collections;
import java.util.concurrent.CopyOnWriteArrayList;
import org.quartz.Scheduler;
import org.quartz.SchedulerFactory;
import org.quartz.impl.StdSchedulerFactory;
import static com.clavain.utils.Generic.getUnixtime;
import com.clavain.workers.NewNodeWatcher;
import static com.clavain.utils.Database.dbScheduleAllCustomJobs;
import com.clavain.workers.DataRetentionWorker;
import java.io.File;
import java.io.ObjectInputStream;
import com.clavain.checks.ReturnServiceCheck;
import com.clavain.executors.MongoCheckExecutor;
import com.clavain.workers.ErrorNotifyExecutor;
import com.google.gson.Gson;
import java.util.List;
            
        } catch (Exception ex)
        {
            System.err.println("Failed to Init basic logging infastructure: " + ex.getLocalizedMessage());
            System.exit(1);
        }
       
       
        try
        {
            // connect to db
            logger.info("Connecting to TokuMX");
            MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
            builder.connectionsPerHost(400);
            builder.autoConnectRetry(true);
            builder.threadsAllowedToBlockForConnectionMultiplier(10);
           
            // speed up inserts, we dont care if we miss some if the shit hits the fan
            builder.writeConcern(WriteConcern.NONE);
            m = new MongoClient( new ServerAddress(p.getProperty("mongo.host")),  builder.build());
 
                        
            // connect to mysql
            connectToDatabase(p);
            
            // PreFilling Nodes, max 100 in concurrent
            logger.info("Loading initial MuninNode details. This can take a few minutes...");
            v_munin_nodes = new CopyOnWriteArrayList<>();
            v_cinterval_plugins = new CopyOnWriteArrayList<>();     
            v_sockets = new CopyOnWriteArrayList<>();
            
            java.sql.Statement stmt = conn.createStatement();
            ResultSet rs = stmt.executeQuery("SELECT * FROM nodes");
            while(rs.next())
            {
                MuninNode mn = new MuninNode();
                mn.setHostname(rs.getString("hostname"));
                mn.setNodename(rs.getString("hostname"));
                mn.setNode_id(rs.getInt("id"));
                mn.setPort(rs.getInt("port"));
                mn.setUser_id(rs.getInt("user_id"));
                mn.setQueryInterval(rs.getInt("query_interval"));
                mn.setStr_via(rs.getString("via_host"));
                mn.setAuthpw(rs.getString("authpw"));
                mn.setGroup(rs.getString("groupname"));
                v_munin_nodes.add(mn);              
                logger.info("* " + mn.getHostname() + " queued for pluginfetch");
            }
            
            // launching quartz scheduler
            logger.info("Launching Scheduler");
            SchedulerFactory sf = new StdSchedulerFactory("quartz.properties");
            sched = sf.getScheduler();   
            sched.start();   
 
            // launching quartz scheduler for custom interval
            logger.info("Launching Custom Interval Scheduler");
            SchedulerFactory sfc = new StdSchedulerFactory("customquartz.properties");
            sched_custom = sfc.getScheduler();   
            sched_custom.start();
            
            // starting API server
            new Thread(new JettyLauncher()).start();
            
            int sleepTime = Integer.parseInt(p.getProperty("startup.sleeptime"));
            int startupIterations = Integer.parseInt(p.getProperty("startup.iterations"));
            // scheduling jobs
            int i = 0;
            for (MuninNode it_mn : v_munin_nodes) {
                if(i == startupIterations)
                {
                    Thread.sleep(sleepTime);
                    i = 0;
                    logger.info("Waiting "+sleepTime+"ms for new scheduling slot");
                }
                scheduleJob(it_mn);
                i++;
            }
            
            // schedule custom interval jobs
            dbScheduleAllCustomJobs();
            
            // add all alerts
            dbAddAllAlerts();
            
            // Service Checks
            logger.info("Launching Service Check Scheduler");
            SchedulerFactory sfsc = new StdSchedulerFactory("checksquartz.properties");
            sched_checks = sfsc.getScheduler();   
            sched_checks.start();             
            // load service checks from database
            stmt = conn.createStatement();
            rs = stmt.executeQuery("SELECT * FROM service_checks");
            while(rs.next())
            {
                Gson gson = new Gson();
                ServiceCheck tc = gson.fromJson(rs.getString("json"), ServiceCheck.class);
                tc.setCid(rs.getInt("id"));
                tc.setUser_id(rs.getInt("user_id"));
                v_serviceChecks.add(tc);
                logger.info("* " + tc.getCheckname() + " Service Check added");
            }            
            // queue service checks
            for (ServiceCheck it_sc : v_serviceChecks) {
                scheduleServiceCheck(it_sc);
            }            
            
            // starting MongoExecutor
            new Thread(new MongoExecutor()).start();
        
            // starting MongoExecutor for Package Tracking and Essential Informations
            new Thread(new MongoEssentialExecutor()).start();            

            // starting MongoExecutor for Service Checks
            new Thread(new MongoCheckExecutor()).start();            

            // starting newnodewatcher
            new Thread(new NewNodeWatcher()).start();
            
            // start pushover sending message
            new Thread(new PushOverLimiter()).start();