A project for online policy adaptation using rollout for meeting end-to-end QoS objectives in networked systems.
import fileinput
import sys

'''
:param: the input paramters to this function is the weights of the services and the address to the yaml file.

This function replace the values of the weights in configuration file with the given values. 

'''

path = "/home/shahab/compute_info/networking/user_based_rules/"

def replace_two_weight(a1,a2,name):
    if (a1>=0) and (a1<=100) and (a2>=0) and (a2<=100):
        counter = 0
        for line in fileinput.input(name, inplace=True):
            if ("weight:" in line):
                if (counter == 0):
                    line = line.replace(line, "      weight: " + str(a1)) + "\n"
                    counter += 1
                elif (counter == 1):
                    line = line.replace(line, "      weight: " + str(100 - a1)) + "\n"
                    counter += 1
                elif (counter == 2):
                    line = line.replace(line, "      weight: " + str(a2)) + "\n"
                    counter += 1
                elif (counter == 3):
                    line = line.replace(line, "      weight: " + str(100 - a2)) + "\n"
                    counter += 1
            sys.stdout.write(line)

def replace_one_weight(a1,name):
    if (a1>=0) and (a1<=100):
        counter = 0
        for line in fileinput.input(name, inplace=True):
            if ("weight:" in line):
                if (counter == 0):
                    line = line.replace(line, "      weight: " + str(a1)) + "\n"
                    counter += 1
                elif (counter == 1):
                    line = line.replace(line, "      weight: " + str(100 - a1)) + "\n"
                    counter += 1
            sys.stdout.write(line)

if (len(sys.argv) > 2):
    replace_two_weight(int(sys.argv[1]),int(sys.argv[2]),path + "user_based_weight.yaml")
else:
    replace_two_weight(int(sys.argv[1]), path + "user_based_weight.yaml")
application/networking/user_based_rules/command_to_file.py
foroughsh/online_policy_adaptation_using_rollout
import fileinput
import sys

'''
We use this file to replace the allocated CPU in the yaml file that defines the computeinfo pods.
'''

path = "/home/shahab/compute_info/scling_rules/"

def replace(a1, a2, a3, name):
    counter = 0
    for line in fileinput.input(name, inplace=True):
        if ("replicas:" in line):
            line = line.replace(line, '  replicas: ' + str(a1)) + "\n"
        if ("cpu:" in line):
            if (counter == 0):
                line = line.replace(line, '            cpu: "' + str(a2)) + 'm"' + "\n"
                counter += 1
            elif (counter == 1):
                line = line.replace(line, '            cpu: "' + str(a3)) + 'm"' + "\n"
                counter += 1
        sys.stdout.write(line)

print(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
replace(int(float(sys.argv[1])), int(float(sys.argv[2])),int(float(sys.argv[3])), path + sys.argv[4])
application/scaling/command_to_file.py
foroughsh/online_policy_adaptation_using_rollout
import numpy as np
from sb3_contrib import MaskablePPO
from online_policy_adaptation_using_rollout.system_model.system_model_scenario_1 import (
    state_space, action_space, services, edges, nodes)
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_1_env.routing_env import RoutingEnv
from online_policy_adaptation_using_rollout.evaluation.evaluate_scenario_1 import (
    compute_value_function_of_policy, compute_delay_table, random_base_policy, greedy_base_policy,
    optimal_base_policy, run_vi, run_pi, run_exact_one_step_online_play, run_monte_carlo_one_step_online_play,
    run_base_policy_online, newton_step_data)

if __name__ == '__main__':
    S = services()
    N = nodes()
    E = edges()

    # DP parameters
    # delta_c = 1
    # c_max = 2
    # delta_p = 0.5
    # p_max = 1
    # theta = 0.05
    # gamma = 0.9

    # MC parameters
    delta_c = 1
    c_max = 5
    delta_p = 0.2
    p_max = 1
    gamma = 0.99
    theta = 0.01

    # Setup spaces
    print("Creating state space")
    states = state_space(services=S, nodes=N, edges=E, delta_c=delta_c, delta_p=delta_p, p_max=p_max, c_max=c_max)
    print("Creating action space")
    actions, component_spaces = action_space(services=S, nodes=N, edges=E, delta_c=delta_c, delta_p=delta_p)

    # Compute delay table
    compute_delay_table(states=states, S=S, filename="delay_table.npy")
    print("Loading delay table")
    with open('delay_table.npy', 'rb') as f:
        delay_table = list(np.load(f, allow_pickle=True).tolist())

    # Compute random policy
    random_policy = random_base_policy(states=states, actions=actions)

    # Compute greedy policy
    print("computing greedy policy")
    greedy_base_policy(states=states, actions=actions, S=S, N=N, E=E, c_max=c_max, p_max=p_max, delay_table=delay_table,
                       filename="pi_greedy.npy")
    print("loading greedy policy")
    with open('pi_greedy.npy', 'rb') as f:
        pi_greedy = list(np.load(f, allow_pickle=True).tolist())
    print("greedy policy loaded")

    # Compute optimal policy
    print("computing optimal policy")
    optimal_base_policy(states=states, actions=actions, filename="pi_opt.npy")
    print("loading optimal policy")
    with open('pi_opt.npy', 'rb') as f:
        pi_opt = list(np.load(f, allow_pickle=True).tolist())
    print("optimal policy loaded")

    print("loading PPO policy")
    env = RoutingEnv()
    try:
        pi_ppo = MaskablePPO.load("/Users/kimham/workspace/NOMS2024_rollouts/sc1/training/self_routing_14.zip")
    except:
        try:
            pi_ppo = MaskablePPO.load("/Users/kim/workspace/NOMS2024_rollouts/sc1/training/self_routing_14.zip")
        except:
            pi_ppo = MaskablePPO.load("/home/kim/workspace/NOMS2024_rollouts/sc1/training/self_routing_14.zip")
    print("PPO policy loaded")

    initial_state = [2.0, 2.0, 1.0, 1.0]
    initial_state_index = states.index(initial_state)
    print(f"initial state: {initial_state_index}")

    # VI
    run_vi(states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max, c_max=c_max, theta=theta,
           delay_table=delay_table)

    # PI
    run_pi(states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max, c_max=c_max, eval_every=1,
           eval_batch_size=10, running_average_size=1, eval_initial_state=len(states) - 1, eval_horizon=10,
           num_iterations=100, theta=theta, delay_table=delay_table)

    # Exact online play
    base_policy = random_policy
    run_exact_one_step_online_play(base_pi=base_policy, states=states, actions=actions, gamma=gamma, theta=theta,
                                   S=S, N=N,
                                   E=E, p_max=p_max, c_max=c_max, initial_state=41, horizon=10)

    # Compute base policy value function

    print("Computing value function of random base policy")
    compute_value_function_of_policy(base_pi=random_policy, filename="random_base_policy_V.npy", batch_size=1,
                                     states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                                     c_max=c_max, delay_table=delay_table, delta_c=delta_c, delta_p=delta_p)

    print("Computing value function of optimal base policy")
    compute_value_function_of_policy(base_pi=pi_opt, filename="opt_base_policy_V.npy", batch_size=1,
                                     states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                                     c_max=c_max, delay_table=delay_table, delta_c=delta_c, delta_p=delta_p)

    print("Computing value function of greedy base policy")
    compute_value_function_of_policy(base_pi=pi_greedy, filename="greedy_base_policy_V.npy", batch_size=1,
                                     states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                                     c_max=c_max, delay_table=delay_table, delta_c=delta_c, delta_p=delta_p)
    print("Computing value function of PPO base policy")
    compute_value_function_of_policy(base_pi=pi_ppo, filename="ppo_base_policy_V.npy", batch_size=1,
                                     states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                                     c_max=c_max, delay_table=delay_table, delta_p=delta_p,
                                     delta_c=delta_c, env=env)

    # Monte Carlo online play

    seeds = [151, 125, 1251, 96, 215, 21, 5678, 2, 3256, 15]

    base_policy = random_policy
    with open('random_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds, time_means, time_stds = run_monte_carlo_one_step_online_play(base_pi=base_policy, states=states,
                                                                              actions=actions, gamma=gamma,
                                                                              S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                                                              initial_state=initial_state_index,
                                                                              horizon=20, batch_size=5,
                                                                              V_base=list(V_base.tolist()), l=1,
                                                                              deterministic_model=True,
                                                                              delay_table=delay_table,
                                                                              num_episodes=5,
                                                                              seeds=seeds, env=env, multiagent=False,
                                                                              component_spaces=None)
    with open('random_rollout_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('random_rollout_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    base_policy = pi_greedy
    with open('greedy_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds, time_means, time_stds = run_monte_carlo_one_step_online_play(base_pi=base_policy, states=states,
                                                                              actions=actions, gamma=gamma,
                                                                              S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                                                              initial_state=initial_state_index,
                                                                              horizon=20, batch_size=5,
                                                                              V_base=list(V_base.tolist()), l=1,
                                                                              deterministic_model=True,
                                                                              delay_table=delay_table,
                                                                              num_episodes=5, seeds=seeds, env=env,
                                                                              multiagent=False,
                                                                              component_spaces=None)
    with open('greedy_rollout_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('greedy_rollout_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    with open('ppo_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    base_policy = pi_ppo
    means, stds, time_means, time_stds = run_monte_carlo_one_step_online_play(base_pi=base_policy, states=states,
                                                                              actions=actions, gamma=gamma,
                                                                              S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                                                              initial_state=initial_state_index,
                                                                              horizon=20, batch_size=5,
                                                                              V_base=list(V_base.tolist()), l=1,
                                                                              deterministic_model=True,
                                                                              delay_table=delay_table,
                                                                              delta_c=delta_c, delta_p=delta_p,
                                                                              num_episodes=5, seeds=seeds,
                                                                              env=env, multiagent=False,
                                                                              component_spaces=None)
    with open('ppo_rollout_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('ppo_rollout_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    # Evaluate base policies
    base_policy = random_policy
    with open('random_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds = run_base_policy_online(base_pi=base_policy, states=states, actions=actions,
                                         S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                         initial_state=initial_state_index,
                                         horizon=20, V_base=list(V_base.tolist()),
                                         delay_table=delay_table,
                                         num_episodes=5, seeds=seeds, delta_c=delta_c,
                                         delta_p=delta_p, env=env)
    with open('random_base_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('random_base_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    base_policy = pi_greedy
    with open('greedy_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds = run_base_policy_online(base_pi=base_policy, states=states, actions=actions,
                                         S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                         initial_state=initial_state_index,
                                         horizon=20, V_base=list(V_base.tolist()),
                                         delay_table=delay_table,
                                         num_episodes=5, seeds=seeds, delta_c=delta_c,
                                         delta_p=delta_p,
                                         env=env)
    with open('greedy_base_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('greedy_base_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    base_policy = pi_ppo
    with open('ppo_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds = run_base_policy_online(base_pi=base_policy, states=states, actions=actions,
                                         S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                         initial_state=initial_state_index,
                                         horizon=20, V_base=list(V_base.tolist()),
                                         delay_table=delay_table,
                                         num_episodes=5, seeds=seeds, delta_c=delta_c,
                                         delta_p=delta_p, env=env)
    # print(means)
    with open('ppo_base_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('ppo_base_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    base_policy = pi_opt
    with open('opt_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    means, stds = run_base_policy_online(base_pi=base_policy, states=states, actions=actions,
                                         S=S, N=N, E=E, p_max=p_max, c_max=c_max,
                                         initial_state=initial_state_index,
                                         horizon=20, V_base=list(V_base.tolist()),
                                         delay_table=delay_table,
                                         num_episodes=5, seeds=seeds, delta_c=delta_c,
                                         delta_p=delta_p, env=env)
    with open('opt_base_reward_means.npy', 'wb') as f:
        np.save(f, np.array(means))
    with open('opt_base_reward_stds.npy', 'wb') as f:
        np.save(f, np.array(stds))

    # Newton step
    print("Start Newton")

    base_policy = random_policy
    alt_policy = pi_greedy
    with open('random_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    with open('greedy_base_policy_V.npy', 'rb') as f:
        V_alt = np.load(f, allow_pickle=True)
    V_rollout_1, V_rollout_2, V_base, V_base_2, T_rollout_slope, T_rollout_intercept, T_base_slope, T_base_intercept = \
        newton_step_data(base_pi=base_policy, states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                         c_max=c_max, initial_state=initial_state_index, batch_size=5,
                         V_base=list(V_base.tolist()), l=1, deterministic_model=True, delay_table=delay_table,
                         alt_pi=alt_policy, alt_V=V_alt, env=env)

    print(f"V_rollout_1: {V_rollout_1}, V_rollout_2: {V_rollout_2}, V_base_1: {V_base}, V_base_2: {V_base_2}, "
          f"T_rollout_slope: {T_rollout_slope}, T_rollout_intercept: {T_rollout_intercept}, "
          f"T_base_slope: {T_base_slope}, T_base_intercept: {T_base_intercept}")

    base_policy = pi_greedy
    alt_policy = random_policy
    with open('greedy_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    with open('random_base_policy_V.npy', 'rb') as f:
        V_alt = np.load(f, allow_pickle=True)
    V_rollout_1, V_rollout_2, V_base, V_base_2, T_rollout_slope, T_rollout_intercept, T_base_slope, T_base_intercept = \
        newton_step_data(base_pi=base_policy, states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                         c_max=c_max, initial_state=initial_state_index, batch_size=5,
                         V_base=list(V_base.tolist()), l=1, deterministic_model=True, delay_table=delay_table,
                         alt_pi=alt_policy, alt_V=V_alt, env=env)

    print(f"V_rollout_1: {V_rollout_1}, V_rollout_2: {V_rollout_2}, V_base_1: {V_base}, V_base_2: {V_base_2}, "
          f"T_rollout_slope: {T_rollout_slope}, T_rollout_intercept: {T_rollout_intercept}, "
          f"T_base_slope: {T_base_slope}, T_base_intercept: {T_base_intercept}")

    base_policy = pi_ppo
    alt_policy = pi_greedy
    with open('ppo_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    with open('greedy_base_policy_V.npy', 'rb') as f:
        V_alt = np.load(f, allow_pickle=True)
    V_rollout_1, V_rollout_2, V_base, V_base_2, T_rollout_slope, T_rollout_intercept, T_base_slope, T_base_intercept = \
        newton_step_data(base_pi=base_policy, states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max,
                         c_max=c_max, initial_state=initial_state_index, batch_size=5,
                         V_base=list(V_base.tolist()), l=1, deterministic_model=True, delay_table=delay_table,
                         alt_pi=alt_policy, alt_V=V_alt, env=env)

    print(f"V_rollout_1: {V_rollout_1}, V_rollout_2: {V_rollout_2}, V_base_1: {V_base}, V_base_2: {V_base_2}, "
          f"T_rollout_slope: {T_rollout_slope}, T_rollout_intercept: {T_rollout_intercept}, "
          f"T_base_slope: {T_base_slope}, T_base_intercept: {T_base_intercept}")

    # Ablation study for lookahead horizon l

    seeds = [151, 125, 1251, 96, 215, 21, 5678, 2, 3256, 15]
    multiagent = True
    base_policy = random_policy
    with open('random_base_policy_V.npy', 'rb') as f:
        V_base = np.load(f, allow_pickle=True)
    for l in range(1, 5):
        means, stds, time_means, time_stds = run_monte_carlo_one_step_online_play(
            base_pi=base_policy, states=states, actions=actions, gamma=gamma, S=S, N=N, E=E, p_max=p_max, c_max=c_max,
            initial_state=initial_state_index, horizon=20, batch_size=5, V_base=list(V_base.tolist()), l=l,
            deterministic_model=True, delay_table=delay_table, num_episodes=1, seeds=seeds, env=env,
            multiagent=multiagent, component_spaces=component_spaces)
        with open(f'random_rollout_reward_means_{l}.npy', 'wb') as f:
            np.save(f, np.array(means))
        with open(f'random_rollout_reward_stds_{l}.npy', 'wb') as f:
            np.save(f, np.array(stds))
        with open(f'random_rollout_time_means_{l}.npy', 'wb') as f:
            np.save(f, np.array(time_means))
        with open(f'random_rollout_time_stds_{l}.npy', 'wb') as f:
            np.save(f, np.array(time_stds))

examples/run_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout
import sys
import argparse
from online_policy_adaptation_using_rollout.ppo_base_policy.training_ppo_agent import PPOBase
from online_policy_adaptation_using_rollout.evaluation.evaluation import EvaluateBasePolicy

if __name__ == '__main__':

    args = sys.argv

    parser = argparse.ArgumentParser(description='Please check the code for options!')
    parser.add_argument("--seed", type=int, default=83)
    parser.add_argument("--target_path_for_models", type=str, default="../trained_models/scenario_2/")
    parser.add_argument("--target_path_to_save_results", type=str, default="../artifacts/scenario_2/")
    parser.add_argument("--num_neurons_per_hidden_layer", type=int, default=128)
    parser.add_argument("--num_layers", type=int, default=3)
    parser.add_argument("--steps_between_updates", type=int, default=512)
    parser.add_argument("--learning_rate", type=float, default=0.0005)
    parser.add_argument("--batch_size", type=int, default=64)
    parser.add_argument("--device", type=str, default="cpu")
    parser.add_argument("--gamma", type=float, default=0.99)
    parser.add_argument("--num_training_timesteps", type=int, default=1000000)
    parser.add_argument("--verbose", type=int, default=0)
    parser.add_argument("--ent_coef", type=float, default=0.05)
    parser.add_argument("--clip_range", type=float, default=0.2)
    parser.add_argument("--path_to_system_model", type=str, default="../trained_models/scenario_2")
    parser.add_argument("--environment_name", type=str, default="routing-env-v2")
    parser.add_argument("--last_iteration", type=int, default=4)

    args = parser.parse_args()

    path_to_save_results = args.target_path_to_save_results
    path_to_save_models = args.target_path_for_models
    path_to_system_model = args.path_to_system_model

    seed = args.seed
    num_neurons_per_hidden_layer = args.num_neurons_per_hidden_layer
    num_layers = args.num_layers
    steps_between_updates = args.steps_between_updates
    learning_rate = args.learning_rate
    batch_size = args.batch_size
    device = args.device
    gamma = args.gamma
    num_training_timesteps = args.num_training_timesteps
    verbose = args.verbose
    ent_coef = args.ent_coef
    clip_range = args.clip_range
    environment_name = args.environment_name
    last_iteration = args.last_iteration

    print(
        "You can update the parameters by invoking function with the following options: \n"
        "\t--seed 83\n"
        "\t--target_path_for_models ../../trained_models/scenario_2/\n"
        "\t--target_path_to_save_results ../../artifacts/scenario_2/\n"
        "\t--num_neurons_per_hidden_layer 128\n"
        "\t--num_layers 3\n"
        "\t--steps_between_updates 512\n"
        "\t--learning_rate 0.0005\n"
        "\t--batch_size 64\n"
        "\t--device cpu\n"
        "\t--gamma 0.99\n"
        "\t--num_training_timesteps 1000000"
        "\t--verbose 0\n"
        "\t--ent_coef 0.05\n"
        "\t--clip_range 0.2"
        "\t--path_to_system_model ../../trained_models/scenario_2"
        "\t--environment_name routing-env-v2"
        "\t--last_iteration 4"
    )

    ####### Training the base PPO policy
    # training_obj = PPOBase(environment_name, path_to_system_model + "system_model.joblib", num_neurons_per_hidden_layer,
    # num_layers, seed, path_to_save_models,
    # path_to_save_results, verbose, steps_between_updates, batch_size, learning_rate, gamma, ent_coef,
    # clip_range, num_training_timesteps, device)
    #
    # training_obj.train_ppo_base()

    ####### Offline training the base PPO policy
    # training_obj = PPOBase(environment_name, path_to_system_model + "new_system_model.joblib", num_neurons_per_hidden_layer,
    # num_layers, seed, path_to_save_models,
    # path_to_save_results, verbose, steps_between_updates, batch_size, learning_rate, gamma, ent_coef,
    # clip_range, num_training_timesteps, device,policy_name="self_routing_new_")
    #
    # training_obj.train_ppo_base()

    ####### Evaluation of the base policy in the environment before change
    # evaluation_obj = EvaluateBasePolicy(path_to_base_model=path_to_save_models, model_name="self_routing_"+str(seed)+"_4.zip",
    #                                     target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
    #                                     env_name=environment_name, path_to_system_model=path_to_system_model + "system_model.joblib")
    #
    # rewards, infos = evaluation_obj.evaluate_with_base_or_offline()
    #
    # print(rewards, infos)

    ####### Evaluation of the base policy in the new environment before change
    # evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                     model_name="self_routing_" + str(seed) + "_" + str(last_iteration) + ".zip",
    #                                     target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
    #                                     env_name=environment_name,
    #                                     path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new, infos_new = evaluation_obj_new.evaluate_with_base_or_offline()
    #
    # print(rewards_new, infos_new)

    ###### Evaluation of the base policy in the new environment before change
    evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
                                        model_name="self_routing_" + str(seed) + "_" + str(last_iteration) + ".zip",
                                        target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
                                        env_name=environment_name,
                                        path_to_system_model=path_to_system_model + "new_system_model.joblib")

    rewards_new, infos_new = evaluation_obj_new.evaluate_with_rollout()

    print(rewards_new, infos_new)

    ####### Evaluatiton of the offline retrained in the new environment
    # evaluation_obj_new_offline = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                         model_name="self_routing_new_" + str(seed) + "_" + str(last_iteration) + ".zip",
    #                                         target_path_to_save_evalaution_results=path_to_save_results,
    #                                         episode_lenth=8,
    #                                         env_name=environment_name,
    #                                         path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new_offline, infos_new_offline = evaluation_obj_new_offline.evaluate_with_base_or_offline()
    #
    # print(rewards_new_offline, infos_new_offline)

    ####### Evaluatiton of the offline retrained in the new environment
    # evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                         model_name="self_routing_new_" + str(seed) + "_" + str(last_iteration) + ".zip",
    #                                         target_path_to_save_evalaution_results=path_to_save_results,
    #                                         episode_lenth=8,
    #                                         env_name=environment_name,
    #                                         path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new, infos_new = evaluation_obj_new.evaluate_with_base_or_offline()
    #
    # print(rewards_new, infos_new)
examples/run_scenario_2.py
foroughsh/online_policy_adaptation_using_rollout
import sys
import argparse
from online_policy_adaptation_using_rollout.ppo_base_policy.training_ppo_agent import PPOBase
from online_policy_adaptation_using_rollout.evaluation.evaluation import EvaluateBasePolicy

if __name__ == '__main__':

    args = sys.argv

    parser = argparse.ArgumentParser(description='Please check the code for options!')
    parser.add_argument("--seed", type=int, default=83)
    parser.add_argument("--target_path_for_models", type=str, default="../trained_models/scenario_3/")
    parser.add_argument("--target_path_to_save_results", type=str, default="../artifacts/scenario_3/")
    parser.add_argument("--num_neurons_per_hidden_layer", type=int, default=128)
    parser.add_argument("--num_layers", type=int, default=3)
    parser.add_argument("--steps_between_updates", type=int, default=512)
    parser.add_argument("--learning_rate", type=float, default=0.0005)
    parser.add_argument("--batch_size", type=int, default=64)
    parser.add_argument("--device", type=str, default="cpu")
    parser.add_argument("--gamma", type=float, default=0.99)
    parser.add_argument("--num_training_timesteps", type=int, default=1000000)
    parser.add_argument("--verbose", type=int, default=0)
    parser.add_argument("--ent_coef", type=float, default=0.05)
    parser.add_argument("--clip_range", type=float, default=0.2)
    parser.add_argument("--path_to_system_model", type=str, default="../trained_models/scenario_3")
    parser.add_argument("--environment_name", type=str, default="routing-env-v3")
    parser.add_argument("--last_iterarion", type=int, default=4)

    args = parser.parse_args()

    path_to_save_results = args.target_path_to_save_results
    path_to_save_models = args.target_path_for_models
    path_to_system_model = args.path_to_system_model

    seed = args.seed
    num_neurons_per_hidden_layer = args.num_neurons_per_hidden_layer
    num_layers = args.num_layers
    steps_between_updates = args.steps_between_updates
    learning_rate = args.learning_rate
    batch_size = args.batch_size
    device = args.device
    gamma = args.gamma
    num_training_timesteps = args.num_training_timesteps
    verbose = args.verbose
    ent_coef = args.ent_coef
    clip_range = args.clip_range
    environment_name = args.environment_name
    last_iteration = args.last_iteration

    print(
        "You can update the parameters by invoking function with the following options: \n"
        "\t--seed 83\n"
        "\t--target_path_for_models ../../trained_models/scenario_3/\n"
        "\t--target_path_to_save_results ../../artifacts/scenario_3/\n"
        "\t--num_neurons_per_hidden_layer 128\n"
        "\t--num_layers 3\n"
        "\t--steps_between_updates 512\n"
        "\t--learning_rate 0.0005\n"
        "\t--batch_size 64\n"
        "\t--device cpu\n"
        "\t--gamma 0.99\n"
        "\t--num_training_timesteps 1000000"
        "\t--verbose 0\n"
        "\t--ent_coef 0.05\n"
        "\t--clip_range 0.2"
        "\t--path_to_system_model ../../trained_models/scenario_3"
        "\t--environment_name routing-env-v3"
        "--last_iterarion 4"
    )

    ####### Training the base PPO policy
    # training_obj = PPOBase(environment_name, path_to_system_model + "system_model.joblib", num_neurons_per_hidden_layer,
    # num_layers, seed, path_to_save_models,
    # path_to_save_results, verbose, steps_between_updates, batch_size, learning_rate, gamma, ent_coef,
    # clip_range, num_training_timesteps, device)
    #
    # training_obj.train_ppo_base()

    ####### Offline training the base PPO policy
    # training_obj = PPOBase(environment_name, path_to_system_model + "new_system_model.joblib", num_neurons_per_hidden_layer,
    # num_layers, seed, path_to_save_models,
    # path_to_save_results, verbose, steps_between_updates, batch_size, learning_rate, gamma, ent_coef,
    # clip_range, num_training_timesteps, device,policy_name="self_routing_new_")
    #
    # training_obj.train_ppo_base()

    ####### Evaluation of the base policy in the environment before change
    # evaluation_obj = EvaluateBasePolicy(path_to_base_model=path_to_save_models, model_name="self_routing_"+str(seed)
    #                                       + "_" + str(last_iteration) + ".zip",
    #                                     target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
    #                                     env_name=environment_name, path_to_system_model=path_to_system_model + "system_model.joblib")
    #
    # rewards, infos = evaluation_obj.evaluate_with_base_or_offline()
    #
    # print(rewards, infos)

    ####### Evaluation of the base policy in the new environment before change
    # evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                     model_name="self_routing_" + str(seed) + "_" + str(last_iteration)+".zip",
    #                                     target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
    #                                     env_name=environment_name,
    #                                     path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new, infos_new = evaluation_obj_new.evaluate_with_base_or_offline()
    #
    # print(rewards_new, infos_new)

    ###### Evaluation of the base policy in the new environment before change
    evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
                                        model_name="self_routing_" + str(seed) + "_" + str(last_iteration) + ".zip",
                                        target_path_to_save_evalaution_results=path_to_save_results, episode_lenth=8,
                                        env_name=environment_name,
                                        path_to_system_model=path_to_system_model + "new_system_model.joblib")

    rewards_new, infos_new = evaluation_obj_new.evaluate_with_rollout()

    print(rewards_new, infos_new)

    ####### Evaluatiton of the offline retrained in the new environment
    # evaluation_obj_new_offline = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                         model_name="self_routing_new_" + str(seed)  + "_" + str(last_iteration) + ".zip",
    #                                         target_path_to_save_evalaution_results=path_to_save_results,
    #                                         episode_lenth=8,
    #                                         env_name=environment_name,
    #                                         path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new_offline, infos_new_offline = evaluation_obj_new_offline.evaluate_with_base_or_offline()
    #
    # print(rewards_new_offline, infos_new_offline)

    ####### Evaluatiton of the offline retrained in the new environment
    # evaluation_obj_new = EvaluateBasePolicy(path_to_base_model=path_to_save_models,
    #                                         model_name="self_routing_new_" + str(seed)  + "_" + str(last_iteration) + ".zip",
    #                                         target_path_to_save_evalaution_results=path_to_save_results,
    #                                         episode_lenth=8,
    #                                         env_name=environment_name,
    #                                         path_to_system_model=path_to_system_model + "new_system_model.joblib")
    #
    # rewards_new, infos_new = evaluation_obj_new.evaluate_with_base_or_offline()
    #
    # print(rewards_new, infos_new)
examples/run_scenario_3.py
foroughsh/online_policy_adaptation_using_rollout
"""
Implements the Policy Iteration algorithm
"""
from typing import List, Tuple
import numpy as np


def policy_evaluation(policy: List, num_states: int, gamma: float, theta: float, T, R) -> List:
    """
    The policy evaluation function

    :param policy: the policy to evaluate
    :param num_states: the number of states
    :param gamma: the discount factor
    :param theta: the theta value for measuring convergence
    :param T: the transition tensor
    :param R: the reward tensor
    :return: the value function of the policy
    """
    V = np.zeros(num_states)
    while True:
        delta = 0
        for s in range(num_states):
            v = 0
            for s_prime in range(num_states):
                for a, action_prob in enumerate(policy[s]):
                    transition_prob = T[s][s_prime][a]
                    r = R[s][a]
                    v += action_prob * transition_prob * (r + gamma * V[s_prime])
            delta = max(delta, np.abs(v - V[s]))
            V[s] = v
        if delta < theta:
            break
    return list(V.tolist())


def one_step_lookahead(s: int, V: List, gamma: float, num_actions: int, num_states: int, T: List, R: List) -> List:
    """
    Performs one-step lookahead maximization

    :param s: the current state
    :param V: the value function
    :param gamma: the discount factor
    :param num_actions: the number of actions
    :param num_states: the number of states
    :param T: the transition tensor
    :param R: the reward tensor
    :return: the action values
    """
    A = np.zeros(num_actions)
    for a in range(num_actions):
        for s_prime in range(num_states):
            transition_prob = T[s][s_prime][a]
            r = R[s][a]
            A[a] += transition_prob * (r + gamma * V[s_prime])
    return A


def policy_improvement(V_pi: List, gamma: float, num_states: int, policy: List, num_actions: int, T: List, R: List) \
        -> Tuple[List, bool, int]:
    """
    The policy improvement functin

    :param V_pi: the value function of the base policy
    :param gamma: the discount factor
    :param num_states: the number of states
    :param policy: the base policy
    :param num_actions: the number of actions
    :param T: the transition tensor
    :param R: the reward tensor
    :return: the policy, a boolean flag indiciating whether the policy has convergend, and the number of changed states
    """
    num_changed_states = 0
    policy_stable = True
    for s in range(num_states):
        chosen_a = np.argmax(policy[s])
        action_values = one_step_lookahead(s, V_pi, gamma=gamma, num_actions=num_actions, num_states=num_states,
                                           T=T, R=R)
        best_a = np.argmax(action_values)
        if chosen_a != best_a:
            policy_stable = False
            num_changed_states += 1
        policy[s] = np.eye(num_actions)[best_a]
    return policy, policy_stable, num_changed_states


def policy_iteration(num_states: int, num_actions: int, T: List, R: List, N: int, gamma: float,
                     eval_every: int, eval_batch_size: int, running_average_size,
                     eval_initial_state: int, eval_horizon: int, theta: float) -> Tuple[List, List, List, List]:
    """
    Implements the policy iteration algorithm

    :param num_states: the number of states
    :param num_actions: the number of actions
    :param T: the transition tensor
    :param R: the reward tensor
    :param N: the number of nodes
    :param gamma: the discount factor
    :param eval_every: the evaluation frequency
    :param eval_batch_size: the batch size for evaluation
    :param running_average_size: the number of measurements to compute a running average
    :param eval_initial_state: the initial state for evaluation
    :param eval_horizon: the horizon for evaluation
    :param theta: the theta perameter to measure convergence
    :return: the optimal policy, its value function, the average returns of the policy, and the running averages.
    """
    print("Staring policy iteration")
    policy = np.ones([num_states, num_actions]) / num_actions
    average_returns = []
    running_average_returns = []
    for i in range(0, N):
        # Evaluate the current policy
        V_pi = policy_evaluation(policy=policy, gamma=gamma, num_states=num_states, theta=theta, T=T, R=R)

        # Will be set to false if we make any changes to the policy
        policy, policy_stable, num_changed_states = policy_improvement(V_pi=V_pi, gamma=gamma, num_states=num_states,
                                                                       num_actions=num_actions, policy=policy, T=T, R=R)

        if i % eval_every == 0:
            avg_return = evaluate_policy(T=T, initial_state=eval_initial_state, policy=policy,
                                         eval_batch_size=eval_batch_size, horizon=eval_horizon, R=R)
            average_returns.append(avg_return)
            running_avg_J = running_average(average_returns, running_average_size)
            running_average_returns.append(running_avg_J)
            print(f"PI, iteration: {i}, stable policy: {policy_stable}, changes: {num_changed_states}, "
                  f"average return: {avg_return}")
        if policy_stable:
            print("PI converged")
            return policy, V_pi, average_returns, running_average_returns


def running_average(x: List, N: int) -> List:
    """
    Utility function to compute a running average of a list

    :param x: the list
    :param N: the number of elements to include in the running average
    :return: the running averaged list
    """
    if len(x) < N:
        N = len(x)
    if len(x) >= N:
        y = np.copy(x)
        y[N - 1:] = np.convolve(x, np.ones((N,)) / N, mode='valid')
    else:
        y = np.zeros_like(x)
    return round(y[-1], 2)


def evaluate_policy(T: List, initial_state: int, eval_batch_size: int, horizon: int, policy: List,
                    R: List) -> float:
    """
    Utility function for evaluating a given policy

    :param T: the transition tensor
    :param initial_state: the initial state
    :param eval_batch_size: the batch size for evaluation
    :param horizon: the horizon for evaluation
    :param policy: the policy to evaluate
    :param R: the reward tensor
    :return: the average return of teh policy
    """
    T_1 = np.transpose(np.array(T), (2, 0, 1))
    returns = []
    for i in range(eval_batch_size):
        s = initial_state
        ret = 0
        for t in range(horizon):
            a = np.random.choice(np.arange(0, len(policy[s])), p=policy[s])
            s_prime = np.random.choice(np.arange(0, len(T_1[a, s, :])), p=T_1[a, s, :])
            r = R[s][a]
            s = s_prime
            ret += r
        returns.append(ret)
    avg_return = np.mean(returns)
    return float(avg_return)

src/online_policy_adaptation_using_rollout/algorithms/policy_iteration_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout
"""
Script containing rollout algorithms for scenario 1
"""
from typing import List, Union, Tuple
import numpy as np
from sb3_contrib import MaskablePPO
from sb3_contrib.common.maskable.utils import get_action_masks
from policy_iteration_scenario_1 import policy_evaluation
from online_policy_adaptation_using_rollout.system_model.system_model_scenario_1 import (
    transition_function, reward_function, convert_dp_s_to_env_s, convert_env_a_to_dp_a)


def exact_one_step_rollout_policy(base_pi: List, V_base, s: int, gamma: float, num_actions: int, num_states: int,
                                  T: List, R: List, theta: float) -> Tuple[List, List]:
    """
    Computes the exact one-step lookahead rollout policy

    :param base_pi: the base policy
    :param V_base: value function of the base policy
    :param s: the current state
    :param gamma: the discount factor
    :param num_actions: the number of actions
    :param num_states: the number of states
    :param T: the transition tensor
    :param R: the reward tensor
    :param theta: the theta parameter for vale iteration convergence
    :return:
    """
    a = exact_one_step_lookahead_maximization(s=s, V=V_base, gamma=gamma, num_actions=num_actions,
                                              num_states=num_states, T=T, R=R)
    rollout_pi = base_pi.copy()
    rollout_pi[s] = np.zeros(len(base_pi[s]))
    rollout_pi[s][a] = 1
    V_rollout = policy_evaluation(policy=rollout_pi, num_states=num_states, gamma=gamma, theta=theta, T=T, R=R)
    return rollout_pi, V_rollout


def exact_one_step_lookahead_maximization(s: int, V: List, gamma: float, num_actions: int, num_states: int, T: List,
                                          R: List) -> int:
    """
    Gets the best action from one-step lookahead maximization

    :param s: the current state
    :param V: the value function
    :param gamma: the discount factor
    :param num_actions: the number of actions
    :param num_states: the number of states
    :param T: the transition tensor
    :param R: the reward tensor
    :return: the best action
    """
    A = np.zeros(num_actions)
    for a in range(num_actions):
        for s_prime in range(num_states):
            transition_prob = T[s][s_prime][a]
            r = R[s][a]
            A[a] += transition_prob * (r + gamma * V[s_prime])
    best_a = np.argmax(A)
    return int(best_a)


def monte_carlo_l_step_rollout_policy(base_pi: Union[List, MaskablePPO], s: int, gamma: float,
                                      batch_size: int, states: List, actions: List, c_max: float,
                                      p_max: float, N: List, E: List, S: List, V_base: List, l: int,
                                      delay_table: List, deterministic_model: bool = False, env=None) \
        -> Tuple[List, List, int]:
    """
    Computes the monte-Carlo l-step rollout policy

    :param base_pi: the base policy
    :param s: the current state
    :param gamma: the discount factor
    :param batch_size: the batch size for evaluation
    :param states: the state space
    :param actions: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param N: the set of nodes
    :param E: the set of edges
    :param S: the set of services
    :param V_base: the value function of the base polichy
    :param l: the lookahead optimization horizon
    :param delay_table: the table with the predicted delays
    :param deterministic_model: boolean flag indicating whether it is a deterministic model or not
    :param env: the environment for evaluation
    :return: the rollout policy and the value function of the rollout policy and the best action in the current state
    """
    if deterministic_model:
        a, rollout_val = deterministic_l_step_lookahead_maximization(
            s=s, gamma=gamma, states=states, actions=actions, c_max=c_max, p_max=p_max, N=N, E=E, S=S, V_base=V_base,
            l=l, delay_table=delay_table)
    else:
        a, rollout_val = monte_carlo_l_step_lookahead_maximization(s=s, gamma=gamma, batch_size=batch_size,
                                                                   states=states,
                                                                   actions=actions, c_max=c_max, p_max=p_max, N=N, E=E,
                                                                   S=S,
                                                                   V_base=V_base, l=l, delay_table=delay_table)
    if isinstance(base_pi, list) or isinstance(base_pi, np.ndarray):
        rollout_pi = base_pi.copy()
        rollout_pi[s] = np.zeros(len(base_pi[s]))
        rollout_pi[s][a] = 1
    else:
        rollout_pi = base_pi
    V_rollout = V_base.copy()
    V_rollout[s] = rollout_val
    return rollout_pi, V_rollout, a


def monte_carlo_l_step_multiagent_rollout_policy(base_pi: Union[List, MaskablePPO], s: int, gamma: float,
                                                 batch_size: int, states: List, actions: List, c_max: float,
                                                 p_max: float, N: List, E: List, S: List, V_base: List, l: int,
                                                 component_spaces,
                                                 delay_table: List, deterministic_model: bool = False, env=None) \
        -> Tuple[List, List, int]:
    """
    Computes the Monte-Carlo l-step multi-agent rollout policy

    :param base_pi: the base policy
    :param s: the current state
    :param gamma: the discount factor
    :param batch_size: the batch size
    :param states: the state space
    :param actions: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param N: the set of nodes
    :param E: the set of edges
    :param S: the set of services
    :param V_base: the value function of the base policy
    :param l: the lookahead horizon
    :param component_spaces: action spaces of differet action components
    :param delay_table: table with predicted service request delays
    :param deterministic_model: boolean flag indicating whether it is a deterministic model or not
    :param env: the environment for evaluation
    :return: the rollout policy and the value function of the rollout policy and the best action in the current state
    """
    if deterministic_model:
        partial_a = []
        rollout_val = 0
        for agent_index in range(len(component_spaces)):
            a, rollout_val = deterministic_l_step_lookahead_maximization_multiagent(
                s=s, gamma=gamma, states=states, actions=actions, c_max=c_max, p_max=p_max, N=N, E=E, S=S,
                V_base=V_base,
                l=l, delay_table=delay_table, component_spaces=component_spaces, base_pi=base_pi,
                agent_index=agent_index, partial_a=partial_a)
            partial_a.append(a[agent_index])
        a = actions.index(tuple(partial_a))
    else:
        a, rollout_val = monte_carlo_l_step_lookahead_maximization(s=s, gamma=gamma, batch_size=batch_size,
                                                                   states=states,
                                                                   actions=actions, c_max=c_max, p_max=p_max, N=N, E=E,
                                                                   S=S,
                                                                   V_base=V_base, l=l, delay_table=delay_table)
    if isinstance(base_pi, list) or isinstance(base_pi, np.ndarray):
        rollout_pi = base_pi.copy()
        rollout_pi[s] = np.zeros(len(base_pi[s]))
        rollout_pi[s][a] = 1
    else:
        rollout_pi = base_pi
    V_rollout = V_base.copy()
    V_rollout[s] = rollout_val
    return rollout_pi, V_rollout, a


def deterministic_l_step_lookahead_maximization(s: int, gamma: float, states: List, actions: List, c_max: float,
                                                p_max: float, N: List, E: List, S: List, V_base: List, l: int,
                                                delay_table: List) -> Tuple[int, float]:
    """
    Performs the deterministic l-step lookahead maximization

    :param s: the current state
    :param gamma: the discount factor
    :param states: the state space
    :param actions: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param N: the set of nodes
    :param E: the set of edges
    :param S: the set of services
    :param V_base: the value function of the base policy
    :param l: the length of the lookahead horizon
    :param delay_table: a table with the predicted delays
    :return: the best action and its Q-factor
    """
    if l == 1:
        A = np.zeros(len(actions))
        for a in range(len(actions)):
            r = reward_function(s=states[int(s)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                services=S, states=states, delay_table=delay_table)
            s_prime = transition_function(s=states[int(s)], a=actions[a], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          services=S)
            s_prime = states.index(s_prime)
            A[a] = r + gamma * V_base[s_prime]
        best_a = np.argmax(A)
        return int(best_a), float(A[best_a])
    else:
        A = np.zeros(len(actions))
        for a in range(len(actions)):
            r = reward_function(s=states[int(s)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                services=S, states=states, delay_table=delay_table)
            s_prime = transition_function(s=states[int(s)], a=actions[a], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          services=S)
            s_prime = states.index(s_prime)
            _, s_prime_val = deterministic_l_step_lookahead_maximization(
                s=s_prime, gamma=gamma, states=states, actions=actions, c_max=c_max, p_max=p_max, N=N,
                E=E, S=S, V_base=V_base, l=l - 1, delay_table=delay_table)
            A[a] = r + gamma * s_prime_val
        best_a = np.argmax(A)
        return int(best_a), float(A[best_a])


def deterministic_l_step_lookahead_maximization_multiagent(s: int, gamma: float, states: List, actions: List,
                                                           c_max: float,
                                                           p_max: float, N: List, E: List, S: List, V_base: List,
                                                           l: int,
                                                           delay_table: List, component_spaces, agent_index: int,
                                                           base_pi, partial_a) -> Tuple[int, float]:
    """
    Performs the deterministic l-step multi-agent lookahead maximization

    :param s: the current state
    :param gamma: the discount factor
    :param states: the state space
    :param actions: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param N: the set of nodes
    :param E: the set of edges
    :param S: the set of services
    :param V_base: the value function of the base policy
    :param l: the length of the lookahead horizon
    :param delay_table: a table with the predicted delays
    :param component_spaces: action spaces for each component of the action space
    :param agent_index: the index of the agent
    :param base_pi: the base policy
    :param partial_a: the previous comleted action components by other agents
    :return: the best action and its Q-factor
    """
    if l == 1:
        A = np.zeros(len(component_spaces[agent_index]))
        base_action_idx = np.random.choice(np.arange(0, len(actions)), p=base_pi[s])
        base_action = actions[base_action_idx]
        a_vec = partial_a.copy()
        a_vec.append(-1)
        for i in range(agent_index + 1, len(component_spaces)):
            a_vec.append(base_action[i])
        for local_a in range(len(component_spaces[agent_index])):
            a_vec[agent_index] = component_spaces[agent_index][local_a]
            a = actions.index(tuple(a_vec))
            r = reward_function(s=states[int(s)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                services=S, states=states, delay_table=delay_table)
            s_prime = transition_function(s=states[int(s)], a=actions[a], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          services=S)
            s_prime = states.index(s_prime)
            A[local_a] = r + gamma * V_base[s_prime]
        best_local_a_idx = np.argmax(A)
        best_a = a_vec.copy()
        best_a[agent_index] = component_spaces[agent_index][best_local_a_idx]
        return best_a, A[best_local_a_idx]
    else:
        A = np.zeros(len(component_spaces[agent_index]))
        base_action_idx = np.random.choice(np.arange(0, len(actions)), p=base_pi[s])
        base_action = actions[base_action_idx]
        a_vec = partial_a.copy()
        a_vec.append(-1)
        for i in range(agent_index + 1, len(component_spaces)):
            a_vec.append(base_action[i])
        for local_a in range(len(component_spaces[agent_index])):
            a_vec[agent_index] = component_spaces[agent_index][local_a]
            a = actions.index(tuple(a_vec))
            r = reward_function(s=states[int(s)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                services=S, states=states, delay_table=delay_table)
            s_prime = transition_function(s=states[int(s)], a=actions[a], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          services=S)
            s_prime = states.index(s_prime)
            _, s_prime_val = deterministic_l_step_lookahead_maximization_multiagent(
                s=s_prime, gamma=gamma, states=states, actions=actions, c_max=c_max, p_max=p_max, N=N,
                E=E, S=S, V_base=V_base, l=l - 1, delay_table=delay_table, component_spaces=component_spaces,
                partial_a=partial_a, agent_index=agent_index, base_pi=base_pi)
            A[local_a] = r + gamma * s_prime_val
        best_local_a_idx = np.argmax(A)
        best_a = a_vec.copy()
        best_a[agent_index] = component_spaces[agent_index][best_local_a_idx]
        return best_a, A[best_local_a_idx]


def monte_carlo_l_step_lookahead_maximization(s: int, gamma: float, batch_size: int, states: List,
                                              actions: List, c_max: float, p_max: float, N: List, E: List, S: List,
                                              V_base: List, l: int, delay_table: List) -> Tuple[int, float]:
    """
    Monte-Carlo l-step lookahead maximization

    :param s: the current state
    :param gamma: the discount factor
    :param batch_size: the batch size for evaluation
    :param states: the state space
    :param actions: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param N: the set of nodes
    :param E: the set of edges
    :param S: the set of services
    :param V_base: the value function of the base policy
    :param l: the length of the lookahead horizon
    :param delay_table: the table with the predicted delays
    :return: the best action and its value
    """
    V_base_prime = V_base.copy()
    for i in range(0, l - 1):
        for s_k in range(len(states)):
            print(f" l-step lookahead, time-index:{i}/{l}, state: {s_k}/{len(states)}")
            A = np.zeros(len(actions))
            for a in range(len(actions)):
                returns = []
                for k in range(batch_size):
                    r = reward_function(s=states[int(s_k)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max,
                                        p_max=p_max, services=S, delay_table=delay_table, states=states)
                    s_prime = transition_function(s=states[int(s_k)], a=actions[a], nodes=N, edges=E, c_max=c_max,
                                                  p_max=p_max, services=S)
                    s_prime = states.index(s_prime)
                    returns.append(r + gamma * V_base_prime[s_prime])
                A[a] = np.mean(returns)
            V_base_prime[s_k] = np.max(A)

    A = np.zeros(len(actions))
    for a in range(len(actions)):
        returns = []
        for k in range(batch_size):
            r = reward_function(s=states[int(s)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                services=S, delay_table=delay_table, states=states)
            s_prime = transition_function(s=states[int(s)], a=actions[a], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          services=S)
            s_prime = states.index(s_prime)
            returns.append(r + gamma * V_base_prime[s_prime])
        A[a] = np.mean(returns)
    best_a = np.argmax(A)
    return int(best_a), float(A[best_a])


def monte_carlo_policy_evaluation(policy: Union[MaskablePPO, List], states: List, actions: List, gamma: float,
                                  batch_size: int, N: List,
                                  E: List, c_max: float, p_max: float, S: List, delay_table: List,
                                  delta_c: float, delta_p: float, eval_state=None,
                                  modified_s: int = None, modified_a: int = None, env=None) -> List:
    """
    Performs Monte-Carlo policy evaluation

    :param policy: the policy to evaluate
    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param batch_size: the batch size for evaluation
    :param N: the set of nodes
    :param E: the set of edges
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param S: the set of services
    :param delay_table: the table with the predicted service request delays
    :param delta_c: the discretization step of the scaling action
    :param delta_p: the discretization step of the routing action
    :param eval_state: the state to use for evaluation
    :param modified_s: a possibly modified state to evaluate
    :param modified_a: a possibly modified action to evaluate
    :param env: the evaluation RL environment
    :return: the value function
    """
    V = np.zeros(len(states))
    horizon = int(1 / (1 - gamma))
    if eval_state is None:
        eval_states = list(range(len(states)))
    else:
        eval_states = [eval_state]
    for s in eval_states:
        returns = []
        for i in range(batch_size):
            ret = 0
            state = s
            for t in range(horizon):
                if modified_s is not None and modified_s == state:
                    a = modified_a
                else:
                    if isinstance(policy, list) or isinstance(policy, np.ndarray):
                        a = np.random.choice(np.arange(0, len(actions)), p=policy[state])
                    else:
                        obs = convert_dp_s_to_env_s(s=states[state])
                        env.set_state(p1=obs[0], p2=obs[1], cpu1=obs[2], cpu2=obs[3])
                        action_masks = get_action_masks(env)
                        a, _ = policy.predict(obs, deterministic=True, action_masks=action_masks)
                        a = convert_env_a_to_dp_a(a=list(a), delta_p=delta_p, delta_c=delta_c)
                        a = actions.index(tuple(a))
                r = reward_function(s=states[int(state)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                    services=S, delay_table=delay_table, states=states)
                # print(f"s: {states[int(state)]}, a: {actions[int(a)]}, r: {r}")
                ret += r
                s_prime = transition_function(s=states[int(state)], a=actions[int(a)], nodes=N, edges=E, c_max=c_max,
                                              p_max=p_max, services=S)
                state = states.index(s_prime)
            returns.append(ret)
        V[s] = np.mean(returns)
    return list(V.tolist())

src/online_policy_adaptation_using_rollout/algorithms/rollouts_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout
"""
Implements the value iteration algorithm for scenario 1
"""
from typing import List, Tuple
import numpy as np


def one_step_lookahead_maximization(s, V, num_actions, num_states, T, gamma, R) -> Tuple[int, float]:
    """
    One-step lookahead maximization for the value iteration algorithm

    :param s: the current state
    :param V: the value function
    :param num_actions: the number of actions
    :param num_states: the number of states
    :param T: the transition tensor
    :param gamma: the discount factor
    :param R: the reward tensor
    :return: the best action and its value
    """
    A = np.zeros(num_actions)
    for a in range(num_actions):
        reward = R[s][a]
        for next_state in range(num_states):
            prob = T[s][next_state][a]
            A[a] += prob * (reward + gamma * V[next_state])
    best_action = np.argmax(A)
    best_action_value = A[best_action]
    return int(best_action), float(best_action_value)


def value_iteration(T: List, num_states: int, num_actions: int, R: List, theta=0.0001, gamma=1.0) -> List:
    """
    The value iteration algorithm

    :param T: the transition tensor
    :param num_states: the number of states
    :param num_actions: the number of actions
    :param R: the reward tensor
    :param theta: the theta parameter to measure convergence
    :param gamma: the discount factor
    :return: the value function
    """
    V = np.zeros(num_states)
    iteration = 0
    while True:
        delta = 0
        for s in range(num_states):
            best_action, best_action_value = one_step_lookahead_maximization(s=s, V=V, num_actions=num_actions,
                                                                             num_states=num_states, T=T, gamma=gamma,
                                                                             R=R)
            delta = max(delta, np.abs(best_action_value - V[s]))
            V[s] = best_action_value
        iteration += 1
        print(f"VI iteration: {iteration}, delta: {delta}, theta: {theta}")
        if delta < theta:
            break
    print("VI converged")
    return V.tolist()


def greedy_policy(T: List, R: List, gamma: float, V: List, num_states: int, num_actions: int) -> List:
    """
    Computes the greedy policy based on a given value function

    :param T: the transition tensor
    :param R: the reward tensor
    :param gamma: the discount factor
    :param V: the value function
    :param num_states: the number of states
    :param num_actions: the number of actions
    :return: the greedy policy
    """
    pi_prime = np.zeros((num_states, num_actions))
    for s in range(0, num_states):
        action_values = np.zeros(num_actions)
        for a in range(0, num_actions):
            action_values[a] = R[s][a]
        best_action = np.argmax(action_values)
        pi_prime[s, best_action] = 1
    return pi_prime

src/online_policy_adaptation_using_rollout/algorithms/value_iteration_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout
import random
import numpy as np
import joblib
import paramiko
import subprocess
import time
import requests
import sys
import os
from pathlib import Path

'''
This function applies the configuration on the master node. This configuration includes routing weights, scalings and blocking upper limits
'''

# x.x.x.x:xxxx (IP and port of book and city services services on the cluster) /home/shahab/searchapp3/data_collection_for_dropping/ (path to the command to file and routing config file on the master node) username (of your account on the master node) password (of your account on the master node) x.x.x.x (IP of master node)

args = sys.argv
IP_port = args[1]
path_to_routing_config = args[2]
path_to_scaling_config = args[3]
server_username = args[4]
server_password = args[5]
server_hostname = args[6]

artificas = "./artifacts/"
path_to_LG = ""


def take_routing_action_master_node(p11, p21, path_to_routing_config, server_hostname, server_username,
                                    server_password):
    '''
    :param p11: The routing weight of service 1 towards the computeinfo node 1.
    The remaining traffic will route towards node 2
    :param p21: The routing weight of service 2 towards the computeinfo node 1.
    The remaining traffic will route towards node 2
    :param path_to_routing_config: This is the path to the file that convert the command to configuration file on the master node.
    :param server_hostname: This is the ip address of the master node.
    :param server_username: This is the username defined on the master node.
    :param server_password: This is the password defined on the master node.
    :return: None

    In this funtion, we change the routing configration file for the services towards the node1 and node2 and then apply
    the configuration file on the master node.
    '''
    routing_weight1 = p11 * 100
    routing_weight2 = p21 * 100
    #################Taking routing actions################
    hostname = server_hostname
    username = server_username
    password = server_password
    commands = ["python " + path_to_routing_config + "/command_to_file.py " +
                str(int(routing_weight1)) + " " + str(int(routing_weight2))]
    commands.append("kubectl apply -f " + path_to_routing_config + "/user_based_weight.yaml")
    client = paramiko.SSHClient()
    # add to known hosts
    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    try:
        client.connect(hostname=hostname, username=username, password=password)
    except:
        print("[!] Cannot connect to the SSH Server")
        exit()

    for command in commands:
        print("=" * 50, command, "=" * 50)
        stdin, stdout, stderr = client.exec_command(command)
        err = stderr.read().decode()
        if err:
            print("The error is:" + err)
    client.close()
    pass

def take_blocking_action_master_node(IP_port):
    '''
    :param IP_port: This is the ip and port address of the services running on the cluster.
    Please note that the ip and port are the same for both services but the URLs are different.
    :return:

    In this function we specify the upper bound of the number of requests that will be replied.
    If the rate is higher than the expetation, the front node will reply by the response with the code 429.
    '''
    #################Taking blocking actions###############
    r = requests.get('http://'+IP_port+'/ratebook/' + str(limit1))
    print(r.text)
    r = requests.get('http://'+IP_port+'/ratecity/' + str(limit2))
    print(r.text)


def take_blocking_action_master_node(limit1, limit2, IP_port, path_to_routing_config, server_hostname, server_username,
                                     server_password):
    r = requests.get('http://' + IP_port + '/rateinfo/' + str(limit2))
    print(r.text)
    r = requests.get('http://' + IP_port + '/ratecompute/' + str(limit1))
    print(r.text)
    pass


def take_scaling_action_on_master_node(c1, c2, IP_port, path_to_scaling_config, server_hostname, server_username,
                                       server_password):
    '''
    :param c1: This is the scaling action on node 1
    :param c2: This is the scaling action on node 2
    :param IP_port: this is the ip and port of the serviecs on the cluster
    :param path_to_scaling_config: This is the path to the file to convert the scaling action to the configuration file on the master node.
    :param server_hostname: This is the ip address of the master node.
    :param server_username: The username defined on the master node.
    :param server_password: The password for the above-mentioed username on the master node.
    :return: None

    In this function, we change the allocated CPU to the computeinfo nodes and apply the configuration on the master node.
    '''
    #################Taking scaling actions################
    scaling1 = c1 * 1000
    scaling2 = c2 * 1000
    #################Taking routing actions################
    hostname = server_hostname
    username = server_username
    password = server_password
    commands = ["python " + path_to_scaling_config + "/command_to_file.py 1 " + str(int(scaling1)) + " " + str(
        int(scaling1)) + " computeinfo1.yaml"]
    commands.append("kubectl apply -f " + path_to_scaling_config + "/computeinfo1.yaml")
    commands.append("python "+path_to_scaling_config+"/command_to_file.py 1 " + str(int(scaling2)) + " " + str(int(scaling2)) + " computeinfo2.yaml")
    commands.append("kubectl apply -f "+path_to_scaling_config+"/computeinfo2.yaml")
    client = paramiko.SSHClient()
    # add to known hosts
    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    try:
        client.connect(hostname=hostname, username=username, password=password)
    except:
        print("[!] Cannot connect to the SSH Server")
        exit()

    for command in commands:
        print("=" * 50, command, "=" * 50)
        stdin, stdout, stderr = client.exec_command(command)
        err = stderr.read().decode()
        if err:
            print("The error is:" + err)
    client.close()


def run_load_generator(LG_name, load_value):
    '''
    :param LG_name: This paramter specifes the name of the service that we like to load
    :param load_value: Load value is the number requests per second (
    Plese note that this is when we like to specify the constant load for the specific time interval.
    We can also specify the load pattern in a file and call the load generator with those parameters.)
    :return: None

    This function start the load generator for the specific service with the specific load pattern.
    In the case of data collection we call the load generator with the contant load for a specif time interval in which the system has specific configuration.
    '''
    # Define the shell command to run
    shell_cmd = "nohup python " + path_to_LG + "async_load_generator.py " + LG_name + " constant " + str(
        load_value) + " " + IP_port + " > /dev/null &"

    # Run the command in the background
    subprocess.Popen(shell_cmd, shell=True)


def kill_load_generator():
    '''
    This function is used to kill all the load generators before or after the process of load generarion.
    :return: None
    '''
    # Define the shell command to run
    shell_cmd = "pkill -f async_load_generator.py"

    # Run the command in the background
    subprocess.Popen(shell_cmd, shell=True)


'''
The load pattern states the values of load that should be generated over time. For instance, the load pattern with values of [5,5,10,15,20] means that over the first time step (in this work is 5 sec) is 5, the second time step it is 5, the third is 15 and so on. 

This function accepts the name of file which includes the load pattern and it returns the load pattern as a set
'''


def get_load_pattern(file_name):
    '''
    We use this function to read specific load pattern from a file. In the case of data collection, we do not use this function.
    :param file_name: Name of the file that includes the load pattern
    :return: This file returns the set consisting of load values.
    '''
    with open(file_name) as file:
        lines = file.readlines
    file.close()
    load_pattern = []
    for line in lines:
        load_pattern.append(float(line))
    return load_pattern


def aggregate_samples(number_of_samples, p11, p21, c1, path_to_files, expected_l1, expected_l2, Timestep=5):
    '''
    We use this function to collect samples for specific load value and specific configuration within specified time steps.

    :param number_of_samples: This parameter specifies how many samples we like to collect for the given load value and configuration.
    We usually need to collect more than one sample on a real testbed sine the system is not deterministic.
    :param p11: The weight of service 1 towards node 1
    :param p21: The weight of service 1 towards node 2
    :param c1: The scaling action for node 1
    :param c2: The scaling action for node 2
    :param path_to_files: The path to the file that we save the samples.
    :param expected_l1: This is the load that we send towards the nodes.
    However, in a real system the load that is sent may be slightly larger or smaller than this value.
    :param expected_l2: This is the load that we send towards the nodes.
    However, in a real system the load that is sent may be slightly larger or smaller than this value.
    :param Timestep: this is the time step of our system when we model it as a discrete model
    :return: None
    '''

    counter = 0

    while counter < number_of_samples:
        print("The counter is:", counter)
        ##############First we clean up all the files
        with open(path_to_files + "load_compute.txt", 'w') as f:
            f.write("")
        f.close()
        with open(path_to_files + "response_compute.txt", "w") as f:
            f.write("")
        f.close()

        with open(path_to_files + "load_info.txt", 'w') as f:
            f.write("")
        f.close()
        with open(path_to_files + "response_info.txt", "w") as f:
            f.write("")
        f.close()

        ###############Wait for the time step
        time.sleep(Timestep)
        ###############Read the state
        with open(path_to_files + "load_compute.txt", 'r') as f:
            lines = f.readlines()
        f.close()

        offered_load = len(lines)

        with open(path_to_files + "response_compute.txt", "r") as f:
            lines = f.readlines()
        f.close()

        info_res_1 = []
        info_res_2 = []
        info_drops = []
        for line in lines:
            req = eval(line)
            if (req["code"] == 200):
                if (req["Node"] == 1):
                    info_res_1.append(req["response"])
                elif (req["Node"] == 2):
                    info_res_2.append(req["response"])
            else:
                info_drops.append(req["response"])
        ###########Info###############
        info_all_req = info_res_1 + info_res_2
        l1 = offered_load
        cl1 = len(info_res_1) + len(info_res_2)
        if (len(info_res_1) > 0):
            d11 = np.mean(info_res_1)
            d11_std = np.std(info_res_1)
        else:
            d11 = 0
            d11_std = 0

        if (len(info_res_2) > 0):
            d12 = np.mean(info_res_2)
            d12_std = np.std(info_res_2)
        else:
            d12 = 0
            d12_std = 0

        if (len(info_all_req) > 0):
            d1 = np.mean(info_all_req)
            d1_99 = np.percentile(info_all_req, 99)
            d1_90 = np.percentile(info_all_req, 90)
            d1_std = np.std(info_all_req)
        else:
            d1 = 0
            d1_99 = 0
            d1_90 = 0
            d1_std = 0

        drop1 = len(info_drops)

        ########Read for service compute#########
        with open(path_to_files + "load_compute.txt", 'r') as f:
            lines = f.readlines()
        f.close()

        offered_load = len(lines)

        with open(path_to_files + "response_compute.txt", "r") as f:
            lines = f.readlines()
        f.close()

        compute_res_1 = []
        compute_res_2 = []
        compute_drops = []
        for line in lines:
            req = eval(line)
            if (req["code"] == 200):
                if (req["Node"] == 1):
                    compute_res_1.append(req["response"])
                elif (req["Node"] == 2):
                    compute_res_2.append(req["response"])
            else:
                compute_drops.append(req["response"])
        ###########Book###############
        compute_all_req = compute_res_1 + compute_res_2
        l2 = offered_load
        cl2 = len(compute_res_1) + len(compute_res_2)
        if (len(compute_res_1) > 0):
            d21 = np.mean(compute_res_1)
            d21_std = np.std(compute_res_1)
        else:
            d21 = 0
            d21_std = 0

        if (len(compute_res_2) > 0):
            d22 = np.mean(compute_res_2)
            d22_std = np.std(compute_res_2)
        else:
            d22 = 0
            d22_std = 0

        if (len(compute_all_req) > 0):
            d2 = np.mean(compute_all_req)
            d2_99 = np.percentile(compute_all_req, 99)
            d2_90 = np.percentile(compute_all_req, 90)
            d2_std = np.std(compute_all_req)
        else:
            d2 = 0
            d2_99 = 0
            d2_90 = 0
            d2_std = 0

        drop2 = len(compute_drops)

        ##############Input of the system model################
        # l1 is the offered load that LG sends out
        # p11 is the routing weight of service 1 toward node 1, p21 is the routing weight of service 2 toward node 1
        # p12 = 1 - p11 that is why we dont save it here.
        # b1 and b2 the blocking rates.
        # l1,l2,p11,p21,b1,b2

        ##############Out put of system model#########
        # cl1, cl2 are the carried load of the system. Carried load is the part of load that system will respond and not drops
        # d1, d2 are the average response time of service 1 and 2 over the specified time step which is 5 sec here.
        # d1_std and d2_std are standard deviation
        # d11, d12,d21,d22 these are the average response time for service i on node j
        # d11_std, d12_std, d21_std, d22_std which are standard deviations
        # drop 1, drop 2 which are the number of dropped requests.
        # cl1, cl2, d1, d2,d1_std,d2_std, d11, d12, d21, d22, d11_std, d12_std, d21_std, d22_std, drop1, drop2

        line = (str(expected_l1) + "," + str(expected_l2) + "," + str(l1) + "," + str(l2) + "," + str(b1) + "," + str(b2)
                + "," + str(p11) + "," + str(p21) + "," + str(c1) + "," + str(c2) + "," + str(cl1)+ "," + str(cl2) + ","
                + str(d1) + "," + str(d1_std) + "," + str(d11) + "," + str(d12) + "," + str(d11_std) + "," + str(d12_std)
                + "," + str(drop1) + "," + str(d1_99) + "," + str(d1_90)+ str(d2) + "," + str(d2_std) + "," + str(d21)
                + "," + str(d22) + "," + str(d21_std) + "," + str(d22_std)
                + "," + str(drop2) + "," + str(d2_99) + "," + str(d2_90) + "\n")

        with open(path_to_files + "data.csv", 'a') as f:
            f.write(line)
        f.close()
        counter += 1


def cleanup_files():
    '''
    We use this function to clean up all related files from previous runs.
    :return: None.
    '''
    file.close()
    path = Path(artificas + "load_compute.txt")
    if (path.is_file()):
        os.remove(artificas + "load_compute.txt")
    path = Path(artificas + "response_compute.txt")
    if (path.is_file()):
        os.remove(artificas + "response_compute.txt")


if __name__ == "__main__":
    # load1 = get_load_pattern("load_pattern_for_service_1")
    # load2 = get_load_pattern("load_pattern_for_service_2")
    info_load = [5, 10, 15, 20]
    compute_load = [1, 2, 3, 4, 5]
    actions = [0, 0.2, 0.4, 0.6, 0.8, 1]
    scaling_actions = [1, 2, 3, 4, 5]

    kill_load_generator()

    path = Path(artificas + "data.csv")
    if (path.is_file()):
        os.remove(artificas + "data.csv")
    with open(artificas + "data.csv", 'w') as file:
        file.write("expected_l1,expected_l2,l1,l2,b1,b2,p11,p21,c1,c2,cl1,cl2,d1,d1_std,d11,d12,d11_std,d12_st,drop1,d1_99,d1_90,"
                   "d2,d2_std,d21,d22,d21_std,d22_st,drop2,d2_99,d2_90\n")

    time_step = 5

    for l1 in info_load:
        for l2 in info_load:
            run_load_generator("compute",l2)
            run_load_generator("info",l1)
            time.sleep(30)
            cleanup_files()
            for p11 in actions:
                for p21 in actions:
                    for b1 in actions:
                        for b2 in actions:
                            limit1 = (1-b1) * l1
                            limit2 = (1-b2) * l2
                            take_routing_action_master_node(p11,p21, IP_port, path_to_routing_config, server_hostname, server_username, server_password)
                            take_blocking_action_master_node(limit1,limit2, IP_port, path_to_routing_config, server_hostname, server_username, server_password)
                            aggregate_samples(2,p11, p21, b1, b2,artificas,l1,l2,time_step)
            kill_load_generator()
src/online_policy_adaptation_using_rollout/data_collection/data_collection.py
foroughsh/online_policy_adaptation_using_rollout
"""
Script with functions for running different algorithms for scenario 1
"""
from typing import List, Union, Tuple, Any
import time
import math
import numpy as np
import numpy.typing as npt
from sb3_contrib import MaskablePPO
from sb3_contrib.common.maskable.utils import get_action_masks
from online_policy_adaptation_using_rollout.algorithms.value_iteration_scenario_1 import value_iteration, greedy_policy
from online_policy_adaptation_using_rollout.algorithms.policy_iteration_scenario_1 import policy_iteration, \
    policy_evaluation
from online_policy_adaptation_using_rollout.algorithms.rollouts_scenario_1 import exact_one_step_rollout_policy, \
    monte_carlo_policy_evaluation, \
    monte_carlo_l_step_rollout_policy, monte_carlo_l_step_multiagent_rollout_policy
from online_policy_adaptation_using_rollout.system_model.system_model_scenario_1 import reward_tensor, \
    transition_tensor, transition_function, \
    precompute_delays, reward_function, convert_env_a_to_dp_a, \
    convert_dp_s_to_env_s
import random


def newton_step_data(base_pi: Union[List, MaskablePPO], states: List, actions: List, gamma: float,
                     S: list, N: list, E: list, p_max: float, c_max: float, initial_state: int,
                     batch_size: int, V_base: List, l: int,
                     deterministic_model: bool, delay_table: List, alt_pi, alt_V, env=None) \
        -> Tuple[float, float, float, float, float, float, float, float]:
    """
    Implements the rollout Newton step in a data-driven fashion through monte-carlo evaluation of the value function

    :param base_pi: the base policy
    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the max value for the routing action
    :param c_max: the max value for the scaling action
    :param initial_state: the initial state
    :param batch_size: the batch size
    :param V_base: the base value function
    :param l: the lookahead horizon
    :param deterministic_model: boolean flag indicating whether the model is deterministic or not
    :param delay_table: a table with delay predictions
    :param alt_pi: an alternative base policy for evaluation
    :param alt_V: an alternative base value function for evaluation
    :param env: the RL environment
    :return: value of initial state with two differnet rollout policies and two base policies, slopes of the
             Bellman operators, intercepts of the Bellman operators
    """
    s = initial_state
    rollout_pi, V_rollout, rollout_a = monte_carlo_l_step_rollout_policy(base_pi=base_pi, s=s, gamma=gamma,
                                                                         batch_size=batch_size, states=states,
                                                                         actions=actions, c_max=c_max,
                                                                         p_max=p_max, N=N,
                                                                         E=E, S=S, V_base=V_base, l=l,
                                                                         deterministic_model=deterministic_model,
                                                                         delay_table=delay_table, env=env)
    V_base_2 = V_base.copy()
    V_base_2[s] = 0.5
    rollout_pi_2, V_rollout_2, rollout_a_2 = monte_carlo_l_step_rollout_policy(base_pi=base_pi, s=s, gamma=gamma,
                                                                               batch_size=batch_size, states=states,
                                                                               actions=actions, c_max=c_max,
                                                                               p_max=p_max, N=N,
                                                                               E=E, S=S, V_base=V_base_2, l=l,
                                                                               deterministic_model=deterministic_model,
                                                                               delay_table=delay_table, env=env)
    alt_pi_2 = alt_pi.copy()
    obs = convert_dp_s_to_env_s(s=states[s])
    env.set_state(p1=obs[0], p2=obs[1], cpu1=obs[2], cpu2=obs[3])
    action_masks = get_action_masks(env)
    base_a, _ = base_pi.predict(obs, deterministic=True, action_masks=action_masks)
    base_a = convert_env_a_to_dp_a(a=list(base_a), delta_p=delta_p, delta_c=delta_c)
    base_a = actions.index(tuple(base_a))
    V_base_3 = monte_carlo_policy_evaluation(policy=alt_pi_2, states=states, actions=actions, gamma=gamma,
                                             batch_size=batch_size, N=N, E=E, c_max=c_max, p_max=p_max, S=S,
                                             delay_table=delay_table, delta_c=1,
                                             delta_p=0.2, modified_s=s, modified_a=base_a, eval_state=None, env=env)

    T_rollout_slope = round((V_rollout[s] - V_rollout_2[s]) / (V_rollout[s] - 0.5), 3)
    T_rollout_intercept = round(V_rollout[s] - T_rollout_slope * V_rollout[s], 3)
    T_base_slope = round((V_base[s] - V_base_3[s]) / (V_base[s] - alt_V[s]), 3)
    T_base_intercept = round(V_base[s] - T_base_slope * V_base[s], 3)

    return V_rollout[s], V_rollout_2[s], V_base[s], V_base_3[
        s], T_rollout_slope, T_rollout_intercept, T_base_slope, T_base_intercept


def run_base_policy_online(base_pi: Union[List, MaskablePPO], states: List, actions: List,
                           S: list, N: list, E: list, p_max: float, c_max: float, initial_state: int,
                           horizon: int, V_base: List,
                           seeds: List[int],
                           delay_table: List, num_episodes: int = 1,
                           delta_p: float = None, delta_c: float = None, env=None) -> Tuple[List[float, List[float]]]:
    """
    Runs the base policy online

    :param base_pi: the base policy to run
    :param states: the state space
    :param actions: the action space
    :param S: the list of services
    :param N: the list of nodes
    :param E: the list of edges
    :param p_max: the maximum routing action value
    :param c_max: the maximum scaling action value
    :param initial_state: the initial state
    :param horizon: the horizon for online play
    :param V_base: the value function of the base policy
    :param seeds: the random seeds
    :param delay_table: the table with predicted delays
    :param num_episodes: the number of episodes to run
    :param delta_p: the discretization step of the routing action
    :param delta_c: the discretization step of the scaling action
    :param env: the evaluation environment
    :return: the mean returns and standard deviations
    """
    reward_trajectories = []
    for k in range(num_episodes):
        np.random.seed(seeds[k])
        random.seed(seeds[k])
        s = initial_state
        print(f"Starting monte-carlo online play, initial state: {s}, episode: {k}/{num_episodes}")
        rewards = []
        for i in range(horizon):
            if isinstance(base_pi, list) or isinstance(base_pi, np.ndarray):
                base_a = int(np.random.choice(np.arange(0, len(actions)), p=base_pi[s]))
                base_a = actions[base_a]
            else:
                obs = convert_dp_s_to_env_s(s=states[s])
                env.set_state(p1=obs[0], p2=obs[1], cpu1=obs[2], cpu2=obs[3])
                action_masks = get_action_masks(env)
                base_a, _ = base_pi.predict(obs, deterministic=True, action_masks=action_masks)
                base_a = convert_env_a_to_dp_a(a=list(base_a), delta_p=delta_p, delta_c=delta_c)
            base_a_idx = actions[actions.index(tuple(base_a))]
            r_base = reward_function(s=states[int(s)], a=base_a_idx, services=S, nodes=N,
                                     edges=E, c_max=c_max,
                                     p_max=p_max, states=states, delay_table=delay_table)
            rewards.append(r_base)
            print(f"s: {states[s]}, V_base[s]:{V_base[s]}, "
                  f"base_pi[s]: {base_a}, "
                  f"r_base: {r_base}")
            s = transition_function(s=states[int(s)], a=base_a_idx, nodes=N, edges=E, c_max=c_max,
                                    p_max=p_max, services=S)
            s = states.index(s)
        reward_trajectories.append(rewards)
    means = []
    stds = []
    for i in range(len(reward_trajectories[0])):
        rewards = []
        for k in range(num_episodes):
            rewards.append(reward_trajectories[k][i])
        rewards = np.array(rewards)
        means.append(np.mean(rewards))
        stds.append(np.std(rewards) * (1 / math.sqrt(num_episodes)))
    return means, stds


def run_monte_carlo_one_step_online_play(base_pi: Union[List, MaskablePPO], states: List, actions: List, gamma: float,
                                         S: list, N: list, E: list, p_max: float, c_max: float, initial_state: int,
                                         horizon: int, batch_size: int, V_base: List, l: int,
                                         seeds: List[int], component_spaces,
                                         deterministic_model: bool, delay_table: List, num_episodes: int = 1,
                                         delta_p: float = None, delta_c: float = None, env=None,
                                         multiagent: bool = False) \
        -> Tuple[List[float], List[float], List[float], List[float]]:
    """
    Runs the rollout algorithm with monte carlo value evaluation

    :param base_pi: the base policy
    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the maximum routing action value
    :param c_max: the maximum scaling action value
    :param initial_state: the initial state
    :param horizon: the horizon for the evaluation
    :param batch_size: the batch size for the evaluation
    :param V_base: the value function of the base policy
    :param l: the lookahead optimization horizon
    :param seeds: the random seeds
    :param component_spaces: the action spaces of each action component
    :param deterministic_model: boolean flag indicating whether it is a deterministic model or not
    :param delay_table: a table with the predicted response times
    :param num_episodes: the number of episodes to run
    :param delta_p: the discretization step of the routing action
    :param delta_c: the discretization step of the scaling action
    :param env: the environment for evaluation
    :param multiagent: a boolean flag indicating whether the multiagent rollout algorithm should be run
    :return: mean returns, std returns, mean times, std times
    """
    reward_trajectories = []
    time_trajectories = []
    for k in range(num_episodes):
        np.random.seed(seeds[k])
        random.seed(seeds[k])
        s = initial_state
        print(f"Starting monte-carlo online play, initial state: {s}, episode: {k}/{num_episodes}, l: {l}")
        rewards = []
        rollout_times = []
        for i in range(horizon):
            start = time.time()
            if not multiagent:
                rollout_pi, V_rollout, rollout_a = monte_carlo_l_step_rollout_policy(base_pi=base_pi, s=s, gamma=gamma,
                                                                                     batch_size=batch_size,
                                                                                     states=states,
                                                                                     actions=actions, c_max=c_max,
                                                                                     p_max=p_max, N=N,
                                                                                     E=E, S=S, V_base=V_base, l=l,
                                                                                     deterministic_model=deterministic_model,
                                                                                     delay_table=delay_table, env=env)
            if multiagent:
                rollout_pi, V_rollout, rollout_a = monte_carlo_l_step_multiagent_rollout_policy(
                    base_pi=base_pi, s=s, gamma=gamma, batch_size=batch_size, states=states, actions=actions,
                    c_max=c_max, p_max=p_max, N=N, E=E, S=S, V_base=V_base, l=l,
                    deterministic_model=deterministic_model,
                    delay_table=delay_table, env=env, component_spaces=component_spaces)
            end = time.time()
            rollout_times.append(end - start)

            if isinstance(base_pi, list) or isinstance(base_pi, np.ndarray):
                base_a = int(np.random.choice(np.arange(0, len(actions)), p=base_pi[s]))
                base_a = actions[base_a]
            else:
                obs = convert_dp_s_to_env_s(s=states[s])
                env.set_state(p1=obs[0], p2=obs[1], cpu1=obs[2], cpu2=obs[3])
                action_masks = get_action_masks(env)
                base_a, _ = base_pi.predict(obs, deterministic=False, action_masks=action_masks)
                base_a = convert_env_a_to_dp_a(a=list(base_a), delta_p=delta_p, delta_c=delta_c)
            r_rollout = reward_function(s=states[int(s)], a=actions[rollout_a], services=S, nodes=N, edges=E,
                                        c_max=c_max,
                                        p_max=p_max, states=states, delay_table=delay_table)
            r_base = reward_function(s=states[int(s)], a=actions[actions.index(tuple(base_a))], services=S, nodes=N,
                                     edges=E, c_max=c_max,
                                     p_max=p_max, states=states, delay_table=delay_table)
            rewards.append(r_rollout)
            print(f"s: {states[s]}, V_base[s]:{V_base[s]}, V_rollout[s]: {V_rollout[s]}, "
                  f"base_pi[s]: {base_a}, "
                  f"rollout_pi[s]: {actions[rollout_a]}, r_base: {r_base}, r_rollout: {r_rollout}")
            s = transition_function(s=states[int(s)], a=actions[rollout_a], nodes=N, edges=E, c_max=c_max,
                                    p_max=p_max, services=S)
            s = states.index(s)
        reward_trajectories.append(rewards)
        time_trajectories.append(rollout_times)
    means = []
    stds = []
    for i in range(len(reward_trajectories[0])):
        rewards = []
        for k in range(num_episodes):
            rewards.append(reward_trajectories[k][i])
        rewards = np.array(rewards)
        means.append(np.mean(rewards))
        stds.append(np.std(rewards))

    time_means = []
    time_stds = []
    for i in range(len(time_trajectories[0])):
        times = []
        for k in range(num_episodes):
            times.append(time_trajectories[k][i])
        times = np.array(times)
        time_means.append(np.mean(times))
        time_stds.append(np.std(times))

    return means, stds, time_means, time_stds


def run_exact_one_step_online_play(base_pi: List, states: List, actions: List, gamma: float, theta: float,
                                   S: list, N: list, E: list, p_max: float, c_max: float, initial_state: int,
                                   horizon: int, delay_table: List) -> None:
    """
    Exact one-step online play

    :param base_pi: the base policy
    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param theta: the theta for value iteration stopping critertion
    :param S: the ser of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the maximum routing action value
    :param c_max: the maximum scaling action value
    :param initial_state: the initial state
    :param horizon: the horizon to run the evaluation
    :param delay_table: a table with the predicted response times
    :return: None
    """
    print("Creating reward tensor")
    R = reward_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, c_max=c_max, p_max=p_max,
                      delay_table=delay_table)
    print("Creating transition tensor")
    T = transition_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, p_max=p_max,
                          c_max=c_max)
    print("Evaluating base policy")
    V_base = policy_evaluation(policy=base_pi, num_states=len(states), gamma=gamma, theta=theta, T=T, R=R)
    s = initial_state
    print(f"Starting exact online play, initial state: {s}")
    for i in range(horizon):
        rollout_pi, V_rollout = exact_one_step_rollout_policy(base_pi=base_pi, V_base=V_base, s=s, gamma=gamma,
                                                              num_actions=len(actions), num_states=len(states), T=T,
                                                              R=R, theta=theta)
        print(f"s: {states[s]}, V_base[s]:{V_base[s]}, V_rollout[s]: {V_rollout[s]}, "
              f"base_pi[s]: {actions[np.argmax(base_pi[s])]}, "
              f"rollout_pi[s]: {actions[np.argmax(rollout_pi[s])]}")
        s = transition_function(s=states[int(s)], a=actions[np.argmax(rollout_pi[s])], nodes=N, edges=E, c_max=c_max,
                                p_max=p_max, services=S)
        s = states.index(s)


def run_vi(states: List, actions: List, gamma: float, S: list, N: list, E: list, p_max: float, c_max: float,
           theta: float, delay_table: List) -> None:
    """
    Runs the value iteration algorithm

    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the maximum value of the routing action
    :param c_max: the maximum value of the scaling action
    :param theta: the theta value to measure convergence
    :param delay_table: the table with the predicted response times of different services in different states
    :return: None
    """
    print("Creating reward tensor")
    R = reward_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, c_max=c_max,
                      p_max=p_max, delay_table=delay_table)
    print("Creating transition tensor")
    T = transition_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, p_max=p_max,
                          c_max=c_max)
    print("Starting Value Iteration")
    V = value_iteration(T=T, num_states=len(states), num_actions=len(actions), R=R, theta=theta, gamma=gamma)
    print(f"Value function: {V}")
    pi = greedy_policy(T=T, R=R, gamma=gamma, V=V, num_states=len(states), num_actions=len(actions))
    print(f"Greedy policy: {pi}")
    print("Saving value function to disk")
    with open('V.npy', 'wb') as f:
        np.save(f, np.asarray(V))
    print("Saving policy to disk")
    with open('pi.npy', 'wb') as f:
        np.save(f, np.asarray(pi))


def run_pi(states: List, actions: List, gamma: float, S: list, N: list, E: list, p_max: float, c_max: float,
           num_iterations: int, eval_every: int, eval_batch_size: int,
           running_average_size, eval_initial_state: int, eval_horizon: int, theta: float, delay_table: List) -> None:
    """
    Runs a given base policy

    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the maximum value for the routing action
    :param c_max: the maximum value for the scaling action
    :param num_iterations: the number of iterations to run
    :param eval_every: frequency of evaluation
    :param eval_batch_size: the batch size for evaluation
    :param running_average_size: the running average size for evaluation
    :param eval_initial_state: the initial state for evaluation
    :param eval_horizon: the horizon for the evaluation
    :param theta: the theta to measure convergence of value iteration
    :param delay_table: the table with the predicted response times of different services in different states
    :return: None
    """
    print("Creating reward tensor")
    R = reward_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, c_max=c_max, p_max=p_max,
                      delay_table=delay_table)
    print("Creating transition tensor")
    T = transition_tensor(state_space=states, action_space=actions, services=S, nodes=N, edges=E, p_max=p_max,
                          c_max=c_max)
    pi, V, average_returns, running_average_returns = policy_iteration(
        T=T, N=num_iterations, gamma=gamma, R=R,
        num_states=len(states), num_actions=len(actions), eval_every=eval_every,
        eval_batch_size=eval_batch_size, eval_initial_state=eval_initial_state, eval_horizon=eval_horizon,
        running_average_size=running_average_size, theta=theta)
    print(f"Policy: {pi}")
    print(f"Value function: {V}")
    print("Saving value function to disk")
    with open('V.npy', 'wb') as f:
        np.save(f, np.asarray(V))
    print("Saving policy to disk")
    with open('pi.npy', 'wb') as f:
        np.save(f, np.asarray(pi))


def compute_value_function_of_policy(base_pi: Union[List, MaskablePPO], filename: str, batch_size: int, states: List,
                                     actions: List, gamma: float, S: list, N: list, E: list, p_max: float,
                                     delta_c: float, delta_p: float,
                                     c_max: float, delay_table: List, env=None) -> None:
    """
    Estimates the value function of a given base policy

    :param base_pi: the base policy
    :param filename: the filename to save the value function
    :param batch_size: the batch size for Monte-Carlo estimation
    :param states: the state space
    :param actions: the action space
    :param gamma: the discount factor
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param p_max: the maximum routing action value
    :param delta_c: the discretization step of the scaling action
    :param delta_p: the discretization step of the routing action
    :param c_max: the maximum value of the scaling action
    :param delay_table: the table with the predicted response times
    :param env: the RL environment
    :return: None
    """
    V_base = monte_carlo_policy_evaluation(policy=base_pi, states=states, actions=actions, gamma=gamma,
                                           batch_size=batch_size, N=N, E=E, c_max=c_max, p_max=p_max, S=S,
                                           delay_table=delay_table, delta_c=delta_c,
                                           delta_p=delta_p, env=env)
    V_base = np.array(V_base)
    with open(filename, 'wb') as f:
        np.save(f, np.asarray(V_base))


def compute_delay_table(states: List, S: List, filename: str) -> None:
    """
    Computes the table with the predicted response times of different services in different states
    :param states: the state space
    :param S: the set of services
    :param filename: the filename to save the table
    :return: None
    """
    delay_table = precompute_delays(states=states, S=S)
    delay_table = np.array(delay_table)
    with open(filename, 'wb') as f:
        np.save(f, np.asarray(delay_table))


def greedy_base_policy(states: List, actions: List, S: List, N: List, E: List, c_max: float, p_max: float,
                       delay_table: List, filename: str) -> None:
    """
    Computes and saves the greedy base policy

    :param states: the state space
    :param actions: the action space
    :param S: the set of services
    :param N: the set of nodes
    :param E: the set of edges
    :param c_max: the maximum scaling action value
    :param p_max: the maximum routing action value
    :param delay_table: a table with the predicted response times
    :param filename: the filename to save the policy
    :return: None
    """
    policy = np.zeros([len(states), len(actions)])
    for i, s in enumerate(states):
        a_rews = []
        for j, a in enumerate(actions):
            a_cost = sum(list(map(lambda x: abs(x), a)))
            a_rews.append(reward_function(s=s, a=a, services=S, nodes=N, edges=E, c_max=c_max, p_max=p_max,
                                          states=states, delay_table=delay_table) - a_cost * 0.001)
        a_rews = np.array(a_rews)
        greedy_a = np.argmax(a_rews)
        policy[i][greedy_a] = 1

    with open(filename, 'wb') as f:
        np.save(f, np.asarray(policy))


def optimal_base_policy(states: List, actions: List, filename: str) -> None:
    """
    Computes and stores the optimal base policy (upper bound)

    :param states: the state space
    :param actions: the action space
    :param filename: the filename to save the policy
    :return: None
    """
    policy = np.zeros([len(states), len(actions)])
    for i, s in enumerate(states):
        if s == [2, 2, 1, 1]:
            a = actions.index(tuple([0, 1, -0.2, -0.2]))
            policy[i][a] = 1
        elif s == [2, 3, 0.8, 0.8]:
            a = actions.index(tuple([0, 1, -0.2, -0.2]))
            policy[i][a] = 1
        elif s == [2, 4, 0.6, 0.6]:
            a = actions.index(tuple([1, 1, -0.2, -0.2]))
            policy[i][a] = 1
        elif s == [3, 5, 0.4, 0.4]:
            a = actions.index(tuple([0, 0, -0.2, -0.2]))
            policy[i][a] = 1
        elif s == [3, 5, 0.2, 0.2]:
            a = actions.index(tuple([0, 0, -0.2, -0.2]))
            policy[i][a] = 1
        else:
            a = actions.index(tuple([0, 0, 0, 0]))
            policy[i][a] = 1

    with open(filename, 'wb') as f:
        np.save(f, np.asarray(policy))


def random_base_policy(states: List, actions: List) -> npt.NDArray[Any]:
    """
    Gets the tabular random base policy

    :param states: the state space
    :param actions: the action space
    :return: a table defining the random base policy
    """
    return np.ones([len(states), len(actions)]) / len(actions)

src/online_policy_adaptation_using_rollout/evaluation/evaluate_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout
from sb3_contrib import MaskablePPO
from sb3_contrib.common.maskable.utils import get_action_masks
from stable_baselines3.common.monitor import Monitor
import gym
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_1_env.routing_env import RoutingEnv
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_2_env.routing_env import RoutingEnv
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_3_env.routing_env import RoutingEnv
import time
import numpy as np
import torch

class EvaluateBasePolicy():
    def __init__(self, path_to_base_model, model_name, target_path_to_save_evalaution_results, episode_lenth, env_name,
                 path_to_system_model):
        self.path_to_base_model = path_to_base_model
        self.model_name = model_name
        self.target_path_to_save_evalaution_results = target_path_to_save_evalaution_results
        self.episode_lenth = episode_lenth
        self.env_name = env_name
        self.path_to_system_model = path_to_system_model
        self.Timings = []

    def evaluate_with_base_or_offline(self):
        env = gym.make(self.env_name, path_to_system_model=self.path_to_system_model)
        env = Monitor(env)

        model = MaskablePPO.load(self.path_to_base_model + self.model_name)

        obs = env.reset()
        obs = env.set_state(1, 5)
        evaluation_step = 0

        rewards = []
        infos = []

        while evaluation_step <= self.episode_lenth:
            action_masks = get_action_masks(env)
            action, states_ = model.predict(obs, deterministic=True, action_masks=action_masks)
            print(action, states_)
            obs, reward, dones, info = env.step(action)
            print(obs, reward, dones, info)
            evaluation_step += 1
            rewards.append(reward)
            infos.append(info)
        return rewards, infos

    def n_step_rollout(self, current_state, steps_number, N, env, base_model, reward_so_far):
        print("The reward so far for state", current_state, " is: ", reward_so_far)
        start = time.time()
        if (steps_number == N):
            values = base_model.policy.predict_values(torch.tensor([current_state]))
            value = values.cpu().detach().numpy().squeeze()
            print("We return the reward from the leaf with the value of ", value + reward_so_far)
            print(50 * "=")
            return value + reward_so_far, -1
        else:
            action_masks = get_action_masks(env)
            counter = env.get_episode_counter()
            p1 = current_state[0]
            b1 = current_state[1]
            print("For the current state of ", current_state, " and action mask is: ", action_masks)
            actions = []
            for i in range(2):
                action = []
                for j in range(3):
                    if (action_masks[i][j] == 1):
                        action.append(j)
                actions.append(action)
            aa = []
            for i in range(len(actions[0])):
                for j in range(len(actions[1])):
                    aa.append([actions[0][i], actions[1][j]])
            print(aa)
            rewards = np.zeros((len(aa)))
            for i in range(len(aa)):
                print(10 * ".", "> The action is: ", aa[i])
                obs, reward, dones, info = env.step(aa[i])
                next_reward_so_far = reward_so_far + reward
                print("The reward so far is: ", next_reward_so_far)
                print(10 * ">", "calling rollout again")
                rollout_reward, _ = self.n_step_rollout(obs, steps_number + 1, N, env, base_model, next_reward_so_far)
                rewards[i] = rollout_reward
                s = env.set_state(p1, b1)
                print("The state of environment is now back to ", s)
                env.set_episode_counter(counter)
            selected_action_by_rollout = np.argmax(rewards)
            end = time.time()
            print("**** The rollout time is :", end - start)
            self.Timings.append(end - start)
            return np.max(rewards), aa[selected_action_by_rollout]

    def evaluate_with_rollout(self):
        env = gym.make(self.env_name, path_to_system_model=self.path_to_system_model)
        env = Monitor(env)

        model = MaskablePPO.load(self.path_to_base_model + self.model_name)

        obs = env.reset()
        obs = env.set_state(1, 5)

        rewards = []

        reward_so_far = 0
        current_state = obs
        steps_number = 0
        N = 1
        episode_length = 8
        t = 0

        agg_reward = 0

        actions_from_rollout = []
        infos = []
        steps = []
        while (t < episode_length):
            start = time.time()
            max_reward, selected_action = self.n_step_rollout(current_state, steps_number, N, env, model, reward_so_far)
            next_obs, obtained_reward, dones, info = env.step(selected_action)
            end = time.time()
            steps.append(end - start)
            agg_reward += obtained_reward
            current_state = next_obs
            t += 1
            env.set_episode_counter(t)
            actions_from_rollout.append(selected_action)
            infos.append(info)
            rewards.append(obtained_reward)
        return rewards, infos
src/online_policy_adaptation_using_rollout/evaluation/evaluation.py
foroughsh/online_policy_adaptation_using_rollout
#
import asyncio
import time
import random
import aiohttp
import numpy as np
import sys

########################## Call function with the following order of the input variables################
# python async_load_generator.py service_name type_of_load value_or_name_of_load_pattern IP_port
# python async_load_generator.py city constant 10 x.x.x.x:30037

### Initialization of the parameters
np.random.seed(0)

# The parameters of the load generator
MIN_INTERVAL = 0.05    # Minimum interval between two requests on the real system
CLIENTS_NO = 1
LD_ID = 1

path_to_artifacts = "./artifacts/"

###############################Input arguments##############################
# The arguments specify which service and which patter should be considered for load generation.
############################################################################
args = sys.argv

service_name = args[1]

load_pattern = args[2]

IP_port = args[4]

if service_name == "compute":
    url = "http://"+IP_port+"/compute"
    username = "premium"
    passwd = "1234"
    load_file_name = path_to_artifacts + "load_compute.txt"
    response_file_name = path_to_artifacts + "response_compute.txt"
    LD_ID = 2
    word1 = ", node1,"
    word2 = ", node2,"
elif service_name == "background":
    url = "http://"+IP_port+"/compute"
    username = "fremium"
    passwd = "1234"
    load_file_name = path_to_artifacts + "load_background.txt"
    response_file_name = path_to_artifacts + "response_backgroun.txt"
    LD_ID = 5
    word1 = ", node1,"
    word2 = ", node2,"
elif service_name == "info":
    url = "http://"+IP_port+"/info"
    username = "fremium"
    passwd = "1234"
    load_file_name = path_to_artifacts + "load_info.txt"
    response_file_name = path_to_artifacts + "response_info.txt"
    LD_ID = 1
    word1 = ", node1,"
    word2 = ", node2,"
elif service_name == "book":
    url = "http://"+IP_port+"/books"
    username = "premium"
    passwd = "1234"
    load_file_name = path_to_artifacts + "load_book.txt"
    response_file_name = path_to_artifacts + "response_book.txt"
    LD_ID = 3
    word1 = "(fetch3)"
    word2 = "(fetch2)"
else:
    url = "http://"+IP_port+"/city"
    username = "fremium"
    passwd = "1234"
    load_file_name = path_to_artifacts + "load_city.txt"
    response_file_name = path_to_artifacts + "response_city.txt"
    LD_ID = 3
    word1 = "(fetch3)"
    word2 = "(fetch2)"

login_url = 'http://'+IP_port+'/login'

################################Load generation methods################################
# Here we specify the function for the load pattern. For instance, the load pattern can be a constant load pattern.
# For constant load pattern the intervals are based on the number of requests that we send out per second.
# Moreover, we can read the load pattern from a file, or we can send requests based on the Poisson process modulated with specific function.
# The output of these functions is the time that sender should wait for each request from the time that the load generator is started.
# We can add a delay in the beginning of the time stamps so the load generator start sending the load after the specified delay.
# In the functions with the load specified in a file, we can see in the loop of delay generation, we have two parameters,
# step_size and iteration. step size specifies the time step in our system and iteration means how many of these time steps we like to have same load value.
# For simplification, we could have iteration equal to 1 but in that case we need to repeat the load value in the file. This function can be modified simply.

def constant(load):
    T = 7000
    delays = []
    interval = 1/load
    delay = interval
    while delay<T:
        delays.append(delay)
        delay += interval
    return delays

def from_load_pattern(file_name, step_size, iteration):
    delays = []
    loads = []
    with open(file_name) as file:
        lines = file.readlines()
    for line in lines:
        loads.append(float(line))

    delay = 0
    l = []
    for i in range(1,len(loads)+1):
        interval = 1/loads[i-1]
        print(interval)
        delay += interval
        while delay < iteration * step_size *i:
            delays.append(delay)
            delay += interval
            l.append(interval)
    return delays, l

def random_constant_load():
    delays = []
    loads = [20]
    delay = 0
    l = []
    for i in range(1,4):
        load = loads[random.randint(0,len(loads)-1)]
        interval = 1/load
        print(interval)
        delay += interval
        while delay < 100*i:
            delays.append(delay)
            delay += interval
            l.append(interval)
    return delays, l

def poisson_modulated_by_f(name):

    Time = 7200
    t = 0
    k = 0
    intervals = []

    lam0 = 30
    A = 10
    T = 1200
    S = []

    lam = lam0 + A + 1

    S.append(t)
    t_old = 0

    while (t < Time):
        r = random.random()
        t = t - np.log(r) / lam
        if (t > Time):
            break
        s = random.random()
        # The average of the Poisson  process as a function (Poisson  process modulated by function f- constant or sin)
        lamT = constant(t, T, lam0, A)
        # A * np.sin((2 * np.pi * t) / (T))
        if (s<=(lamT / lam)):
            k = k + 1
            t_old = S[-1]
            S.append(t)
            intervals.append(t - t_old)
    interval_a = np.array(intervals)
    interval_a = interval_a
    print("min: ", min(interval_a))
    print("max: ", max(interval_a))
    print("avg: ", np.mean(interval_a))
    current_time = time.time()
    times = []
    times.append(current_time)
    delays = [interval_a[0]]
    sum_delays = interval_a[0]
    for i in range(len(interval_a)):
        current_time += interval_a[i]
        times.append(current_time)
        sum_delays += interval_a[i]
        delays.append(sum_delays)
    with open(name, 'w') as f:
        for item in times:
            f.write("%s\n" % item)
    f.close()
    return times, delays


if load_pattern == "constant":
    load_value = float(args[3])
    intervals = constant(load_value)
else:
    load_pattern_file_name = args[3]
    intervals,l = from_load_pattern(load_pattern_file_name, 5, 1)

########################## Sending the requests using the asyncio library. ##########################
# Here is a very useful page to understand the concurrency better https://realpython.com/python-concurrency/
# In out application, we can send requests as a specific users too.
# Therefore, here we first send a request to the login page and login as a specific user and then access to the page we desire.
async def do_find_one(wait_time, counter, url, username = "premium", passwd = "1234", load_file_name = "load_compute.txt", response_file_name = "response_compute.txt", LD_ID=2):

    custom_header = {"Accept": "*/*", "Connection": "keep-alive", "Cache-Control": "max-age=0",
                     "Upgrade-Insecure-Requests": "1", "Origin": url,
                     "Content-Type": "application/x-www-form-urlencoded",
                     "Referer": url}
    d = dict()
    d["username"] = username
    d["passwd"] = passwd
    jar = aiohttp.CookieJar(unsafe=True)
    await asyncio.sleep(wait_time)
    start = time.time()
    async with aiohttp.ClientSession(trust_env=True,headers=custom_header,requote_redirect_url=True, cookie_jar=jar) as session:
        sent_text = dict()
        async with session.post(login_url, data=d) as response:
            sent_text["timesentout"] = start
            sent_text["timestamp"] = time.time()
            sent_text["ID"] = counter
            sent_text["LD_ID"] = LD_ID
            with open(load_file_name,"a") as load_file:
                load_file.write(str(sent_text)+"\n")
                load_file.flush()
            load_file.close()
            document = await response.text()
            print(document)
    end = time.time()
    sent_response = dict()
    sent_response["timestamp"] = str(time.time())
    sent_response["response"] = (end - start)
    sent_response["requestID"] = counter
    sent_response["LD_ID"] = LD_ID
    if (word1 in document):
        sent_response["Node"] = 1
    elif (word2 in document):
        sent_response["Node"] = 2
    else:
        sent_response["Node"] = 0
    if ("429 Too Many Requests" in document):
        sent_response["code"] = 429
    else:
        sent_response["code"] = 200
    print("start:{} response time:{}".format(str(start), str(end - start)))
    if (sent_response["response"] > 0):
        with open(response_file_name, "a") as response_file:
            response_file.write(str(sent_response) + "\n")
            response_file.flush()
    response_file.close()
    await session.close()


############################# The main function in which we start the load generator.###################
# In the main function all the coroutines that are responsible to send a request in speccific timestep
# (after specific delay) are generated. The main function ends whenever all coroutines are done their task.
async def main():
    print("This is the IP and port values:",IP_port)
    tasks = []
    print("max waiting time:{}".format(max(intervals)))
    counter = 0
    for j in intervals:
        counter += 1
        current_time = time.time()
        task = asyncio.ensure_future(do_find_one(j, counter,url,username,passwd,load_file_name,response_file_name,LD_ID))
        tasks.append(task)

    print("gather start")
    await asyncio.gather(*tasks, return_exceptions=True)
    print("gather end")

############For python lower or equal to 3.10
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()

############For python higher than 3.10
# if __name__ == '__main__':
#     loop = asyncio.new_event_loop()
#     asyncio.set_event_loop(loop)
#     try:
#         asyncio.run(main())
#     except KeyboardInterrupt:
#         pass

src/online_policy_adaptation_using_rollout/load_generator/async_load_generator.py
foroughsh/online_policy_adaptation_using_rollout
from gym.envs.registration import register

register(
    id='routing-env-v1',
    entry_point='online_policy_adaptation_using_rollout.ppo_base_policy.scenario_1_env.routing_env:RoutingEnv',
    kwargs={'path_to_system_model': None}
)
src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_1_env/__init__.py
foroughsh/online_policy_adaptation_using_rollout
from typing import Tuple
import gym
import numpy as np
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_1_env.routing_middle_ware import RoutingMiddleWare
from typing import List

path = "/"

THERESHOL_D1 = 0.95
THERESHOL_D2 = 0.115

def reward_function(d1, d2, cpu1, cpu2, p1, p2):
    if (d1<=THERESHOL_D1) and (d2<=THERESHOL_D2):
        overal_cpu = (cpu1 + cpu2)/10000
        reward = (1-overal_cpu) * (1-p1) * (1-p2)
    else:
        reward = 0
    return reward


class RoutingEnv(gym.Env):

    def __init__(self, path_to_system_model):
        self.path_to_system_model = path_to_system_model
        self.observation_space = gym.spaces.Box(low=np.array([0, 0, 0, 0]), dtype=np.float32,
                                                high=np.array([1, 1, 5000, 5000]),
                                                shape=(4,))
        self.action_space = gym.spaces.MultiDiscrete([3,3,3,3])

        self.middleware = RoutingMiddleWare(self.path_to_system_model)

        self.l1 = 4
        self.cl1 = 4
        self.l2 = 15
        self.cl2 = 15

        self.d1 = 0
        self.d2 = 0

        self.cpu1 = 2
        self.p1 = 1
        self.cpu2 = 2
        self.p2 = 1

        self.s = [self.p1, self.p2, self.cpu1, self.cpu2]
        self.t = 1
        self.reset()

    def action_masks(self) -> List[bool]:
        if self.p1==0:
            mask1 = [1, 0, 1]
        elif self.p1==1:
            mask1 = [1, 1, 0]
        else:
            mask1 = [1, 1, 1]

        if self.p2==0:
            mask2 = [1, 0, 1]
        elif self.p2==1:
            mask2 = [1, 1, 0]
        else:
            mask2 = [1, 1, 1]

        if self.cpu1==2000:
            mask3 = [1, 0, 1]
        elif self.cpu1==5000:
            mask3 = [1, 1, 0]
        else:
            mask3 = [1, 1, 1]

        if self.cpu2==2000:
            mask4 = [1, 0, 1]
        elif self.cpu2==5000:
            mask4 = [1, 1, 0]
        else:
            mask4 = [1, 1, 1]
        mask = [mask1, mask2, mask3, mask4]
        return mask


    def step(self, a: gym.spaces.MultiDiscrete) -> Tuple[np.ndarray, int, bool, dict]:
        info = {}
        done = False
        next_state, d1, d2 = self.middleware.read_state_from_system(a)
        p1 = next_state[0]
        p2 = next_state[1]
        cpu1 = next_state[2]
        cpu2 = next_state[3]

        cl1 = self.l1
        cl2 = self.l2

        reward = reward_function(d1, d2, cpu1, cpu2, p1, p2)

        info["l1"] = 4
        info["cl1"] = cl1
        info["l2"] = 15
        info["cl2"] = cl2
        info["d1"] = d1
        info["d2"] = d2
        info["cpu1"] = cpu1
        info["cpu2"] = cpu2
        info["p1"] = p1
        info["p2"] = p2

        self.l1 = 4
        self.cl1 = cl1
        self.l2 = 15
        self.cl2 = cl2

        self.d1 = d1
        self.d2 = d2

        self.cpu1 = cpu1
        self.cpu2 = cpu2
        self.p1 = p1
        self.p2 = p2
        self.s = next_state

        info["r"] = reward
        self.t += 1

        return np.array([self.p1, self.p2, self.cpu1, self.cpu2]), reward, done, info

    def reset(self) -> np.ndarray:
        self.t = 1

        self.middleware.reset()

        state, d1, d2 = self.middleware.read_state_from_system([0,0,0,0])

        self.l1 = 4
        self.cl1 = 4
        self.l2 = 15
        self.cl2 = 15
        self.p1 = state[0]
        self.p2 = state[1]
        self.cpu1 = state[2]
        self.cpu2 = state[3]

        self.d1 = d1
        self.d2 = d2

        self.s = state
        return np.array([self.p1, self.p2, self.cpu1, self.cpu2])

    def reset_to_specific_state(self, p1, p2, c1, cp2) -> np.ndarray:
        self.t = 1

        self.middleware.reset_to_specific_state(p1, p2, c1, cp2)

        state, d1, d2 = self.middleware.read_state_from_system([0,0,0,0])

        self.l1 = 4
        self.cl1 = 4
        self.l2 = 15
        self.cl2 = 15
        self.p1 = state[0]
        self.p2 = state[1]
        self.cpu1 = state[2]
        self.cpu2 = state[3]

        self.d1 = d1
        self.d2 = d2

        self.s = state
        return np.array([self.p1, self.p2, self.cpu1, self.cpu2])

    def set_state(self, p1, p2, cpu1, cpu2):
        print("(EVN) Received state is: ", [p1, p2, cpu1, cpu2])
        self.p1 = p1
        self.p2 = p2
        self.s = [p1, p2, cpu1, cpu2]
        print("(EVN) Current state: ", self.s)
        self.middleware.set_state(p1, p2, cpu1, cpu2)
        return self.s

    def get_episode_counter(self):
        return self.t

    def set_episode_counter(self, t):
        self.t = t

# env = RoutingEnv()
# for i in range(0,10):
#     print("Step ----> ", i)
#     action = [0,2,1,0]
#     print("In the loop:",action)
#     s, reward, done, info = env.step(action)
#     print(s, reward, done, info)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_1_env/routing_env.py
foroughsh/online_policy_adaptation_using_rollout
import random
import numpy as np
import joblib

np.set_printoptions(suppress=True)

def action_to_next_config(actions, configurations):
    new_configuration = np.zeros((configurations.shape))
    for i in range(0, 4):
        if i < len(configurations)/2:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 0.2
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 0.2
            else:
                raise ValueError("bug")
            if new_configuration[i]<0:
                new_configuration[i]= 0
            elif new_configuration[i]>1:
                new_configuration[i] = 1
        else:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 1000
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 1000
            else:
                raise ValueError("bug")
            if new_configuration[i]<2000:
                new_configuration[i]= 2000
            elif new_configuration[i]>5000:
                new_configuration[i] = 5000
    return new_configuration


class RoutingMiddleWare():

    def __init__(self, system_model_path):
        self.path = system_model_path
        self.delay_model = joblib.load(self.path)
        self.state_size = 4
        self.state = np.zeros(self.state_size, dtype=float)
        ######actions#######################
        self.cpu1 = 2
        self.cpu2 = 2
        self.p1 = 1
        self.p2 = 1
        self.l1 = 4
        self.l2 = 15
        self.cl1 = self.l1
        self.cl2 = self.l2
        #######initial response times##############
        [[self.d1, self.d2]] = self.delay_model.predict([[self.cpu1, self.cpu2, self.p1, self.p2, self.l1, self.cl1, self.l2, self.cl2]])
        #######state######################
        self.state[0] = self.p1
        self.state[1] = self.p2
        self.state[2] = self.cpu1
        self.state[3] = self.cpu2

        self.state_counter = 0
        self.LD_counter = 0

    def get_state(self):
        return self.state

    def read_state_from_system(self, action):
        configuration = np.array([self.p1, self.p2, self.cpu1, self.cpu2])

        new_configuration = action_to_next_config(action, configuration)
        self.p1 = new_configuration[0]
        self.p2 = new_configuration[1]
        self.cpu1 = new_configuration[2]
        self.cpu2 = new_configuration[3]

        self.cl1 = self.l1
        self.cl2 = self.l2

        [[self.d1, self.d2]] = self.delay_model.predict([[self.cpu1, self.cpu2, self.p1, self.p2, self.l1, self.cl1, self.l2, self.cl2]])

        self.state[0] = self.p1
        self.state[1] = self.p2
        self.state[2] = self.cpu1
        self.state[3] = self.cpu2

        self.state_counter += 1
        self.LD_counter += 1
        return self.state, self.d1, self.d2

    def reset(self):

        self.state_counter = 0
        self.l1 = 4
        self.l2 = 15
        self.cl1 = self.l1
        self.cl2 = self.l2

        self.p1 = random.randint(0, 1) / 5
        self.p2 = random.randint(0, 1) / 5
        self.cpu1 = random.randint(2, 5) * 1000
        self.cpu2 = random.randint(2, 5) * 1000
        self.cl1 = self.l1
        self.cl2 = self.l2

        [[self.d1, self.d2]] = self.delay_model.predict(
            [[self.cpu1, self.cpu2, self.p1, self.p2, self.l1, self.cl1, self.l2, self.cl2]])

        self.state[0] = self.p1
        self.state[1] = self.p2
        self.state[2] = self.cpu1
        self.state[3] = self.cpu2

        return self.state

    def reset_to_specific_state(self, p1, p2, c1, c2):
        self.state_counter = 0
        self.cpu1 = c1
        self.cpu2 = c2
        self.p1 = p1
        self.p2 = p2
        self.l1 = 4
        self.cl1 = self.l1
        self.l2 = 15
        self.cl2 = self.l2

        #######initial response times##############
        [[self.d1, self.d2]] = self.delay_model.predict(
            [[self.cpu1, self.cpu2, self.p1, self.p2, self.l1, self.cl1, self.l2, self.cl2]])
        #######state######################
        self.state[0] = self.p1
        self.state[1] = self.p2
        self.state[2] = self.cpu1
        self.state[3] = self.cpu2

        return self.state

    def set_state(self, p1, p2, cpu1, cpu2):
        print("(SYS) Received state is: ", [p1, p2, cpu1, cpu2])
        self.state = [p1, p2, cpu1, cpu2]
        self.p1 = p1
        self.p2 = p2
        self.cpu1 = cpu1
        self.cpu2 = cpu2
        print("(SYS) Current state: ", self.state)
        return self.state

# test_routing = RoutingMiddleWare()
# # new_cpu, new_routing, new_blocking = 1000, 0.2, 0.2
# # for i in range(8):
# #     new_cpu, new_routing, new_blocking = action_to_next_config(i, new_cpu, new_routing, new_blocking)
# #     print(new_cpu, new_routing, new_blocking)
# for i in range(0,10):
#     print("State before the action: ", test_routing.state)
#     action = [0,1,2,1]
#     state, d1, d2= test_routing.read_state_from_system(action)
#     print(action,state, d1, d2)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_1_env/routing_middle_ware.py
foroughsh/online_policy_adaptation_using_rollout
from gym.envs.registration import register

register(
    id='routing-env-v2',
    entry_point='online_policy_adaptation_using_rollout.ppo_base_policy.scenario_2_env.routing_env:RoutingEnv',
    kwargs={'path_to_system_model': None}
)
src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_2_env/__init__.py
foroughsh/online_policy_adaptation_using_rollout
from typing import Tuple
import gym
import numpy as np
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_2_env.routing_middle_ware import RoutingMiddleWare
from typing import List
path = "/"

THERESHOL_D1 = 0.12

def reward_function(d1, cpu1,p):
    d1 = round(d1,3)
    reward = 0
    if (d1<=THERESHOL_D1):
        reward = (1 - cpu1 / 5)
    else:
        if (d1 > THERESHOL_D1):
            reward = THERESHOL_D1 - d1
    return round(reward,3)

class RoutingEnv(gym.Env):

    def __init__(self, path_to_system_model):
        self.observation_space = gym.spaces.Box(low=np.array([0, 0]), dtype=np.float32,
                                                high=np.array([1, 5]),
                                                shape=(2,))
        self.action_space = gym.spaces.MultiDiscrete([3,3])
        self.path_to_system_model = path_to_system_model
        self.middleware = RoutingMiddleWare(self.path_to_system_model)

        self.l1 = 5
        self.cl1 = self.l1

        self.d1 = 0

        self.cpu1 = 1
        self.p1 = 1

        self.s = [self.p1, self.cpu1]
        self.t = 1
        self.reset()

    def action_masks(self) -> List[bool]:
        if self.p1==0:
            mask1 = [1, 0, 1]
        elif self.p1==1:
            mask1 = [1, 1, 0]
        else:
            mask1 = [1, 1, 1]

        if self.cpu1==1:
            mask2 = [1, 0, 1]
        elif self.cpu1==5:
            mask2 = [1, 1, 0]
        else:
            mask2 = [1, 1, 1]


        mask = [mask1, mask2]
        return mask

    def step(self, a: gym.spaces.MultiDiscrete) -> Tuple[np.ndarray, int, bool, dict]:
        info = {}
        done = False
        next_state, d1 = self.middleware.read_state_from_system(a)
        p1 = next_state[0]
        cpu1 = next_state[1]

        cl1 = self.l1

        reward = reward_function(d1, cpu1,p1)
        # print("state: ", next_state)

        info["l1"] = 5
        info["cl1"] = cl1
        info["d1"] = d1
        info["cpu1"] = cpu1
        info["p1"] = p1

        self.l1 = 5
        self.cl1 = cl1

        self.d1 = d1

        self.cpu1 = cpu1
        self.p1 = p1
        self.s = next_state

        info["r"] = reward
        self.t += 1

        return np.array([self.p1, self.cpu1]), reward, done, info

    def reset(self) -> np.ndarray:
        self.t = 1

        self.middleware.reset()

        state, d1 = self.middleware.read_state_from_system([0,0])

        self.l1 = 5
        self.cl1 = self.l1
        self.p1 = state[0]
        self.cpu1 = state[1]

        self.d1 = d1

        self.s = state
        return np.array([self.p1, self.cpu1])

    def reset_to_specific_state(self, p1, c1) -> np.ndarray:
        self.t = 1

        self.middleware.reset_to_specific_state(p1, c1)

        state, d1 = self.middleware.read_state_from_system([0,0])

        self.l1 = 5
        self.cl1 = self.l1
        self.p1 = state[0]
        self.cpu1 = state[1]

        self.d1 = d1

        self.s = state
        return np.array([self.p1, self.cpu1])

    def set_state(self, p1, cpu1):
        self.p1 = p1
        self.s = [p1, cpu1]
        self.middleware.set_state(p1, cpu1)
        return self.s

    def get_episode_counter(self):
        return self.t

    def set_episode_counter(self, t):
        self.t = t
# env = RoutingEnv()
# for i in range(0,10):
#     print("Step ----> ", i)
#     action = [0,2,1,0]
#     print("In the loop:",action)
#     s, reward, done, info = env.step(action)
#     print(s, reward, done, info)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_2_env/routing_env.py
foroughsh/online_policy_adaptation_using_rollout
import random
import numpy as np
import joblib

np.set_printoptions(suppress=True)

def action_to_next_config(actions, configurations):
    new_configuration = np.zeros((configurations.shape))
    for i in range(0, 2):
        if i < len(configurations)/2:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 0.2
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 0.2
            else:
                raise ValueError("bug")
            if new_configuration[i]<0:
                new_configuration[i]= 0
            elif new_configuration[i]>1:
                new_configuration[i] = 1
        else:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 1
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 1
            else:
                raise ValueError("bug")
            if new_configuration[i]<1:
                new_configuration[i]= 1
            elif new_configuration[i]>5:
                new_configuration[i] = 5
    return new_configuration


class RoutingMiddleWare():

    def __init__(self, path_to_system_model):
        self.path = path_to_system_model
        self.delay_model = joblib.load(self.path)
        self.state_size = 2
        self.state = np.zeros(self.state_size, dtype=float)
        ######actions#######################
        self.cpu1 = 1
        self.p1 = 1
        self.l1 = 5
        self.cl1 = self.l1
        self.b1 = 0
        #######initial response times##############
        #### feature order in our system model: ["c1", "p11", "l1", "cl1"]
        [self.d1] = self.delay_model.predict([[self.cpu1, self.p1, self.l1, self.cl1]])
        #######state######################
        self.state[0] = self.p1
        self.state[1] = self.cpu1

        self.state_counter = 0
        self.LD_counter = 0

    def get_state(self):
        return self.state

    def read_state_from_system(self, action):
        configuration = np.array([self.p1, self.cpu1])

        new_configuration = action_to_next_config(action, configuration)
        self.p1 = new_configuration[0]
        self.cpu1 = new_configuration[1]

        self.cl1 = self.l1

        [self.d1] = self.delay_model.predict([[self.cpu1, self.p1, self.l1, self.cl1]])

        self.state[0] = self.p1
        self.state[1] = self.cpu1

        self.state_counter += 1
        self.LD_counter += 1
        return self.state, self.d1

    def reset(self):

        self.state_counter = 0
        self.l1 = 5
        self.cl1 = self.l1

        self.p1 = random.randint(0, 1) / 5
        self.cpu1 = random.randint(1, 5)
        self.cl1 = self.l1

        [self.d1] = self.delay_model.predict(
            [[self.cpu1, self.p1, self.l1, self.cl1]])

        self.state[0] = self.p1
        self.state[1] = self.cpu1

        return self.state

    def reset_to_specific_state(self, p1, c1):
        self.state_counter = 0
        self.cpu1 = c1
        self.p1 = p1
        self.l1 = 5
        self.cl1 = self.l1

        #######initial response times##############
        [self.d1] = self.delay_model.predict(
            [[self.cpu1, self.p1, self.l1, self.cl1]])
        #######state######################
        self.state[0] = self.p1
        self.state[1] = self.cpu1

        return self.state

    def set_state(self, p1, cpu1):
        self.state = [p1, cpu1]
        self.p1 = p1
        self.cpu1 = cpu1
        return self.state

# test_routing = RoutingMiddleWare()
# for i in range(0,10):
#     print("State before the action: ", test_routing.state)
#     action = [0,1,2,1]
#     state, d1, d2= test_routing.read_state_from_system(action)
#     print(action,state, d1, d2)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_2_env/routing_middle_ware.py
foroughsh/online_policy_adaptation_using_rollout
from gym.envs.registration import register

register(
    id='routing-env-v3',
    entry_point='online_policy_adaptation_using_rollout.ppo_base_policy.scenario_3_env.routing_env:RoutingEnv',
    kwargs={'path_to_system_model': None}
)
src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_3_env/__init__.py
foroughsh/online_policy_adaptation_using_rollout
from typing import Tuple
import gym
import numpy as np
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_3_env.routing_middle_ware import RoutingMiddleWare
from typing import List

path = "/"

THERESHOL_D1 = 0.115
def reward_function(d1, cpu1,p):
    d1 = round(d1,3)
    reward = 0
    if (d1<=THERESHOL_D1):
        reward = (1-cpu1/5) * (1-p)
    else:
        if (d1 > THERESHOL_D1):
            reward = THERESHOL_D1 - d1
    return round(reward,3)

class RoutingEnv(gym.Env):

    def __init__(self, path_to_system_model):
        self.observation_space = gym.spaces.Box(low=np.array([5, 0, 0]), dtype=np.float32,
                                                high=np.array([15, 1, 5]),
                                                shape=(3,))
        self.action_space = gym.spaces.MultiDiscrete([3,3])
        self.path_to_system_model = path_to_system_model
        self.middleware = RoutingMiddleWare(self.path_to_system_model)

        self.l1 = 5
        self.cl1 = self.l1

        self.d1 = 0

        self.cpu1 = 1
        self.p1 = 1

        self.s = [self.l1, self.p1, self.cpu1]
        self.t = 1
        self.reset()
        self.path_to_mask = "/Users/foro/PycharmProjects/NOMS2024_rollouts/sc5/data/Not_in_OP_before.txt"

    def action_masks(self) -> List[bool]:
        if self.p1==0:
            mask1 = [1, 0, 1]
        elif self.p1==1:
            mask1 = [1, 1, 0]
        else:
            mask1 = [1, 1, 1]

        if self.cpu1==1:
            mask2 = [1, 0, 1]
        elif self.cpu1==5:
            mask2 = [1, 1, 0]
        else:
            mask2 = [1, 1, 1]


        mask = [mask1, mask2]
        return mask

    def step(self, a: gym.spaces.MultiDiscrete) -> Tuple[np.ndarray, int, bool, dict]:
        info = {}
        done = False
        next_state, d1 = self.middleware.read_state_from_system(a)
        self.l1 = next_state[0]
        self.p1 = next_state[1]
        self.cpu1 = next_state[2]

        self.cl1 = self.l1

        reward = reward_function(d1, self.cpu1,self.p1)

        info["l1"] = self.l1
        info["cl1"] = self.cl1
        info["d1"] = d1
        info["cpu1"] = self.cpu1
        info["p1"] = self.p1

        self.d1 = d1

        self.s = next_state

        info["r"] = reward
        # if self.t > 10:
        #     done = True
        self.t += 1

        return np.array([self.l1, self.p1, self.cpu1]), reward, done, info

    def reset(self) -> np.ndarray:
        self.t = 1

        self.middleware.reset()

        state, d1 = self.middleware.read_state_from_system([0,0])

        self.l1 = state[0]
        self.cl1 = self.l1
        self.p1 = state[1]
        self.cpu1 = state[2]

        self.d1 = d1

        self.s = state
        return np.array([self.l1, self.p1, self.cpu1])

    def reset_to_specific_state(self, l1, p1, c1) -> np.ndarray:
        self.t = 1

        self.middleware.reset_to_specific_state(l1, p1, c1)

        state, d1 = self.middleware.read_state_from_system([0,0])

        self.l1 = state[0]
        self.cl1 = self.l1
        self.p1 = state[1]
        self.cpu1 = state[2]

        self.d1 = d1

        self.s = state
        return np.array([self.l1, self.p1, self.cpu1])

    def set_state(self, l1, p1, cpu1):
        self.p1 = p1
        self.s = [l1, p1, cpu1]
        self.l1 = l1
        self.cpu1 = cpu1
        self.middleware.set_state(l1, p1, cpu1)
        return self.s

    def get_state(self):
        return self.s

    def get_episode_counter(self):
        return self.t

    def set_episode_counter(self, t):
        self.t = t

# env = RoutingEnv()
# env.set_state(25, 0.4, 1)
# # for i in range(0,10):
# #     print("Step ----> ", i)
# #     action = [0,2]
# #     print("In the loop:",action)
# #     s, reward, done, info = env.step(action)
# #     print(s, reward, done, info)
# print(env.get_state())
# mask = env.action_masks()
# print(mask)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_3_env/routing_env.py
foroughsh/online_policy_adaptation_using_rollout
import random
import numpy as np
import joblib

np.set_printoptions(suppress=True)

def action_to_next_config(actions, configurations):
    new_configuration = np.zeros((configurations.shape))
    for i in range(0, 2):
        if i < len(configurations)/2:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 0.2
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 0.2
            else:
                raise ValueError("bug")
            if new_configuration[i]<0:
                new_configuration[i]= 0
            elif new_configuration[i]>1:
                new_configuration[i] = 1
        else:
            if actions[i] == 0:
                new_configuration[i] = configurations[i]
            elif actions[i] == 1:
                new_configuration[i] = configurations[i] - 1
            elif actions[i] == 2:
                new_configuration[i] = configurations[i] + 1
            else:
                raise ValueError("bug")
            if new_configuration[i]<1:
                new_configuration[i]= 1
            elif new_configuration[i]>5:
                new_configuration[i] = 5
    return new_configuration


class RoutingMiddleWare():

    def __init__(self, path_to_system_model):
        self.path = path_to_system_model
        self.delay_model = joblib.load(self.path)
        self.state_size = 3
        self.state = np.zeros(self.state_size, dtype=float)
        ######actions#######################
        self.cpu1 = 1
        self.p1 = 1
        self.l1 = 5
        self.cl1 = self.l1
        self.b1 = 0
        #######initial response times##############
        [self.d1] = self.delay_model.predict([[self.cpu1, self.p1, self.l1, self.cl1]])
        #######state######################
        self.state[0] = self.l1
        self.state[1] = self.p1
        self.state[2] = self.cpu1

        self.state_counter = 0
        self.LD_counter = 0

    def get_state(self):
        return self.state

    def read_state_from_system(self, action):
        configuration = np.array([self.p1, self.cpu1])

        new_configuration = action_to_next_config(action, configuration)
        self.p1 = new_configuration[0]
        self.cpu1 = new_configuration[1]
        random_number = random.random()
        if (self.LD_counter)%20==0:
            if random_number<0.33:
                self.l1 = 5
            elif random_number<0.66:
                self.l1 = 10
            else:
                self.l1 = 15
        self.cl1 = self.l1
        [self.d1] = self.delay_model.predict([[self.cpu1, self.p1, self.l1, self.cl1]])

        self.state[0] = self.l1
        self.state[1] = self.p1
        self.state[2] = self.cpu1

        self.state_counter += 1
        self.LD_counter += 1
        return self.state, self.d1

    def reset(self):

        self.cl1 = self.l1

        self.p1 = random.randint(0, 1) / 5
        self.cpu1 = random.randint(1, 5)
        self.cl1 = self.l1

        [self.d1] = self.delay_model.predict(
            [[self.cpu1, self.p1, self.l1, self.cl1]])

        self.state[0] = self.l1
        self.state[1] = self.p1
        self.state[2] = self.cpu1

        return self.state

    def reset_to_specific_state(self, l1, p1, c1):
        self.LD_counter = 1
        self.cpu1 = c1
        self.p1 = p1
        self.l1 = l1
        self.cl1 = self.l1

        #######initial response times##############
        [self.d1] = self.delay_model.predict(
            [[self.cpu1, self.p1, self.l1, self.cl1]])
        #######state######################
        self.state[0] = self.l1
        self.state[1] = self.p1
        self.state[2] = self.cpu1

        return self.state

    def set_state(self, l1, p1, cpu1):
        self.state = [l1, p1, cpu1]
        self.p1 = p1
        self.cpu1 = cpu1
        self.l1 = l1
        return self.state

# test_routing = RoutingMiddleWare()
# for i in range(0,20):
#     print("State before the action: ", test_routing.state)
#     action = [random.randint(0,2),random.randint(0,2)]
#     state, d1= test_routing.read_state_from_system(action)
#     print(action,state, d1)

src/online_policy_adaptation_using_rollout/ppo_base_policy/scenario_3_env/routing_middle_ware.py
foroughsh/online_policy_adaptation_using_rollout
import random
import torch
import numpy as np
from stable_baselines3.common.monitor import Monitor
import gym
from stable_baselines3.common.callbacks import BaseCallback
import time
from sb3_contrib import MaskablePPO
from sb3_contrib.common.maskable.utils import get_action_masks

########## To train the agent for different scenarios, just import the environment for that scenario.
# Please note that for when we load the gym env in the main function we need to revise the version numbre based on the scenario number.
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_1_env.routing_env import RoutingEnv
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_2_env.routing_env import RoutingEnv
from online_policy_adaptation_using_rollout.ppo_base_policy.scenario_3_env.routing_env import RoutingEnv
class CustomCallback(BaseCallback):
    """
    A custom callback that derives from ``BaseCallback``.

    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug
    """
    def __init__(self, seed: int, model, env, path_to_models, path_to_results, policy_name="self_routing_", verbose=0, eval_every: int = 200):
        super(CustomCallback, self).__init__(verbose)
        self.iter = 0
        self.eval_every = eval_every
        self.seed = seed
        self.env = env
        self.path_to_models = path_to_models
        self.path_to_results = path_to_results
        self.model = model
        self.policy_name = policy_name

    def _on_training_start(self) -> None:
        """
        This method is called before the first rollout starts.
        """
        pass

    def _on_rollout_start(self) -> None:
        """
        A rollout is the collection of environment interaction
        using the current policy.
        This event is triggered before collecting new samples.
        """
        pass

    def _on_step(self) -> bool:
        """
        This method will be called by the model after each call to `env.step()`.

        For child callback (of an `EventCallback`), this will be called
        when the event is triggered.

        :return: (bool) If the callback returns False, training is aborted early.
        """
        return True

    def _on_rollout_end(self) -> None:
        """
        This event is triggered before updating the policy.
        """
        print(f"Training iteration: {self.iter}")
        if (self.iter+1) % self.eval_every == 0:
            start = time.time()
            state = self.env.reset()
            s = state
            N = 1
            max_horizon = 7
            overall_rewards = []
            state_actions = []
            for i in range(N):
                done = False
                t = 0
                rewards = []
                actions = []
                infos = []
                while not done and t <= max_horizon:
                    print("state: ", s)
                    action_masks = get_action_masks(self.env)
                    print("action_mask:", action_masks)
                    a, _ = self.model.predict(s, deterministic=True, action_masks=action_masks)
                    print("current action for ",s," is ", a)
                    state_actions.append((s, a))
                    s, r, done, info = self.env.step(a)
                    print("next state: ", s)
                    print("reward: ", r)
                    print("info:", info)
                    print(50*"-")
                    rewards.append(round(r,3))
                    actions.append(a)
                    t+= 1
                    infos.append(info)
                overall_rewards.append(np.sum(rewards))
            avg_R = np.mean(overall_rewards)
            with open(self.path_to_results + f"avg_reward_{self.seed}.txt", 'a') as file:
                line = str(avg_R)+ "," + str(rewards) + "," + str(state) + "," + str(s) + "," + str(actions) + "\n"
                file.write(line)
            file.close()
            with open(self.path_to_results + f"infos{self.seed}.txt", 'a') as file:
                line = str(infos) + "\n"
                file.write(line)
            file.close()
            print(f"[EVAL] Training iteration: {self.iter}, Average R:{avg_R},\n list of (load, action): {state_actions}")
            self.env.reset()
            end = time.time()
            print(f"[EVAL] Training time: {end-start}")

        if self.iter % (self.eval_every*1) == 0:
            self.model.save(self.path_to_models + self.policy_name + str(self.seed)+"_"+ str(self.iter))
        self.iter += 1

    def _on_training_end(self) -> None:
        """
        This event is triggered before exiting the `learn()` method.
        """
        pass

class PPOBase():

    def __init__(self,environment_name, path_to_system_model, num_neurons_per_hidden_layer, num_layers, seed, path_to_save_models,
                 path_to_save_results, verbose, steps_between_updates, batch_size, learning_rate, gamma, ent_coef,
                 clip_range, num_training_timesteps, device, policy_name="self_routing"):
        self.environment_name = environment_name
        self.path_to_system_model = path_to_system_model
        self.num_neurons_per_hidden_layer = num_neurons_per_hidden_layer
        self.num_layers = num_layers
        self.seed = seed
        self.path_to_save_models = path_to_save_models
        self.path_to_save_results = path_to_save_results
        self.verbose = verbose
        self.steps_between_updates = steps_between_updates
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.gamma = gamma
        self.ent_coef = ent_coef
        self.clip_range = clip_range
        self.num_training_timesteps = num_training_timesteps
        self.device = device
        self.policy_name = policy_name

    def train_ppo_base(self):
        # For other senarios the version of the environment is different.
        eval_env = gym.make(self.environment_name, path_to_system_model=self.path_to_system_model)
        env = gym.make(self.environment_name, path_to_system_model=self.path_to_system_model)
        env = Monitor(env)
        env.reset()

        action_masks = get_action_masks(env)
        # Hparams
        policy_kwargs = dict(net_arch=[self.num_neurons_per_hidden_layer] * self.num_layers)

        # Set seed for reproducibility
        random.seed(self.seed)
        np.random.seed(self.seed)
        torch.manual_seed(self.seed)

        # Train
        model = MaskablePPO("MlpPolicy", env, verbose=self.verbose,
                    policy_kwargs=policy_kwargs, n_steps=self.steps_between_updates,
                    batch_size=self.batch_size, learning_rate=self.learning_rate, seed=self.seed,
                    device=self.device, gamma=self.gamma, ent_coef=self.ent_coef, clip_range=self.clip_range)

        cb = CustomCallback(seed=self.seed, env=eval_env, model=model, path_to_models=self.path_to_save_models,
                            path_to_results=self.path_to_save_results, eval_every=2, policy_name=self.policy_name)

        start = time.time()
        model.learn(total_timesteps=self.num_training_timesteps, callback=cb)
        end = time.time()

        return end-start

src/online_policy_adaptation_using_rollout/ppo_base_policy/training_ppo_agent.py
foroughsh/online_policy_adaptation_using_rollout
import random
import sys

import joblib
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split

#####################Overview######################################
# In this file we add the function to learn the system model periodically.
# However, we have the plan to add the increamental learning of the system mdoel soon.
# Moreover, our experience shows that with type of system we have, random forest performs best in terms of both accuracy and training time.
# Nevertheless, it can be easily replaced with other regressors.

####################### Input arguments ###########################
args = sys.argv

number_of_training_iteration = args[1]
data_path = args[2]

if len(args)>3:
    target_path = args[3]

##################### Evaluation and training functions ##########################
def test_train_nmae_r2score(test_predicted_values, train_predicted_values, test_set, train_set):
    col_size = train_set.shape[1]
    test_nmaes = []
    train_names = []
    test_r2s = []
    train_r2s = []
    for i in range(col_size):
        diff_test = np.abs(test_predicted_values[:,i] - test_set.iloc[:,i])
        test_nmae = (diff_test.mean()) / test_set.iloc[:,i].mean()
        test_r2score = r2_score(test_set.iloc[:,i], test_predicted_values[:,i])
        diff_train = np.abs(train_predicted_values[:,i] - train_set.iloc[:,i])
        train_nmae = (diff_train.mean()) / train_set.iloc[:,i].mean()
        train_r2score = r2_score(train_set.iloc[:,i], train_predicted_values[:,i])
        test_nmaes.append(test_nmae)
        train_names.append(train_nmae)
        test_r2s.append(test_r2score)
        train_r2s.append(train_r2score)
        print("avg diff: ", diff_test.mean())
    return test_nmaes, train_names, test_r2s, train_r2s

def offline_training(data_path, x_features, y_features, iteration, test_size=0.05, n_estimators=120, target_path="./"):

    '''
    This function is used for the offline learning of the system model using the collected data
    :param data_path: Path the file that data is located.
    :param x_features: The feature set for the design matrix
    :param y_features: The labels for the target
    :return: The accuracy of the model in terms of NMAE and R2
    '''
    data = pd.read_csv(data_path)

    seed = 0
    random.seed(seed)
    np.random.seed(seed)

    # Splitting the data set into train and test sets
    X = data[x_features]
    Y = data[y_features]
    merge_raw = pd.concat([X, Y], axis=1)
    train, test = train_test_split(merge_raw, test_size=test_size)
    train = merge_raw

    # Sort index in train, test
    train = train.sort_index(axis=0)
    test = test.sort_index(axis=0)

    # Preparing the train and test set
    X_train = train[x_features].iloc[1:]
    X_test = test[x_features].iloc[1:]
    Y_train = train[y_features].iloc[1:]
    Y_test = test[y_features].iloc[1:]

    # Training the random forest regressor
    regressor = RandomForestRegressor(n_estimators=n_estimators)
    regressor.fit(X_train.values, Y_train)
    pred_test = regressor.predict(X_test)
    predict_test = (np.array(pred_test)).reshape(pred_test.shape[0], Y_test.shape[1])
    pred_train = regressor.predict(X_train)
    predict_train = (np.array(pred_train)).reshape(pred_train.shape[0], Y_test.shape[1])

    # Evalauting the learned model
    test_nmae, train_nmae, test_r2score, train_r2score = test_train_nmae_r2score(predict_test, predict_train, Y_test,
                                                                                 Y_train)

    # Saving the learned model in the given path
    joblib.dump(regressor, target_path + "system_model" + str(iteration)+".joblib")

    return test_nmae, train_nmae, test_r2score, train_r2score

if __name__ == "__main__":
    x_features = ["c1", "p11", "l1", "cl1"]
    y_features = ["d1"]
    # In these iterations the regressor is updated reading the updated data files.
    for i in range(number_of_training_iteration):
        test_nmae, train_nmae, test_r2score, train_r2score = offline_training(data_path, x_features, y_features, i)
src/online_policy_adaptation_using_rollout/system_model/learning_system_model.py
foroughsh/online_policy_adaptation_using_rollout
"""
Defines the system model for scenario 1
"""
from typing import List, Tuple, Any
import numpy as np
import numpy.typing as npt
import itertools
import joblib

# Environment constants
l_1 = 4.0
c_l_1 = 4.0
l_2 = 15.0
c_l_2 = 15.0
O_1 = 0.95
O_2 = 0.115
print("Loading delay model from disk")
try:
    delay_model = joblib.load("/Users/kimham/Dropbox/NOMS24/SC1/delays_RF_model.joblib")
except:
    try:
        delay_model = joblib.load("/Users/kim/Dropbox/NOMS24/SC1/delays_RF_model.joblib")
    except:
        delay_model = joblib.load("/home/kim/Dropbox/NOMS24/SC1/delays_RF_model.joblib")
print("Delay model loaded successfully")


def services() -> List[int]:
    """
    :return: the set of services
    """
    return [1, 2]


def nodes() -> List[int]:
    """
    :return: the set of nodes
    """
    return [0, 1]


def edges() -> List[Tuple[int, int]]:
    """
    :return: the set of edges
    """
    return [(0, 1)]


def precompute_delays(states: List, S: List) -> npt.NDArray[Any]:
    """
    Function for precomputing the delay table

    :param states: the state space
    :param S: the list of services
    :return: the delay table
    """
    delay_table = np.zeros((len(states), len(S)))
    for i, s in enumerate(states):
        print(f"computing delay for state {i}/{len(states)}")
        [[d_1, d_2]] = delay_model.predict([[s[0] * 1000, s[1] * 1000, s[2], s[3], l_1, c_l_1, l_2, c_l_2]])
        delay_table[i] = [d_1, d_2]
    return delay_table


def state_space(services: List[int], nodes: List[int], edges: List[Tuple[int, int]], delta_c: float,
                delta_p: float, c_max: float, p_max: float) -> List:
    """
    Gets the state space of the model

    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :param delta_c: the discretization step of the scaling action
    :param delta_p: the discretization step of the routing action
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :return: The state space
    """
    routing_weights = list(
        map(lambda x: round(x, 3), list(np.linspace(0, p_max, num=int((p_max / delta_p)) + 1).tolist())))
    cpu_allocations = list(map(lambda x: round(x, 3), list(np.linspace(1, c_max, num=int((c_max / delta_c))).tolist())))
    component_spaces = []
    for _ in nodes:
        component_spaces.append(cpu_allocations)
    for _ in services:
        for _ in edges:
            component_spaces.append(routing_weights)
    state_space = [list(t) for t in list(itertools.product(*component_spaces))]
    return state_space


def action_space(services: List[int], nodes: List[int], edges: List[Tuple[int, int]], delta_c: float, delta_p: float) \
        -> List:
    """
    Computes the action space of the model

    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :param delta_c: the discretization step of the scaling action
    :param delta_p: the discretization step of the routing action
    :return: the action space
    """
    routing_increments = [-delta_p, 0, delta_p]
    cpu_increments = [-delta_c, 0, delta_c]
    component_spaces = []
    for _ in nodes:
        component_spaces.append(cpu_increments)
    for _ in services:
        for _ in edges:
            component_spaces.append(routing_increments)
    action_space = list(itertools.product(*component_spaces))
    return action_space, component_spaces


def transition_tensor(state_space: List, action_space: List, c_max: float, p_max: float, services: List[int],
                      nodes: List[int], edges: List[Tuple[int, int]]) -> List:
    """
    Gets the transition tensor of the system model

    :param state_space: the state space
    :param action_space: the action space
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :return: The transition tensor
    """
    T = []
    for i, s in enumerate(state_space):
        print(f"Generating transition tensor, {i}/{len(state_space) - 1}")
        s_transitions = []
        for s_prime in state_space:
            s_prime_transitions = []
            for a in action_space:
                true_s_prime = transition_function(s=s, a=a, services=services, nodes=nodes, edges=edges, c_max=c_max,
                                                   p_max=p_max)
                if list(s_prime) == list(true_s_prime):
                    s_prime_transitions.append(1)
                else:
                    s_prime_transitions.append(0)
            s_transitions.append(s_prime_transitions)
        T.append(s_transitions)
    return T


def reward_function(s: List, a: List, services: List[int], nodes: List[int], edges: List[Tuple[int, int]],
                    c_max: float, p_max: float, states: List, delay_table=None) -> float:
    """
    The reward function of the model

    :param s: the state
    :param a: the action
    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param states: the state space
    :param delay_table: the table with the precomputed delays for different services in different states
    :return: the reward
    """
    s_prime = transition_function(s=s, a=a, services=services, nodes=nodes, edges=edges, c_max=c_max, p_max=p_max)
    if delay_table is None:
        [[d_1, d_2]] = delay_model.predict([[s_prime[0] * 1000, s_prime[1] * 1000, s_prime[2], s_prime[3],
                                             l_1, c_l_1, l_2, c_l_2]])
    else:
        [d_1, d_2] = delay_table[states.index(s_prime)]
    if (d_1 <= O_1) and (d_2 <= O_2):
        overall_cpu = (s[0] + s[1]) / 10
        reward = (1 - overall_cpu) * (1 - s[2]) * (1 - s[3])
    else:
        reward = 0
        if d_1 > O_1:
            reward -= d_1 - O_1
        if d_2 > O_2:
            reward -= d_2 - O_2
    return reward


def reward_tensor(state_space: List, action_space: List, services: List[int], nodes: List[int],
                  edges: List[Tuple[int, int]], c_max: float, p_max: float, delay_table: List) -> List:
    """
    Computes the reward tensor

    :param state_space: the state space
    :param action_space: the action space
    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :param delay_table: the table with the precomputed delays
    :return: the reward tensor
    """
    R = []
    for s in state_space:
        rewards = []
        for a in action_space:
            rewards.append(reward_function(s=s, a=a, services=services, nodes=nodes, edges=edges, c_max=c_max,
                                           p_max=p_max, states=state_space, delay_table=delay_table))
        R.append(rewards)
    return R


def transition_function(s: List, a: List, services: List[int], nodes: List[int], edges: List[Tuple[int, int]],
                        c_max: float, p_max: float) -> List:
    """
    The transition function of the model

    :param s: the current state
    :param a: the current action
    :param services: the set of services
    :param nodes: the set of nodes
    :param edges: the set of edges
    :param c_max: the maximum value of the scaling action
    :param p_max: the maximum value of the routing action
    :return: the next state
    """
    s_prime = []
    for i in range(len(nodes)):
        s_prime.append(round(max(1.0, min(c_max, s[i] + a[i])), 2))
    for i in range(len(nodes), len(nodes) + len(services) * len(edges)):
        s_prime.append(round(max(0.0, min(p_max, s[i] + a[i])), 2))
    return s_prime


def convert_dp_s_to_env_s(s: List) -> npt.NDArray[Any]:
    """
    Utility function for converting between two state representations

    :param s: convert DP state to RL state
    :return: the RL state representation
    """
    return np.array([s[2], s[3], s[0] * 1000, s[1] * 1000])


def convert_env_a_to_dp_a(a: List, delta_c: float, delta_p: float) -> List:
    """
    Utility function for converting between two action representations

    :param a: the RL action representation
    :param delta_c: the scaling action discretization step
    :param delta_p: the routing action discretization step
    :return: the DP action representation
    """
    converted = [a[2], a[3], a[0], a[1]]

    if converted[0] == 1:
        converted[0] = -delta_c
    elif converted[0] == 2:
        converted[0] = delta_c
    elif converted[0] == 0:
        converted[0] = 0

    if converted[1] == 1:
        converted[1] = -delta_c
    elif converted[1] == 2:
        converted[1] = delta_c
    elif converted[1] == 0:
        converted[1] = 0

    if converted[2] == 1:
        converted[2] = -delta_p
    elif converted[2] == 2:
        converted[2] = delta_p
    elif converted[2] == 0:
        converted[2] = 0

    if converted[3] == 1:
        converted[3] = -delta_p
    elif converted[3] == 2:
        converted[3] = delta_p
    elif converted[3] == 0:
        converted[3] = 0
    return converted

src/online_policy_adaptation_using_rollout/system_model/system_model_scenario_1.py
foroughsh/online_policy_adaptation_using_rollout