(lyapunov, perturbations=None, limits=None, positive=False,
                    num_samples=None, actions=None)
Compute a safe state-action pair for sampling.

    This function returns the most uncertain state-action pair close to the
    current policy (as a result of the perturbations) that is safe (maps
    back into the region of attraction).

    Parameters
    ----------
    lyapunov : instance of `Lyapunov'
        A Lyapunov instance with an up-to-date safe set.
    perturbations : ndarray
        An array that, on each row, has a perturbation that is added to the
        baseline policy in `lyapunov.policy`.
    limits : ndarray, optional
        The actuator limits. Of the form [(u_1_min, u_1_max), (u_2_min,..)...].
        If provided, state-action pairs are clipped to ensure the limits.
    positive : bool
        Whether the Lyapunov function is positive-definite (radially
        increasing). If not, additional checks are carried out to ensure
        safety of samples.
    num_samples : int, optional
        Number of samples to select (uniformly at random) from the safe
        states within lyapunov.discretization as testing points.
    actions : ndarray
        A list of actions to evaluate for each state. Ignored if perturbations
        is not None.

    Returns
    -------
    state-action : ndarray
        A row-vector that contains a safe state-action pair that is
        promising for obtaining future observations.
    var : float
        The uncertainty remaining at this state.
Compute a safe state-action pair for sampling.
Compute a safe state - action pair for sampling .
def get_safe_sample(lyapunov, perturbations=None, limits=None, positive=False,
                    num_samples=None, actions=None):
    """Compute a safe state-action pair for sampling.

    This function returns the most uncertain state-action pair close to the
    current policy (as a result of the perturbations) that is safe (maps
    back into the region of attraction).

    Parameters
    ----------
    lyapunov : instance of `Lyapunov'
        A Lyapunov instance with an up-to-date safe set.
    perturbations : ndarray
        An array that, on each row, has a perturbation that is added to the
        baseline policy in `lyapunov.policy`.
    limits : ndarray, optional
        The actuator limits. Of the form [(u_1_min, u_1_max), (u_2_min,..)...].
        If provided, state-action pairs are clipped to ensure the limits.
    positive : bool
        Whether the Lyapunov function is positive-definite (radially
        increasing). If not, additional checks are carried out to ensure
        safety of samples.
    num_samples : int, optional
        Number of samples to select (uniformly at random) from the safe
        states within lyapunov.discretization as testing points.
    actions : ndarray
        A list of actions to evaluate for each state. Ignored if perturbations
        is not None.

    Returns
    -------
    state-action : ndarray
        A row-vector that contains a safe state-action pair that is
        promising for obtaining future observations.
    var : float
        The uncertainty remaining at this state.

    """
    state_dim = lyapunov.discretization.ndim
    if perturbations is None:
        action_dim = actions.shape[1]
    else:
        action_dim = perturbations.shape[1]
    action_limits = limits

    storage = get_storage(_STORAGE, index=lyapunov)

    if storage is None:
        tf_states = tf.placeholder(config.dtype, shape=[None, state_dim])
        tf_actions = lyapunov.policy(tf_states)

        # Placeholder for state-actions to evaluate
        tf_state_actions = tf.placeholder(config.dtype,
                                          shape=[None, state_dim + action_dim])

        # Account for deviations of the next value due to uncertainty
        tf_mean, tf_std = lyapunov.dynamics(tf_state_actions)
        tf_bound = tf.reduce_sum(tf_std, axis=1, keepdims=True)
        tf_lv = lyapunov.lipschitz_lyapunov(tf_mean)
        tf_error = tf.reduce_sum(tf_lv * tf_std, axis=1, keepdims=True)
        tf_mean_future_values = lyapunov.lyapunov_function(tf_mean)

        # Check whether the value is below c_max
        tf_future_values = tf_mean_future_values + tf_error
        tf_maps_inside = tf.less(tf_future_values, lyapunov.c_max,
                                 name='maps_inside_levelset')

        # Put everything into storage
        storage = [('tf_states', tf_states),
                   ('tf_actions', tf_actions),
                   ('tf_state_actions', tf_state_actions),
                   ('tf_mean', tf_mean),
                   ('tf_bound', tf_bound),
                   ('tf_maps_inside', tf_maps_inside)]
        set_storage(_STORAGE, storage, index=lyapunov)
    else:
        (tf_states, tf_actions, tf_state_actions, tf_mean, tf_bound,
         tf_maps_inside) = storage.values()

    # Subsample from all safe states within the discretization
    safe_idx = np.where(lyapunov.safe_set)
    safe_states = lyapunov.discretization.index_to_state(safe_idx)
    if num_samples is not None and len(safe_states) > num_samples:
        idx = np.random.choice(len(safe_states), num_samples, replace=True)
        safe_states = safe_states[idx]

    # Update the feed_dict accordingly
    feed_dict = lyapunov.feed_dict
    feed_dict[tf_states] = safe_states

    if perturbations is None:
        # Generate all state-action pairs
        arrays = [arr.ravel() for arr in np.meshgrid(safe_states,
                                                     actions,
                                                     indexing='ij')]
        state_actions = np.column_stack(arrays)
    else:
        # Generate state-action pairs around the current policy
        safe_actions = tf_actions.eval(feed_dict=feed_dict)
        state_actions = perturb_actions(safe_states,
                                        safe_actions,
                                        perturbations=perturbations,
                                        limits=action_limits)

    # Update feed value
    lyapunov.feed_dict[tf_state_actions] = state_actions

    # Evaluate the safety of the proposed state-action pairs
    session = tf.get_default_session()
    (maps_inside, mean, bound) = session.run([tf_maps_inside, tf_mean,
                                              tf_bound],
                                             feed_dict=lyapunov.feed_dict)
    maps_inside = maps_inside.squeeze(axis=1)

    # Check whether states map back to the safe set in expectation
    if not positive:
        next_state_index = lyapunov.discretization.state_to_index(mean)
        safe_in_expectation = lyapunov.safe_set[next_state_index]
        maps_inside &= safe_in_expectation

    # Return only state-actions pairs that are safe
    bound_safe = bound[maps_inside]
    if len(bound_safe) == 0:
        # Nothing is safe, so revert to backup policy
        msg = "No safe state-action pairs found! Using backup policy ..."
        warnings.warn(msg, RuntimeWarning)
        zero_perturbation = np.array([[0.]], dtype=config.np_dtype)
        state_actions = perturb_actions(safe_states,
                                        safe_actions,
                                        perturbations=zero_perturbation,
                                        limits=action_limits)
        lyapunov.feed_dict[tf_state_actions] = state_actions
        bound = session.run(tf_bound, feed_dict=lyapunov.feed_dict)
        max_id = np.argmax(bound)
        max_bound = bound[max_id].squeeze()
        return state_actions[[max_id]], max_bound
    else:
        max_id = np.argmax(bound_safe)
        max_bound = bound_safe[max_id].squeeze()
        return state_actions[maps_inside, :][[max_id]], max_bound
def get_safe_sample ( lyapunov , perturbations = None , limits = None , positive = False , num_samples = None , actions = None ) : state_dim = lyapunov . discretization . ndim if perturbations is None : action_dim = actions . shape [ 1 ] else : action_dim = perturbations . shape [ 1 ] action_limits = limits storage = get_storage ( _STORAGE , index = lyapunov ) if storage is None : tf_states = tf . placeholder ( config . dtype , shape = [ None , state_dim ] ) tf_actions = lyapunov . policy ( tf_states ) # Placeholder for state-actions to evaluate tf_state_actions = tf . placeholder ( config . dtype , shape = [ None , state_dim + action_dim ] ) # Account for deviations of the next value due to uncertainty tf_mean , tf_std = lyapunov . dynamics ( tf_state_actions ) tf_bound = tf . reduce_sum ( tf_std , axis = 1 , keepdims = True ) tf_lv = lyapunov . lipschitz_lyapunov ( tf_mean ) tf_error = tf . reduce_sum ( tf_lv * tf_std , axis = 1 , keepdims = True ) tf_mean_future_values = lyapunov . lyapunov_function ( tf_mean ) # Check whether the value is below c_max tf_future_values = tf_mean_future_values + tf_error tf_maps_inside = tf . less ( tf_future_values , lyapunov . c_max , name = 'maps_inside_levelset' ) # Put everything into storage storage = [ ( 'tf_states' , tf_states ) , ( 'tf_actions' , tf_actions ) , ( 'tf_state_actions' , tf_state_actions ) , ( 'tf_mean' , tf_mean ) , ( 'tf_bound' , tf_bound ) , ( 'tf_maps_inside' , tf_maps_inside ) ] set_storage ( _STORAGE , storage , index = lyapunov ) else : ( tf_states , tf_actions , tf_state_actions , tf_mean , tf_bound , tf_maps_inside ) = storage . values ( ) # Subsample from all safe states within the discretization safe_idx = np . where ( lyapunov . safe_set ) safe_states = lyapunov . discretization . index_to_state ( safe_idx ) if num_samples is not None and len ( safe_states ) > num_samples : idx = np . random . choice ( len ( safe_states ) , num_samples , replace = True ) safe_states = safe_states [ idx ] # Update the feed_dict accordingly feed_dict = lyapunov . feed_dict feed_dict [ tf_states ] = safe_states if perturbations is None : # Generate all state-action pairs arrays = [ arr . ravel ( ) for arr in np . meshgrid ( safe_states , actions , indexing = 'ij' ) ] state_actions = np . column_stack ( arrays ) else : # Generate state-action pairs around the current policy safe_actions = tf_actions . eval ( feed_dict = feed_dict ) state_actions = perturb_actions ( safe_states , safe_actions , perturbations = perturbations , limits = action_limits ) # Update feed value lyapunov . feed_dict [ tf_state_actions ] = state_actions # Evaluate the safety of the proposed state-action pairs session = tf . get_default_session ( ) ( maps_inside , mean , bound ) = session . run ( [ tf_maps_inside , tf_mean , tf_bound ] , feed_dict = lyapunov . feed_dict ) maps_inside = maps_inside . squeeze ( axis = 1 ) # Check whether states map back to the safe set in expectation if not positive : next_state_index = lyapunov . discretization . state_to_index ( mean ) safe_in_expectation = lyapunov . safe_set [ next_state_index ] maps_inside &= safe_in_expectation # Return only state-actions pairs that are safe bound_safe = bound [ maps_inside ] if len ( bound_safe ) == 0 : # Nothing is safe, so revert to backup policy msg = "No safe state-action pairs found! Using backup policy ..." warnings . warn ( msg , RuntimeWarning ) zero_perturbation = np . array ( [ [ 0. ] ] , dtype = config . np_dtype ) state_actions = perturb_actions ( safe_states , safe_actions , perturbations = zero_perturbation , limits = action_limits ) lyapunov . feed_dict [ tf_state_actions ] = state_actions bound = session . run ( tf_bound , feed_dict = lyapunov . feed_dict ) max_id = np . argmax ( bound ) max_bound = bound [ max_id ] . squeeze ( ) return state_actions [ [ max_id ] ] , max_bound else : max_id = np . argmax ( bound_safe ) max_bound = bound_safe [ max_id ] . squeeze ( ) return state_actions [ maps_inside , : ] [ [ max_id ] ] , max_bound