Note
Click here to download the full example code
Example: Solving an OpenAI Gym environment with CGP.¶
This examples demonstrates how to solve an OpenAI Gym environment (https://gym.openai.com/envs/) with Cartesian genetic programming. We choose the “MountainCarContinuous” environment due to its continuous observation and action spaces.
Preparatory steps: Install the OpenAI Gym package: pip install gym
# The docopt str is added explicitly to ensure compatibility with
# sphinx-gallery.
docopt_str = """
Usage:
example_parametrized_nodes.py [--max-generations=<N>] [--visualize-final-champion]
Options:
-h --help
--max-generations=<N> Maximum number of generations [default: 500]
--visualize-final-champion Create animation of final champion in the mountain car env.
"""
import functools
import warnings
import matplotlib.pyplot as plt
import numpy as np
import sympy
from docopt import docopt
import cgp
try:
import gym
except ImportError:
raise ImportError(
"Failed to import the OpenAI Gym package. Please install it via `pip install gym`."
)
args = docopt(docopt_str)
For more flexibility in the evolved expressions, we define two constants that can be used in the expressions, with values 0.1 and 10.
class ConstantFloatZeroPointOne(cgp.ConstantFloat):
_def_output = "0.1"
class ConstantFloatTen(cgp.ConstantFloat):
_def_output = "10.0"
Then we define the objective function for the evolution. The inner objective accepts a Python callable as input. This callable determines the action taken by the agent upon receiving observations from the environment. The fitness of the given callable on the task is then computed as the cumulative reward over a fixed number of episodes.
def inner_objective(f, seed, n_runs_per_individual, n_total_steps, *, render):
env = gym.make("MountainCarContinuous-v0")
env.seed(seed)
cum_reward_all_episodes = []
for _ in range(n_runs_per_individual):
observation = env.reset()
cum_reward_this_episode = 0
for _ in range(n_total_steps):
if render:
env.render()
continuous_action = f(*observation)
observation, reward, done, _ = env.step([continuous_action])
cum_reward_this_episode += reward
if done:
cum_reward_all_episodes.append(cum_reward_this_episode)
cum_reward_this_episode = 0
observation = env.reset()
env.close()
return cum_reward_all_episodes
The objective then takes an individual, evaluates the inner objective, and updates the fitness of the individual. If the expression of the individual leads to a division by zero, this error is caught and the individual gets a fitness of -infinity assigned.
def objective(ind, seed, n_runs_per_individual, n_total_steps):
if not ind.fitness_is_None():
return ind
f = ind.to_func()
try:
with warnings.catch_warnings(): # ignore warnings due to zero division
warnings.filterwarnings(
"ignore", message="divide by zero encountered in double_scalars"
)
warnings.filterwarnings(
"ignore", message="invalid value encountered in double_scalars"
)
cum_reward_all_episodes = inner_objective(
f, seed, n_runs_per_individual, n_total_steps, render=False
)
# more episodes are better, more reward is better
n_episodes = float(len(cum_reward_all_episodes))
mean_cum_reward = np.mean(cum_reward_all_episodes)
ind.fitness = n_episodes / n_runs_per_individual + mean_cum_reward
except ZeroDivisionError:
ind.fitness = -np.inf
return ind
We then define the main loop for the evolution, which consists of:
parameters for the population, the genome of individuals, and the evolutionary algorithm.
creating a Population instance and instantiating the evolutionary algorithm.
defining a recording callback closure for bookkeeping of the progression of the evolution.
Finally, we call the evolve method to perform the evolutionary search.
def evolve(seed):
objective_params = {"n_runs_per_individual": 3, "n_total_steps": 2000}
genome_params = {
"n_inputs": 2,
"primitives": (
cgp.Add,
cgp.Sub,
cgp.Mul,
cgp.Div,
cgp.ConstantFloat,
ConstantFloatZeroPointOne,
ConstantFloatTen,
),
}
ea_params = {"n_processes": 4}
evolve_params = {
"max_generations": int(args["--max-generations"]),
"termination_fitness": 100.0,
}
pop = cgp.Population(genome_params=genome_params)
ea = cgp.ea.MuPlusLambda(**ea_params)
history = {}
history["expr_champion"] = []
history["fitness_champion"] = []
def recording_callback(pop):
history["expr_champion"].append(pop.champion.to_sympy())
history["fitness_champion"].append(pop.champion.fitness)
obj = functools.partial(
objective,
seed=seed,
n_runs_per_individual=objective_params["n_runs_per_individual"],
n_total_steps=objective_params["n_total_steps"],
)
pop = cgp.evolve(
obj, pop, ea, **evolve_params, print_progress=True, callback=recording_callback
)
return history, pop.champion
For visualization, we define a function to plot the fitness over generations.
def plot_fitness_over_generation_index(history):
width = 6.0
fig = plt.figure(figsize=(width, width / 1.618))
ax = fig.add_axes([0.15, 0.15, 0.8, 0.8])
ax.set_xlabel("Generation index")
ax.set_ylabel("Fitness champion")
ax.plot(history["fitness_champion"])
fig.savefig("example_mountain_car.pdf", dpi=300)
We define a function that checks whether the best expression fulfills the “solving criteria”, i.e., average reward of at least 90.0 over 100 consecutive trials. (https://github.com/openai/gym/wiki/Leaderboard#mountaincarcontinuous-v0)
def evaluate_champion(ind):
env = gym.make("MountainCarContinuous-v0")
env.seed(seed)
observation = env.reset()
f = ind.to_func()
cum_reward_all_episodes = []
cum_reward_this_episode = 0
while len(cum_reward_all_episodes) < 100:
continuous_action = f(*observation)
observation, reward, done, _ = env.step([continuous_action])
cum_reward_this_episode += reward
if done:
cum_reward_all_episodes.append(cum_reward_this_episode)
cum_reward_this_episode = 0
observation = env.reset()
env.close()
cum_reward_average = np.mean(cum_reward_all_episodes)
print(f"average reward over 100 consecutive trials: {cum_reward_average:.05f}", end="")
if cum_reward_average >= 90.0:
print("-> environment solved!")
else:
print()
return cum_reward_all_episodes
Furthermore, we define a function for visualizing the agent’s behaviour for each expression that increase over the currently best performing individual.
def visualize_behaviour_for_evolutionary_jumps(seed, history, only_final_solution=True):
n_runs_per_individual = 1
n_total_steps = 999
max_fitness = -np.inf
for i, fitness in enumerate(history["fitness_champion"]):
if only_final_solution and i != (len(history["fitness_champion"]) - 1):
continue
if fitness > max_fitness:
expr = history["expr_champion"][i]
expr_str = str(expr).replace("x_0", "x").replace("x_1", "dx/dt")
print(f'visualizing behaviour for expression "{expr_str}" (fitness: {fitness:.05f})')
x_0, x_1 = sympy.symbols("x_0, x_1")
f_lambdify = sympy.lambdify([x_0, x_1], expr)
def f(x, v):
return f_lambdify(x, v)
inner_objective(f, seed, n_runs_per_individual, n_total_steps, render=True)
max_fitness = fitness
Finally, we execute the evolution and visualize the results. To animate the behavior of the car for the found expression, uncomment the last line of the example.
if __name__ == "__main__":
seed = 1234
print("starting evolution")
history, champion = evolve(seed)
print("evolution ended")
max_fitness = history["fitness_champion"][-1]
best_expr = history["expr_champion"][-1]
best_expr_str = str(best_expr).replace("x_0", "x").replace("x_1", "dx/dt")
print(f'solution with highest fitness: "{best_expr_str}" (fitness: {max_fitness:.05f})')
plot_fitness_over_generation_index(history)
evaluate_champion(champion)
if args["--visualize-final-champion"]:
visualize_behaviour_for_evolutionary_jumps(seed, history)
![example mountain car](../_images/sphx_glr_example_mountain_car_001.png)
Out:
starting evolution
[2/500] max fitness: 0.9953348693623039[K
[3/500] max fitness: 0.9953348693623039[K
[4/500] max fitness: 0.9953348693623039[K
[5/500] max fitness: 1.0009999999999994[K
[6/500] max fitness: 1.0009999999999994[K
[7/500] max fitness: 1.0009999999999994[K
[8/500] max fitness: 1.0009999999999994[K
[9/500] max fitness: 1.0009999999999994[K
[10/500] max fitness: 1.0009999999999994[K
[11/500] max fitness: 1.0009999999999994[K
[12/500] max fitness: 1.0009999999999994[K
[13/500] max fitness: 1.0009999999999994[K
[14/500] max fitness: 1.0009999999999994[K
[15/500] max fitness: 1.0009999999999994[K
[16/500] max fitness: 1.0009999999999994[K
[17/500] max fitness: 1.0009999999999994[K
[18/500] max fitness: 1.0009999999999994[K
[19/500] max fitness: 1.0009999999999994[K
[20/500] max fitness: 1.0009999999999994[K
[21/500] max fitness: 1.0009999999999994[K
[22/500] max fitness: 1.0009999999999994[K
[23/500] max fitness: 1.0009999999999994[K
[24/500] max fitness: 1.0009999999999994[K
[25/500] max fitness: 1.0009999999999994[K
[26/500] max fitness: 1.0009999999999994[K
[27/500] max fitness: 1.0009999999999994[K
[28/500] max fitness: 1.0009999999999994[K
[29/500] max fitness: 1.0009999999999994[K
[30/500] max fitness: 1.0009999999999994[K
[31/500] max fitness: 1.9999766010447824[K
[32/500] max fitness: 1.9999766010447824[K
[33/500] max fitness: 1.9999766010447824[K
[34/500] max fitness: 1.9999766010447824[K
[35/500] max fitness: 1.9999766010447824[K
[36/500] max fitness: 1.9999766010447824[K
[37/500] max fitness: 1.9999766010447824[K
[38/500] max fitness: 1.9999766010447824[K
[39/500] max fitness: 1.9999766010447824[K
[40/500] max fitness: 1.9999766010447824[K
[41/500] max fitness: 1.9999766010447824[K
[42/500] max fitness: 1.9999766010447824[K
[43/500] max fitness: 1.9999766010447824[K
[44/500] max fitness: 1.9999766010447824[K
[45/500] max fitness: 1.9999766010447824[K
[46/500] max fitness: 1.9999766010447824[K
[47/500] max fitness: 1.9999766010447824[K
[48/500] max fitness: 1.9999766010447824[K
[49/500] max fitness: 1.9999766010447824[K
[50/500] max fitness: 1.9999766010447824[K
[51/500] max fitness: 1.9999807842072206[K
[52/500] max fitness: 1.9999807842072206[K
[53/500] max fitness: 1.9999807842072206[K
[54/500] max fitness: 1.9999807842072206[K
[55/500] max fitness: 1.9999807842072206[K
[56/500] max fitness: 2.0[K
[57/500] max fitness: 2.0[K
[58/500] max fitness: 2.0[K
[59/500] max fitness: 2.0[K
[60/500] max fitness: 2.0[K
[61/500] max fitness: 2.0[K
[62/500] max fitness: 2.0[K
[63/500] max fitness: 2.0[K
[64/500] max fitness: 2.0[K
[65/500] max fitness: 2.0[K
[66/500] max fitness: 2.0[K
[67/500] max fitness: 2.0[K
[68/500] max fitness: 2.0[K
[69/500] max fitness: 2.0[K
[70/500] max fitness: 2.0[K
[71/500] max fitness: 2.0[K
[72/500] max fitness: 2.0[K
[73/500] max fitness: 2.0[K
[74/500] max fitness: 2.0[K
[75/500] max fitness: 2.0[K
[76/500] max fitness: 2.0[K
[77/500] max fitness: 2.0[K
[78/500] max fitness: 2.0[K
[79/500] max fitness: 2.0[K
[80/500] max fitness: 2.0[K
[81/500] max fitness: 2.0[K
[82/500] max fitness: 2.0[K
[83/500] max fitness: 2.0[K
[84/500] max fitness: 2.0[K
[85/500] max fitness: 2.0[K
[86/500] max fitness: 2.0[K
[87/500] max fitness: 2.0[K
[88/500] max fitness: 2.0[K
[89/500] max fitness: 2.0[K
[90/500] max fitness: 2.0[K
[91/500] max fitness: 2.0[K
[92/500] max fitness: 2.0[K
[93/500] max fitness: 2.0[K
[94/500] max fitness: 2.0[K
[95/500] max fitness: 2.0[K
[96/500] max fitness: 2.0[K
[97/500] max fitness: 2.0[K
[98/500] max fitness: 2.0[K
[99/500] max fitness: 2.0[K
[100/500] max fitness: 2.0[K
[101/500] max fitness: 2.0[K
[102/500] max fitness: 2.0[K
[103/500] max fitness: 2.0[K
[104/500] max fitness: 2.0[K
[105/500] max fitness: 2.0[K
[106/500] max fitness: 2.0[K
[107/500] max fitness: 2.0[K
[108/500] max fitness: 2.0[K
[109/500] max fitness: 2.0[K
[110/500] max fitness: 2.0[K
[111/500] max fitness: 2.0[K
[112/500] max fitness: 2.0[K
[113/500] max fitness: 2.0[K
[114/500] max fitness: 2.0[K
[115/500] max fitness: 2.0[K
[116/500] max fitness: 2.0[K
[117/500] max fitness: 2.0[K
[118/500] max fitness: 2.0[K
[119/500] max fitness: 2.0[K
[120/500] max fitness: 2.0[K
[121/500] max fitness: 2.0[K
[122/500] max fitness: 2.0[K
[123/500] max fitness: 2.0[K
[124/500] max fitness: 2.0[K
[125/500] max fitness: 2.0[K
[126/500] max fitness: 2.0[K
[127/500] max fitness: 2.0[K
[128/500] max fitness: 2.0[K
[129/500] max fitness: 2.0[K
[130/500] max fitness: 2.0[K
[131/500] max fitness: 2.0[K
[132/500] max fitness: 2.0[K
[133/500] max fitness: 2.0[K
[134/500] max fitness: 2.0[K
[135/500] max fitness: 2.0[K
[136/500] max fitness: 2.0[K
[137/500] max fitness: 2.0[K
[138/500] max fitness: 2.0[K
[139/500] max fitness: 2.0[K
[140/500] max fitness: 2.0[K
[141/500] max fitness: 2.0[K
[142/500] max fitness: 2.0[K
[143/500] max fitness: 2.0[K
[144/500] max fitness: 2.0[K
[145/500] max fitness: 2.0[K
[146/500] max fitness: 2.0[K
[147/500] max fitness: 2.0[K
[148/500] max fitness: 2.0[K
[149/500] max fitness: 2.0[K
[150/500] max fitness: 2.0[K
[151/500] max fitness: 2.0[K
[152/500] max fitness: 2.0[K
[153/500] max fitness: 2.0[K
[154/500] max fitness: 2.0[K
[155/500] max fitness: 2.0[K
[156/500] max fitness: 2.0[K
[157/500] max fitness: 2.0[K
[158/500] max fitness: 2.0[K
[159/500] max fitness: 2.0[K
[160/500] max fitness: 2.0[K
[161/500] max fitness: 2.0[K
[162/500] max fitness: 2.0[K
[163/500] max fitness: 2.0[K
[164/500] max fitness: 2.0[K
[165/500] max fitness: 2.0[K
[166/500] max fitness: 2.0[K
[167/500] max fitness: 2.0[K
[168/500] max fitness: 2.0[K
[169/500] max fitness: 2.0[K
[170/500] max fitness: 2.0[K
[171/500] max fitness: 2.0[K
[172/500] max fitness: 2.0[K
[173/500] max fitness: 2.0[K
[174/500] max fitness: 2.0[K
[175/500] max fitness: 2.0[K
[176/500] max fitness: 2.0[K
[177/500] max fitness: 2.0[K
[178/500] max fitness: 2.0[K
[179/500] max fitness: 2.0[K
[180/500] max fitness: 2.0[K
[181/500] max fitness: 2.0[K
[182/500] max fitness: 2.0[K
[183/500] max fitness: 2.0[K
[184/500] max fitness: 2.0[K
[185/500] max fitness: 2.0[K
[186/500] max fitness: 2.0[K
[187/500] max fitness: 2.0[K
[188/500] max fitness: 2.0[K
[189/500] max fitness: 2.0[K
[190/500] max fitness: 2.0[K
[191/500] max fitness: 2.0[K
[192/500] max fitness: 2.0[K
[193/500] max fitness: 2.0[K
[194/500] max fitness: 2.0[K
[195/500] max fitness: 2.0[K
[196/500] max fitness: 2.0[K
[197/500] max fitness: 2.0[K
[198/500] max fitness: 2.0[K
[199/500] max fitness: 2.0[K
[200/500] max fitness: 2.0[K
[201/500] max fitness: 2.0[K
[202/500] max fitness: 2.0[K
[203/500] max fitness: 2.0[K
[204/500] max fitness: 2.0[K
[205/500] max fitness: 2.0[K
[206/500] max fitness: 2.0[K
[207/500] max fitness: 2.0[K
[208/500] max fitness: 2.0[K
[209/500] max fitness: 2.0[K
[210/500] max fitness: 2.0[K
[211/500] max fitness: 2.0[K
[212/500] max fitness: 2.0[K
[213/500] max fitness: 2.0[K
[214/500] max fitness: 2.0[K
[215/500] max fitness: 2.0[K
[216/500] max fitness: 2.0[K
[217/500] max fitness: 2.0[K
[218/500] max fitness: 2.0[K
[219/500] max fitness: 2.0[K
[220/500] max fitness: 2.0[K
[221/500] max fitness: 2.0[K
[222/500] max fitness: 2.0[K
[223/500] max fitness: 2.0[K
[224/500] max fitness: 2.0[K
[225/500] max fitness: 2.0[K
[226/500] max fitness: 2.0[K
[227/500] max fitness: 2.0[K
[228/500] max fitness: 2.0[K
[229/500] max fitness: 2.0[K
[230/500] max fitness: 2.0[K
[231/500] max fitness: 2.0[K
[232/500] max fitness: 2.0[K
[233/500] max fitness: 2.0[K
[234/500] max fitness: 2.0[K
[235/500] max fitness: 2.0[K
[236/500] max fitness: 2.0[K
[237/500] max fitness: 2.0[K
[238/500] max fitness: 2.0[K
[239/500] max fitness: 2.0[K
[240/500] max fitness: 2.0[K
[241/500] max fitness: 2.0[K
[242/500] max fitness: 2.0[K
[243/500] max fitness: 2.0[K
[244/500] max fitness: 2.0[K
[245/500] max fitness: 2.0[K
[246/500] max fitness: 2.0[K
[247/500] max fitness: 2.0[K
[248/500] max fitness: 2.0[K
[249/500] max fitness: 2.0[K
[250/500] max fitness: 2.0[K
[251/500] max fitness: 2.0[K
[252/500] max fitness: 2.0[K
[253/500] max fitness: 2.0[K
[254/500] max fitness: 2.0[K
[255/500] max fitness: 2.0[K
[256/500] max fitness: 2.0[K
[257/500] max fitness: 2.0[K
[258/500] max fitness: 2.0[K
[259/500] max fitness: 2.0[K
[260/500] max fitness: 100.9023420338344[K
evolution ended
solution with highest fitness: "10.0*dx/dt*(-x + (x - dx/dt)*(x + dx/dt))/(0.1*dx/dt*(x - dx/dt) + 1.0)" (fitness: 100.90234)
<string>:5: RuntimeWarning: divide by zero encountered in double_scalars
average reward over 100 consecutive trials: 97.94716-> environment solved!
Total running time of the script: ( 0 minutes 49.120 seconds)