I have the following code for a reinforcement learning using proximal policy optimization. It gives the following run time error.
File "C:\Users\Asus\Desktop\RL\agent.py", line 91, in update_policy
loss.backward(retain_graph=True)
File "C:\Users\Asus\anaconda3\lib\site-packages\torch\_tensor.py", line 492, in backward
torch.autograd.backward(
File "C:\Users\Asus\anaconda3\lib\site-packages\torch\autograd\__init__.py", line 251, in backward
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [128, 4096]], which is output 0 of AsStridedBackward0, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
The code is
import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import Categorical
import torch.nn.functional as F
import numpy as np
# Define the neural network for the policy
class PolicyNetwork(nn.Module):
def __init__(self, input_size, output_size):
super(PolicyNetwork, self).__init__()
self.fc = nn.Linear(input_size, 128)
self.fc2 = nn.Linear(128, output_size)
def forward(self, x):
x = torch.relu(self.fc(x))
x = self.fc2(x)
return torch.softmax(x, dim=-1)
# Define the Proximal Policy Optimization agent
class PPOAgent:
def __init__(self, input_size, output_size, lr=1e-3, gamma=0.99, epsilon=0.2, value_coef=0.5, entropy_coef=0.01):
self.policy = PolicyNetwork(input_size, output_size)
self.optimizer = optim.Adam(self.policy.parameters(), lr=lr)
self.gamma = gamma
self.epsilon = epsilon
self.value_coef = value_coef
self.entropy_coef = entropy_coef
def select_action(self, state):
#print("state: ", state)
xstate = torch.from_numpy(state).float()
probs = self.policy(xstate)
m = Categorical(probs)
action = m.sample()
return action.item(), m.log_prob(action)
def update_policy(self, states, actions, rewards, log_probs, values, next_values, dones):
torch.autograd.set_detect_anomaly(True)
returns = self.compute_returns(rewards, dones)
"""
print("values", values, "next_values", next_values)
print("size of values", len(values[0]), "size of next_values", len(next_values))
print("type:",type(values[0]), type(values))
print("returns", returns)
print("size of returns", len(returns))
print("type:", type(returns))
"""
#convert values to tensor
values = torch.tensor(values).float()
advantages = returns - values
print("advantages: ", advantages)
for _ in range(ppo_epochs):
for i in range(len(states)):
state = torch.from_numpy(states[i]).float()
action = torch.tensor(actions[i])
old_log_prob = log_probs[i]
value = values[i]
next_value = next_values[i]
advantage = advantages[i]
return_ = returns[i]
# Compute the new log probability and value
new_probs = self.policy(state)
new_log_prob = torch.log(new_probs[action])
new_value = self.get_value(states[i])
# Compute the surrogate loss
ratio = torch.exp(new_log_prob - old_log_prob)
surr1 = ratio * advantage
surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * advantage
policy_loss = -torch.min(surr1, surr2).mean()
# Compute the value loss
value_loss = F.mse_loss(new_value, return_)
# Compute the entropy loss
entropy_loss = -torch.sum(new_probs * torch.log(new_probs + 1e-10))
# Total loss
loss = policy_loss + self.value_coef * value_loss - self.entropy_coef * entropy_loss
# Optimize the policy
self.optimizer.zero_grad()
print("loss: ", loss)
with torch.autograd.detect_anomaly():
loss.backward(retain_graph=True)
self.optimizer.step()
def compute_returns(self, rewards, dones):
returns = []
R = 0
for reward, done in zip(reversed(rewards), reversed(dones)):
if done:
R = 0
R = reward + self.gamma * R
returns.insert(0, R)
returns = torch.tensor(returns).float()
returns = (returns - returns.mean()) / (returns.std() + 1e-8)
return returns
def get_value(self, state):
print("ggstate: ", state)
state = torch.from_numpy(state).float()
return self.policy(state)
# Set your environment parameters
input_size = 64 # Assuming a flat representation of the chess board as input
output_size = 64*64 # Number of legal moves in your chess environment
# Initialize the PPO agent
agent = PPOAgent(input_size, output_size)
# Training loop
num_episodes = 100
ppo_epochs = 4
"""
for episode in range(num_episodes):
state = 0
done = False
states, actions, rewards, log_probs, values, next_values, dones = [], [], [], [], [], [], []
while not done:
action, log_prob = agent.select_action(state)
next_state, reward, done, _ = env.step(action)
states.append(state)
actions.append(action)
rewards.append(reward)
log_probs.append(log_prob)
values.append(agent.get_value(state))
next_values.append(agent.get_value(next_state))
dones.append(done)
state = next_state
agent.update_policy(states, actions, rewards, log_probs, values, next_values, dones)
"""
I recently added "retain_graph=True" as parameter to backward function. Also I added torch.autograd.set_detect_anomaly(True) but still gives the error.
