I have the following code for a reinforcement learning using proximal policy optimization. It gives the following run time error.

File "C:\Users\Asus\Desktop\RL\agent.py", line 91, in update_policy
  File "C:\Users\Asus\anaconda3\lib\site-packages\torch\_tensor.py", line 492, in backward
  File "C:\Users\Asus\anaconda3\lib\site-packages\torch\autograd\__init__.py", line 251, in backward
    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [128, 4096]], which is output 0 of AsStridedBackward0, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

The code is

import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import Categorical
import torch.nn.functional as F
import numpy as np

# Define the neural network for the policy
class PolicyNetwork(nn.Module):
    def __init__(self, input_size, output_size):
        super(PolicyNetwork, self).__init__()
        self.fc = nn.Linear(input_size, 128)
        self.fc2 = nn.Linear(128, output_size)

    def forward(self, x):
        x = torch.relu(self.fc(x))
        x = self.fc2(x)
        return torch.softmax(x, dim=-1)

# Define the Proximal Policy Optimization agent
class PPOAgent:
    def __init__(self, input_size, output_size, lr=1e-3, gamma=0.99, epsilon=0.2, value_coef=0.5, entropy_coef=0.01):
        self.policy = PolicyNetwork(input_size, output_size)
        self.optimizer = optim.Adam(self.policy.parameters(), lr=lr)
        self.gamma = gamma
        self.epsilon = epsilon
        self.value_coef = value_coef
        self.entropy_coef = entropy_coef

    def select_action(self, state):
        #print("state: ", state)
        xstate = torch.from_numpy(state).float()
        probs = self.policy(xstate)
        m = Categorical(probs)
        action = m.sample()
        return action.item(), m.log_prob(action)

    def update_policy(self, states, actions, rewards, log_probs, values, next_values, dones):
        returns = self.compute_returns(rewards, dones)
        print("values", values, "next_values", next_values)
        print("size of values", len(values[0]), "size of next_values", len(next_values))
        print("type:",type(values[0]), type(values))
        print("returns", returns)
        print("size of returns", len(returns))
        print("type:", type(returns))
        #convert values to tensor
        values = torch.tensor(values).float()
        advantages = returns - values
        print("advantages: ", advantages)

        for _ in range(ppo_epochs):
            for i in range(len(states)):
                state = torch.from_numpy(states[i]).float()
                action = torch.tensor(actions[i])
                old_log_prob = log_probs[i]
                value = values[i]
                next_value = next_values[i]
                advantage = advantages[i]
                return_ = returns[i]

                # Compute the new log probability and value
                new_probs = self.policy(state)
                new_log_prob = torch.log(new_probs[action])
                new_value = self.get_value(states[i])

                # Compute the surrogate loss
                ratio = torch.exp(new_log_prob - old_log_prob)
                surr1 = ratio * advantage
                surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * advantage
                policy_loss = -torch.min(surr1, surr2).mean()

                # Compute the value loss
                value_loss = F.mse_loss(new_value, return_)


                # Compute the entropy loss
                entropy_loss = -torch.sum(new_probs * torch.log(new_probs + 1e-10))

                # Total loss
                loss = policy_loss + self.value_coef * value_loss - self.entropy_coef * entropy_loss

                # Optimize the policy
                print("loss: ", loss)
                with torch.autograd.detect_anomaly():

    def compute_returns(self, rewards, dones):
        returns = []
        R = 0

        for reward, done in zip(reversed(rewards), reversed(dones)):
            if done:
                R = 0
            R = reward + self.gamma * R
            returns.insert(0, R)

        returns = torch.tensor(returns).float()
        returns = (returns - returns.mean()) / (returns.std() + 1e-8)

        return returns

    def get_value(self, state):
        print("ggstate: ", state)
        state = torch.from_numpy(state).float()
        return self.policy(state)

# Set your environment parameters
input_size = 64  # Assuming a flat representation of the chess board as input
output_size = 64*64 # Number of legal moves in your chess environment

# Initialize the PPO agent
agent = PPOAgent(input_size, output_size)

# Training loop
num_episodes = 100
ppo_epochs = 4
for episode in range(num_episodes):
    state = 0
    done = False

    states, actions, rewards, log_probs, values, next_values, dones = [], [], [], [], [], [], []

    while not done:
        action, log_prob = agent.select_action(state)
        next_state, reward, done, _ = env.step(action)


        state = next_state

    agent.update_policy(states, actions, rewards, log_probs, values, next_values, dones)

I recently added "retain_graph=True" as parameter to backward function. Also I added torch.autograd.set_detect_anomaly(True) but still gives the error.



Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.