update save and load

rl/ac.py

@@ -93,14 +93,14 @@ class ACAgent(Agent):
     def diagnostics(self):
         return {'value': self.last_state_value}
 
-    def save(self, path):
-        torch.save(self.model.state_dict(), path)
+    def save(self, file_path):
+        torch.save(self.model.state_dict(), file_path)
 
     @classmethod
-    def load(cls, path, encoder):
+    def load(cls, file_path, encoder):
         device = 'cuda' if torch.cuda.is_available() else 'cpu'
         model = AlphaGoModel(encoder.get_input_shape(), is_policy_net=True).to(device)
-        model.load_state_dict(torch.load(path))
+        model.load_state_dict(torch.load(file_path))
         return cls(model, encoder)
 
 # def load_ac_agent(path, encoder):

rl/ac_pass.py

@@ -0,0 +1,111 @@
+import torch
+import numpy as np
+
+from tugo import encoders
+from tugo.game_logic import goboard
+from tugo.agents import Agent
+
+
+class ACAgent(Agent):
+    def __init__(self, model, encoder):
+        Agent.__init__(self)
+        self.model = model
+        self.encoder = encoder
+        self.collector = None
+        self.temperature = 1.0
+
+        self.last_state_value = 0
+
+    def set_temperature(self, temperature):
+        self.temperature = temperature
+
+    def set_collector(self, collector):
+        self.collector = collector
+
+    def select_move(self, game_state):
+        num_moves = self.encoder.board_width * self.encoder.board_height
+
+        board_tensor = self.encoder.encode(game_state)
+        x = torch.from_numpy(np.array([board_tensor])).float().to(self.model.device)
+
+        actions, values = self.model(x)
+        move_probs = actions[0].detach().cpu().numpy()
+        estimated_value = values[0][0].item()
+        self.last_state_value = float(estimated_value)
+
+        # Prevent move probs from getting stuck at 0 or 1.
+        move_probs = np.power(move_probs, 1.0 / self.temperature)
+        move_probs = move_probs / np.sum(move_probs)
+        eps = 1e-6
+        move_probs = np.clip(move_probs, eps, 1 - eps)
+        # Re-normalize to get another probability distribution.
+        move_probs = move_probs / np.sum(move_probs)
+
+        # Turn the probabilities into a ranked list of moves.
+        candidates = np.arange(num_moves)
+        ranked_moves = np.random.choice(
+            candidates, num_moves, replace=False, p=move_probs)
+        for point_idx in ranked_moves:
+            point = self.encoder.decode_point_index(point_idx)
+            true_move = goboard.Move.play(point)
+            if not game_state.is_valid_move(true_move):
+                true_move = goboard.Move.pass_turn()
+            if self.collector is not None:
+                self.collector.record_decision(
+                    state=board_tensor,
+                    action=point_idx,
+                    estimated_value=estimated_value
+                )
+            return true_move
+        # No legal, non-self-destructive moves less.
+        return goboard.Move.pass_turn()
+
+    def train(self, experience, lr=0.1, batch_size=128):
+        optimizer = torch.optim.SGD(self.model.parameters(), lr=lr, clipvalue=0.2)
+        criterion = torch.nn.CrossEntropyLoss()
+
+        n = experience.states.shape[0]
+        num_moves = self.encoder.num_points()
+        policy_target = np.zeros((n, num_moves))
+        value_target = np.zeros((n,))
+        for i in range(n):
+            action = experience.actions[i]
+            reward = experience.rewards[i]
+            policy_target[i][action] = experience.advantages[i]
+            value_target[i] = reward
+
+        self.model.train()
+        optimizer.zero_grad()
+        policy_target_t = torch.from_numpy(policy_target).float().to(self.model.device)
+        value_target_t = torch.from_numpy(value_target).float().to(self.model.device)
+        states_t = torch.from_numpy(experience.states).float().to(self.model.device)
+        actions_pred, values_pred = self.model(states_t)
+        loss1 = criterion(actions_pred, policy_target_t)
+        loss2 = criterion(values_pred, value_target_t)
+        loss = loss1 + loss2
+        loss.backward()
+        optimizer.step()
+
+    def serialize(self, file_path):
+        torch.save({
+            'model_state_dict': self.model.state_dict(),
+            'encoder_name': self.encoder.name(),
+            'encoder_board_width': self.encoder.board_width,
+            'encoder_board_height': self.encoder.board_height,
+        }, file_path)
+
+    def diagnostics(self):
+        return {'value': self.last_state_value}
+
+
+def load_passing_ac_agent(file_path):
+    checkpoint = torch.load(file_path)
+    model = TheModelClass()  # TheModelClass should be replaced with your model class
+    model.load_state_dict(checkpoint['model_state_dict'])
+    encoder_name = checkpoint['encoder_name']
+    board_width = checkpoint['encoder_board_width']
+    board_height = checkpoint['encoder_board_height']
+    encoder = encoders.get_encoder_by_name(
+        encoder_name,
+        (board_width, board_height))
+    return ACAgent(model, encoder)

rl/experience.py

@@ -43,13 +43,13 @@ class ExperienceBuffer:
         self.rewards = rewards
         self.advantages = advantages
 
-    def save(self, filename):
-        with open(filename, 'wb') as f:
+    def save(self, file_path):
+        with open(file_path, 'wb') as f:
             pickle.dump((self.states, self.actions, self.rewards, self.advantages), f)
 
     @classmethod
-    def load(cls, filename):
-        with open(filename, 'rb') as f:
+    def load(cls, file_path):
+        with open(file_path, 'rb') as f:
             states, actions, rewards, advantages = pickle.load(f)
         return cls(
             states=states,

rl/q.py

@@ -113,7 +113,7 @@ class QAgent(Agent):
     def diagnostics(self):
         return {'value': self.last_move_value}
 
-    def save(self, filename):
+    def save(self, file_path):
         torch.save({
             'model': self.model.state_dict(),
             'encoder': {
@@ -121,11 +121,11 @@ class QAgent(Agent):
                 'board_width': self.encoder.board_width,
                 'board_height': self.encoder.board_height,
             }
-        }, filename)
+        }, file_path)
 
     @classmethod
-    def load(cls, filename):
-        checkpoint = torch.load(filename)
+    def load(cls, file_path):
+        checkpoint = torch.load(file_path)
 
         # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder
         model = Model()  # The Model class should be defined appropriately
@@ -139,17 +139,17 @@ class QAgent(Agent):
         return cls(model, encoder)
 
 
-def load_q_agent(filename):
-    # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。
-    checkpoint = torch.load(filename)
-
-    # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder
-    model = Model()  # The Model class should be defined appropriately
-    model.load_state_dict(checkpoint['model'])
-
-    encoder_info = checkpoint['encoder']
-    encoder = encoders.get_encoder_by_name(
-        encoder_info['name'],
-        (encoder_info['board_width'], encoder_info['board_height']))
-
-    return QAgent(model, encoder)
+# def load_q_agent(file_path):
+#     # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。
+#     checkpoint = torch.load(file_path)
+#
+#     # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder
+#     model = Model()  # The Model class should be defined appropriately
+#     model.load_state_dict(checkpoint['model'])
+#
+#     encoder_info = checkpoint['encoder']
+#     encoder = encoders.get_encoder_by_name(
+#         encoder_info['name'],
+#         (encoder_info['board_width'], encoder_info['board_height']))
+#
+#     return QAgent(model, encoder)

rl/simulate.py

@@ -0,0 +1,58 @@
+from tugo import rl
+from tugo.game_logic import scoring
+from tugo.game_logic import goboard_fast as goboard
+from tugo.game_logic.gotypes import Player
+
+from collections import namedtuple
+
+
+class GameRecord(namedtuple('GameRecord', 'moves winner margin')):
+    pass
+
+
+def simulate_game(black_player, white_player):
+    moves = []
+    game = goboard.GameState.new_game(19)
+    agents = {
+        Player.black: black_player,
+        Player.white: white_player,
+    }
+    while not game.is_over():
+        next_move = agents[game.next_player].select_move(game)
+        moves.append(next_move)
+        game = game.apply_move(next_move)
+
+    game_result = scoring.compute_game_result(game)
+    # print(game_result)
+
+    return GameRecord(
+        moves=moves,
+        winner=game_result.winner,
+        margin=game_result.winning_margin,
+    )
+
+
+def experience_simulation(num_games, agent1, agent2):
+    collectors = [rl.ExperienceCollector() for _ in range(2)]
+    agents = [agent1, agent2]
+
+    color1 = Player.black
+    for i in range(num_games):
+        for collector, agent in zip(collectors, agents):
+            collector.begin_episode()
+            agent.set_collector(collector)
+
+        if color1 == Player.black:
+            black_player, white_player = agent1, agent2
+        else:
+            white_player, black_player = agent2, agent1
+        game_record = simulate_game(black_player, white_player)
+
+        winner_reward = 1 if game_record.winner == color1 else -1
+        for collector in collectors:
+            collector.complete_episode(reward=winner_reward)
+            winner_reward *= -1
+
+        color1 = color1.other
+
+    return rl.combine_experience(collectors)

rl/value.py

@@ -3,10 +3,10 @@ import torch.nn as nn
 import torch.optim as optim
 from torch.distributions import Categorical
 
-from dlgo import encoders
-from dlgo import goboard
-from dlgo.agent import Agent
-from dlgo.agent.helpers import is_point_an_eye
+from tugo import encoders
+from tugo import goboard
+from tugo.agents import Agent
+from tugo.agents.helpers import is_point_an_eye
 
 
 class ValueAgent(Agent):
@@ -109,7 +109,7 @@ class ValueAgent(Agent):
     def diagnostics(self):
         return {'value': self.last_move_value}
 
-    def save(self, filename):
+    def save(self, file_path):
         torch.save({
             'model': self.model.state_dict(),
             'encoder': {
@@ -117,11 +117,11 @@ class ValueAgent(Agent):
                 'board_width': self.encoder.board_width,
                 'board_height': self.encoder.board_height,
             }
-        }, filename)
+        }, file_path)
 
     @classmethod
-    def load(cls, filename):
-        checkpoint = torch.load(filename)
+    def load(cls, file_path):
+        checkpoint = torch.load(file_path)
 
         model = Model()  # The Model class should be defined appropriately
         model.load_state_dict(checkpoint['model'])
@@ -134,9 +134,9 @@ class ValueAgent(Agent):
         return cls(model, encoder)
 
 
-def load_value_agent(filename):
+def load_value_agent(file_path):
     # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。
-    checkpoint = torch.load(filename)
+    checkpoint = torch.load(file_path)
 
     model = Model()  # The Model class should be defined appropriately
     model.load_state_dict(checkpoint['model'])