diff --git a/agents/policy_gradient_agent.py b/agents/policy_gradient_agent.py index c9aa74f..8e80991 100644 --- a/agents/policy_gradient_agent.py +++ b/agents/policy_gradient_agent.py @@ -81,12 +81,12 @@ class PolicyAgent(Agent): loss.backward() opt.step() - def save(self, path): - torch.save(self._model.state_dict(), path) + def save(self, file_path): + torch.save(self._model.state_dict(), file_path) @classmethod - def load(cls, path, encoder): + def load(cls, file_path, encoder): device = 'cuda' if torch.cuda.is_available() else 'cpu' model = AlphaGoModel(encoder.get_input_shape(), is_policy_net=True).to(device) - model.load_state_dict(torch.load(path)) + model.load_state_dict(torch.load(file_path)) return cls(model, encoder) diff --git a/agents/predict.py b/agents/predict.py index 77f2a13..83ea33f 100644 --- a/agents/predict.py +++ b/agents/predict.py @@ -38,12 +38,12 @@ class DeepLearningAgent(Agent): return goboard.Move.play(point) return goboard.Move.pass_turn() - def save(self, path): - torch.save(self.model.state_dict(), path) + def save(self, file_path): + torch.save(self.model.state_dict(), file_path) @classmethod - def load(cls, path, encoder): + def load(cls, file_path, encoder): device = 'cuda' if torch.cuda.is_available() else 'cpu' model = AlphaGoModel(encoder.get_input_shape(), is_policy_net=True).to(device) - model.load_state_dict(torch.load(path)) + model.load_state_dict(torch.load(file_path)) return cls(model, encoder) diff --git a/alphago_policy_rl.py b/alphago_policy_rl.py index e1dfda1..a0e10dc 100644 --- a/alphago_policy_rl.py +++ b/alphago_policy_rl.py @@ -1,3 +1,4 @@ +""" reinforcement learning，在监督学习之后，进行自我对弈的强化学习的策略网络 """ import torch from tqdm import tqdm diff --git a/alphago_policy_sl.py b/alphago_policy_sl.py index 668cb27..c29dbf4 100644 --- a/alphago_policy_sl.py +++ b/alphago_policy_sl.py @@ -1,3 +1,4 @@ +""" supervised learning，采用监督学习的策略网络 """ from tugo.data_processing.parallel_processor import GoDataProcessor from tugo.encoders.alphago import AlphaGoEncoder from tugo.agents.predict import DeepLearningAgent diff --git a/alphago_value.py b/alphago_value.py new file mode 100644 index 0000000..27d8c0e --- /dev/null +++ b/alphago_value.py @@ -0,0 +1,21 @@ +""" AlphaGo 的价值网络部分，用于评估棋盘状态的价值，即估算当前的局面走某步棋后的胜率 """ +# from tugo.models.alphago import alphago_model +from tugo.models.alphago import AlphaGoModel +from tugo.encoders.alphago import AlphaGoEncoder +from tugo.rl.value import ValueAgent +from tugo.rl.experience import ExperienceBuffer + +# init_value +rows, cols = 19, 19 +encoder = AlphaGoEncoder() +input_shape = (encoder.num_planes, rows, cols) +alphago_value_network = AlphaGoModel(input_shape) + +alphago_value = ValueAgent(alphago_value_network, encoder) + +# train_value +experience = ExperienceBuffer.load('checkpoints/alphago_rl_experience.h5') + +alphago_value.train(experience) + +alphago_value.save('checkpoints/alphago_value.h5') diff --git a/rl/ac.py b/rl/ac.py index 61da5af..ca3ddcf 100644 --- a/rl/ac.py +++ b/rl/ac.py @@ -93,14 +93,14 @@ class ACAgent(Agent): def diagnostics(self): return {'value': self.last_state_value} - def save(self, path): - torch.save(self.model.state_dict(), path) + def save(self, file_path): + torch.save(self.model.state_dict(), file_path) @classmethod - def load(cls, path, encoder): + def load(cls, file_path, encoder): device = 'cuda' if torch.cuda.is_available() else 'cpu' model = AlphaGoModel(encoder.get_input_shape(), is_policy_net=True).to(device) - model.load_state_dict(torch.load(path)) + model.load_state_dict(torch.load(file_path)) return cls(model, encoder) # def load_ac_agent(path, encoder): diff --git a/rl/ac_pass.py b/rl/ac_pass.py index e69de29..50fe50d 100644 --- a/rl/ac_pass.py +++ b/rl/ac_pass.py @@ -0,0 +1,111 @@ +import torch +import numpy as np + +from tugo import encoders +from tugo.game_logic import goboard +from tugo.agents import Agent + + +class ACAgent(Agent): + def __init__(self, model, encoder): + Agent.__init__(self) + self.model = model + self.encoder = encoder + self.collector = None + self.temperature = 1.0 + + self.last_state_value = 0 + + def set_temperature(self, temperature): + self.temperature = temperature + + def set_collector(self, collector): + self.collector = collector + + def select_move(self, game_state): + num_moves = self.encoder.board_width * self.encoder.board_height + + board_tensor = self.encoder.encode(game_state) + x = torch.from_numpy(np.array([board_tensor])).float().to(self.model.device) + + actions, values = self.model(x) + move_probs = actions[0].detach().cpu().numpy() + estimated_value = values[0][0].item() + self.last_state_value = float(estimated_value) + + # Prevent move probs from getting stuck at 0 or 1. + move_probs = np.power(move_probs, 1.0 / self.temperature) + move_probs = move_probs / np.sum(move_probs) + eps = 1e-6 + move_probs = np.clip(move_probs, eps, 1 - eps) + # Re-normalize to get another probability distribution. + move_probs = move_probs / np.sum(move_probs) + + # Turn the probabilities into a ranked list of moves. + candidates = np.arange(num_moves) + ranked_moves = np.random.choice( + candidates, num_moves, replace=False, p=move_probs) + for point_idx in ranked_moves: + point = self.encoder.decode_point_index(point_idx) + true_move = goboard.Move.play(point) + if not game_state.is_valid_move(true_move): + true_move = goboard.Move.pass_turn() + if self.collector is not None: + self.collector.record_decision( + state=board_tensor, + action=point_idx, + estimated_value=estimated_value + ) + return true_move + # No legal, non-self-destructive moves less. + return goboard.Move.pass_turn() + + def train(self, experience, lr=0.1, batch_size=128): + optimizer = torch.optim.SGD(self.model.parameters(), lr=lr, clipvalue=0.2) + criterion = torch.nn.CrossEntropyLoss() + + n = experience.states.shape[0] + num_moves = self.encoder.num_points() + policy_target = np.zeros((n, num_moves)) + value_target = np.zeros((n,)) + for i in range(n): + action = experience.actions[i] + reward = experience.rewards[i] + policy_target[i][action] = experience.advantages[i] + value_target[i] = reward + + self.model.train() + optimizer.zero_grad() + policy_target_t = torch.from_numpy(policy_target).float().to(self.model.device) + value_target_t = torch.from_numpy(value_target).float().to(self.model.device) + states_t = torch.from_numpy(experience.states).float().to(self.model.device) + actions_pred, values_pred = self.model(states_t) + loss1 = criterion(actions_pred, policy_target_t) + loss2 = criterion(values_pred, value_target_t) + loss = loss1 + loss2 + loss.backward() + optimizer.step() + + def serialize(self, file_path): + torch.save({ + 'model_state_dict': self.model.state_dict(), + 'encoder_name': self.encoder.name(), + 'encoder_board_width': self.encoder.board_width, + 'encoder_board_height': self.encoder.board_height, + }, file_path) + + def diagnostics(self): + return {'value': self.last_state_value} + + +def load_passing_ac_agent(file_path): + checkpoint = torch.load(file_path) + model = TheModelClass() # TheModelClass should be replaced with your model class + model.load_state_dict(checkpoint['model_state_dict']) + encoder_name = checkpoint['encoder_name'] + board_width = checkpoint['encoder_board_width'] + board_height = checkpoint['encoder_board_height'] + encoder = encoders.get_encoder_by_name( + encoder_name, + (board_width, board_height)) + return ACAgent(model, encoder) diff --git a/rl/experience.py b/rl/experience.py index 4946e7f..cc55298 100644 --- a/rl/experience.py +++ b/rl/experience.py @@ -43,13 +43,13 @@ class ExperienceBuffer: self.rewards = rewards self.advantages = advantages - def save(self, filename): - with open(filename, 'wb') as f: + def save(self, file_path): + with open(file_path, 'wb') as f: pickle.dump((self.states, self.actions, self.rewards, self.advantages), f) @classmethod - def load(cls, filename): - with open(filename, 'rb') as f: + def load(cls, file_path): + with open(file_path, 'rb') as f: states, actions, rewards, advantages = pickle.load(f) return cls( states=states, diff --git a/rl/q.py b/rl/q.py index 9f34a15..d2629ab 100644 --- a/rl/q.py +++ b/rl/q.py @@ -113,7 +113,7 @@ class QAgent(Agent): def diagnostics(self): return {'value': self.last_move_value} - def save(self, filename): + def save(self, file_path): torch.save({ 'model': self.model.state_dict(), 'encoder': { @@ -121,11 +121,11 @@ class QAgent(Agent): 'board_width': self.encoder.board_width, 'board_height': self.encoder.board_height, } - }, filename) + }, file_path) @classmethod - def load(cls, filename): - checkpoint = torch.load(filename) + def load(cls, file_path): + checkpoint = torch.load(file_path) # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder model = Model() # The Model class should be defined appropriately @@ -139,17 +139,17 @@ class QAgent(Agent): return cls(model, encoder) -def load_q_agent(filename): - # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。 - checkpoint = torch.load(filename) - - # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder - model = Model() # The Model class should be defined appropriately - model.load_state_dict(checkpoint['model']) - - encoder_info = checkpoint['encoder'] - encoder = encoders.get_encoder_by_name( - encoder_info['name'], - (encoder_info['board_width'], encoder_info['board_height'])) - - return QAgent(model, encoder) +# def load_q_agent(file_path): +# # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。 +# checkpoint = torch.load(file_path) +# +# # Assuming `model` is a PyTorch model here and `encoder` is a DLGo encoder +# model = Model() # The Model class should be defined appropriately +# model.load_state_dict(checkpoint['model']) +# +# encoder_info = checkpoint['encoder'] +# encoder = encoders.get_encoder_by_name( +# encoder_info['name'], +# (encoder_info['board_width'], encoder_info['board_height'])) +# +# return QAgent(model, encoder) diff --git a/rl/simulate.py b/rl/simulate.py index e69de29..b1cacc2 100644 --- a/rl/simulate.py +++ b/rl/simulate.py @@ -0,0 +1,58 @@ +from tugo import rl +from tugo.game_logic import scoring +from tugo.game_logic import goboard_fast as goboard +from tugo.game_logic.gotypes import Player + +from collections import namedtuple + + +class GameRecord(namedtuple('GameRecord', 'moves winner margin')): + pass + + +def simulate_game(black_player, white_player): + moves = [] + game = goboard.GameState.new_game(19) + agents = { + Player.black: black_player, + Player.white: white_player, + } + while not game.is_over(): + next_move = agents[game.next_player].select_move(game) + moves.append(next_move) + game = game.apply_move(next_move) + + game_result = scoring.compute_game_result(game) + # print(game_result) + + return GameRecord( + moves=moves, + winner=game_result.winner, + margin=game_result.winning_margin, + ) + + +def experience_simulation(num_games, agent1, agent2): + collectors = [rl.ExperienceCollector() for _ in range(2)] + agents = [agent1, agent2] + + color1 = Player.black + for i in range(num_games): + for collector, agent in zip(collectors, agents): + collector.begin_episode() + agent.set_collector(collector) + + if color1 == Player.black: + black_player, white_player = agent1, agent2 + else: + white_player, black_player = agent2, agent1 + game_record = simulate_game(black_player, white_player) + + winner_reward = 1 if game_record.winner == color1 else -1 + for collector in collectors: + collector.complete_episode(reward=winner_reward) + winner_reward *= -1 + + color1 = color1.other + + return rl.combine_experience(collectors) diff --git a/rl/value.py b/rl/value.py index 2af6e44..10b22a6 100644 --- a/rl/value.py +++ b/rl/value.py @@ -3,10 +3,10 @@ import torch.nn as nn import torch.optim as optim from torch.distributions import Categorical -from dlgo import encoders -from dlgo import goboard -from dlgo.agent import Agent -from dlgo.agent.helpers import is_point_an_eye +from tugo import encoders +from tugo import goboard +from tugo.agents import Agent +from tugo.agents.helpers import is_point_an_eye class ValueAgent(Agent): @@ -109,7 +109,7 @@ class ValueAgent(Agent): def diagnostics(self): return {'value': self.last_move_value} - def save(self, filename): + def save(self, file_path): torch.save({ 'model': self.model.state_dict(), 'encoder': { @@ -117,11 +117,11 @@ class ValueAgent(Agent): 'board_width': self.encoder.board_width, 'board_height': self.encoder.board_height, } - }, filename) + }, file_path) @classmethod - def load(cls, filename): - checkpoint = torch.load(filename) + def load(cls, file_path): + checkpoint = torch.load(file_path) model = Model() # The Model class should be defined appropriately model.load_state_dict(checkpoint['model']) @@ -134,9 +134,9 @@ class ValueAgent(Agent): return cls(model, encoder) -def load_value_agent(filename): +def load_value_agent(file_path): # 我假设 Model 类是预先定义好的 PyTorch 模型，你需要根据实际情况替换或实现这个模型。 - checkpoint = torch.load(filename) + checkpoint = torch.load(file_path) model = Model() # The Model class should be defined appropriately model.load_state_dict(checkpoint['model']) diff --git a/tdl.md b/tdl.md new file mode 100644 index 0000000..4984b98 --- /dev/null +++ b/tdl.md @@ -0,0 +1,16 @@ + +充分利用PyTorch的能力并优化代码主要包括以下几个方面： + +1. **使用GPU计算**： PyTorch支持使用GPU进行计算，这可以大大加速模型的训练和推理速度。您可以通过使用`.to(device)`方法将数据和模型发送到GPU，其中`device`是一个特定的设备指示器，如`device = torch.device("cuda:0")`。 + +2. **使用TorchScript**：TorchScript是一个静态图表示，提供了优化和序列化能力。通过将模型转换为TorchScript，可以提高模型的运行速度并优化内存使用。 + +3. **使用DataLoader**：PyTorch的`DataLoader`类可以自动进行数据分批，洗牌和多线程数据加载。这样可以使数据加载更高效，且易于管理。 + +4. **使用模型训练工具**：PyTorch提供了许多方便的模型训练工具，如学习率调度器(`torch.optim.lr_scheduler`)和模型保存/加载函数(`torch.save()`, `torch.load()`)。 + +5. **使用自动混合精度训练 (AMP)**：这是一个在PyTorch中进行混合精度训练的工具，可以提高训练速度并减少内存消耗。 + +6. **优化网络结构**：对于神经网络，合理的网络结构和参数设置也能够极大地提高训练和推断的速度，例如使用Batch Normalization，更合适的激活函数等。 + +以上提及的优化方法需要根据具体模型和需求来进行修改和设置。