'''
In this example the Q-learning algorithm is used via self-play
to learn the state-action values for all Tic-Tac-Toe positions.
'''
from capstone.game.games import TicTacToe
from capstone.game.utils import tic2pdf
from capstone.rl import Environment, GameMDP
from capstone.rl.learners import QLearningSelfPlay
from capstone.rl.value_functions import TabularF

game = TicTacToe()
env = Environment(GameMDP(game))
qlearning = QLearningSelfPlay(env, n_episodes=1000, random_state=0)
qlearning.learn()

for move in game.legal_moves():
    print('-' * 80)
    value = qlearning.qf[(game, move)]
    new_game = game.copy().make_move(move)
    print(value)
    print(new_game)

'''
The Q-learning algorithm is used to learn the state-action values for all
Tic-Tac-Toe positions by playing games against itself (self-play).
'''
from capstone.game.games import TicTacToe
from capstone.game.players import GreedyQF, RandPlayer
from capstone.game.utils import play_series, tic2pdf
from capstone.rl import Environment, GameMDP
from capstone.rl.learners import QLearningSelfPlay

game = TicTacToe()
env = Environment(GameMDP(game))
qlearning = QLearningSelfPlay(env, n_episodes=100000, verbose=0)
qlearning.learn()

for move in game.legal_moves():
    print('-' * 80)
    value = qlearning.qf[game, move]
    new_game = game.copy().make_move(move)
    print(value)
    print(new_game)

players = [GreedyQF(qlearning.qf), RandPlayer()]
play_series(TicTacToe(), players, n_matches=10000)

# show number of unvisited state?

Fix Tic-Tac-Toe Q-learning tabular self-play example
Fix Tic-Tac-Toe Q-learning tabular self-play example

  '''
- In this example the Q-learning algorithm is used via self-play
- to learn the state-action values for all Tic-Tac-Toe positions.
+ The Q-learning algorithm is used to learn the state-action values for all
+ Tic-Tac-Toe positions by playing games against itself (self-play).
  '''
  from capstone.game.games import TicTacToe
+ from capstone.game.players import GreedyQF, RandPlayer
- from capstone.game.utils import tic2pdf
+ from capstone.game.utils import play_series, tic2pdf
  from capstone.rl import Environment, GameMDP
  from capstone.rl.learners import QLearningSelfPlay
- from capstone.rl.value_functions import TabularF
  
  game = TicTacToe()
  env = Environment(GameMDP(game))
- qlearning = QLearningSelfPlay(env, n_episodes=1000, random_state=0)
+ qlearning = QLearningSelfPlay(env, n_episodes=100000, verbose=0)
  qlearning.learn()
  
  for move in game.legal_moves():
      print('-' * 80)
-     value = qlearning.qf[(game, move)]
+     value = qlearning.qf[game, move]
      new_game = game.copy().make_move(move)
      print(value)
      print(new_game)
  
+ players = [GreedyQF(qlearning.qf), RandPlayer()]
+ play_series(TicTacToe(), players, n_matches=10000)
+ 
+ # show number of unvisited state?
+ 
Fix Tic-Tac-Toe Q-learning tabular self-play example
## Code Before:
'''
In this example the Q-learning algorithm is used via self-play
to learn the state-action values for all Tic-Tac-Toe positions.
'''
from capstone.game.games import TicTacToe
from capstone.game.utils import tic2pdf
from capstone.rl import Environment, GameMDP
from capstone.rl.learners import QLearningSelfPlay
from capstone.rl.value_functions import TabularF

game = TicTacToe()
env = Environment(GameMDP(game))
qlearning = QLearningSelfPlay(env, n_episodes=1000, random_state=0)
qlearning.learn()

for move in game.legal_moves():
    print('-' * 80)
    value = qlearning.qf[(game, move)]
    new_game = game.copy().make_move(move)
    print(value)
    print(new_game)

## Instruction:
Fix Tic-Tac-Toe Q-learning tabular self-play example
## Code After:
'''
The Q-learning algorithm is used to learn the state-action values for all
Tic-Tac-Toe positions by playing games against itself (self-play).
'''
from capstone.game.games import TicTacToe
from capstone.game.players import GreedyQF, RandPlayer
from capstone.game.utils import play_series, tic2pdf
from capstone.rl import Environment, GameMDP
from capstone.rl.learners import QLearningSelfPlay

game = TicTacToe()
env = Environment(GameMDP(game))
qlearning = QLearningSelfPlay(env, n_episodes=100000, verbose=0)
qlearning.learn()

for move in game.legal_moves():
    print('-' * 80)
    value = qlearning.qf[game, move]
    new_game = game.copy().make_move(move)
    print(value)
    print(new_game)

players = [GreedyQF(qlearning.qf), RandPlayer()]
play_series(TicTacToe(), players, n_matches=10000)

# show number of unvisited state?
