Changes to allow learning for multiple agents together

gym_torcs.py

@@ -16,7 +16,7 @@ class TorcsEnv:
 
     initial_reset = False
 
-    obs_dim = 29
+    obs_dim = 65
     act_dim = 3
 
     def __init__(self, vision=False, throttle=False, gear_change=False):
@@ -210,16 +210,8 @@ def get_obs(self):
         return self.observation
 
     def reset_torcs(self):
-       #print("relaunch torcs")
+        print("relaunch torcs")
         os.system('pkill torcs')
-        time.sleep(0.5)
-        if self.vision is True:
-            os.system('torcs -nofuel -nodamage -nolaptime -vision &')
-        else:
-            os.system('torcs -nofuel -nolaptime &')
-        time.sleep(0.5)
-        os.system('sh scripts/autostart.sh')
-        time.sleep(0.5)
 
     def agent_to_torcs(self, u):
         torcs_action = {'steer': u[0]}

multi_ddpg.py

@@ -0,0 +1,30 @@
+import threading
+import multiprocessing
+import numpy as np
+#import matplotlib.pyplot as plt
+import tensorflow as tf
+#import tensorflow.contrib.slim as slim
+import playGame_DDPG
+#matplotlib inline
+import os
+from random import choice
+from time import sleep
+from time import time
+import snakeoil3_gym as snakeoil3
+#import pymp
+
+with tf.device("/cpu:0"): 
+        num_workers = 6 #multiprocessing.cpu_count() #use this when you want to use all the cpus
+        print("numb of workers is" + str(num_workers))
+
+with tf.Session() as sess:
+        worker_threads = []
+#with pymp.Parallel(4) as p:		#uncomment this for parallelization of threads
+        for i in range(num_workers):
+                worker_work = lambda: (playGame_DDPG.playGame(f_diagnostics=""+str(i), train_indicator=0, port=3101+i))
+                print("hi i am here \n")
+                t = threading.Thread(target=(worker_work))
+                print("active thread count is: " + str(threading.active_count()) + "\n")
+                t.start()
+                sleep(0.5)
+                worker_threads.append(t)

playGame_DDPG.py

@@ -13,7 +13,7 @@
 import timeit
 import math
 import sys
-
+import os
 from configurations import *
 from ddpg import *
 
@@ -25,23 +25,28 @@
 def playGame(f_diagnostics, train_indicator, port=3101):    # 1 means Train, 0 means simply Run
 
 	action_dim = 3  #Steering/Acceleration/Brake
-	state_dim = 29  #Number of sensors input
+	state_dim = 65  #of sensors input
 	env_name = 'Torcs_Env'
-	agent = DDPG(env_name, state_dim, action_dim)
+	save_location = "./weights/"+str(port)+"/"
+	agent = DDPG(env_name, state_dim, action_dim, save_location)
 
 	# Generate a Torcs environment
 	print("I have been asked to use port: ", port)
 	env = TorcsEnv(vision=False, throttle=True, gear_change=False) 
-
-	client = snakeoil3.Client(p=port, vision=False)  # Open new UDP in vtorcs
-	client.MAX_STEPS = np.inf
+	ob = None
+	while ob is None:
+		try:
+			client = snakeoil3.Client(p=port, vision=False)  # Open new UDP in vtorcs
+			client.MAX_STEPS = np.inf
 
-	client.get_servers_input(0)  # Get the initial input from torcs
+			client.get_servers_input(0)  # Get the initial input from torcs
 
-	obs = client.S.d  # Get the current full-observation from torcs
-	ob = env.make_observation(obs)
+			obs = client.S.d  # Get the current full-observation from torcs
+			ob = env.make_observation(obs)
 
-	s_t = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm))
+			s_t = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm, ob.opponents))
+		except:
+			pass
 
 	EXPLORE = total_explore
 	episode_count = max_eps
@@ -54,12 +59,12 @@ def playGame(f_diagnostics, train_indicator, port=3101):    # 1 means Train, 0 m
 	best_reward = -100000
 	running_avg_reward = 0.
 
+
 	print("TORCS Experiment Start.")
 	for i in range(episode_count):
 
 		save_indicator = 0
 		early_stop = 1
-		# Counting the total reward and total steps in the current episode
 		total_reward = 0.
 		info = {'termination_cause':0}
 		distance_traversed = 0.
@@ -76,32 +81,44 @@ def playGame(f_diagnostics, train_indicator, port=3101):    # 1 means Train, 0 m
 			    epsilon = max(epsilon, epsilon_steady_state) 
 			    a_t = agent.noise_action(s_t,epsilon) #Take noisy actions during training
 			else:
-			    a_t = agent.action(s_t)		# a_t is of the form: [steer, accel, brake]
-
-			ob, r_t, done, info = env.step(step, client, a_t, early_stop)
-			if done:
-				break
-			analyse_info(info, printing=False)
-
-			s_t1 = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm))
-			distance_traversed += ob.speedX*np.cos(ob.angle) #Assuming 1 step = 1 second
-			speed_array.append(ob.speedX*np.cos(ob.angle))
-			trackPos_array.append(ob.trackPos)
-
+			    a_t = agent.action(s_t)	# [steer, accel, brake]
+			try:
+				ob, r_t, done, info = env.step(step, client, a_t, early_stop)
+				if done:
+					break
+				# print done
+				# print 'Action taken'
+				analyse_info(info, printing=False)
+
+				s_t1 = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm, ob.opponents))
+				distance_traversed += ob.speedX*np.cos(ob.angle) #Assuming 1 step = 1 second
+				speed_array.append(ob.speedX*np.cos(ob.angle))
+				trackPos_array.append(ob.trackPos)
 
 			#Checking for nan rewards: TODO: This was actually below the following block
-			if (math.isnan( r_t )):
-				r_t = 0.0
-				for bad_r in range( 50 ):
-					print( 'Bad Reward Found' )
-				break #Introduced by Anirban
-
+				if (math.isnan( r_t )):
+					r_t = 0.0
+					for bad_r in range( 50 ):
+						print( 'Bad Reward Found' )
+					break #Introduced by Anirban
 
 			# Add to replay buffer only if training
-			if (train_indicator):
-				agent.perceive(s_t,a_t,r_t,s_t1,done) # Add experience to replay buffer
-
-
+				if (train_indicator):
+					agent.perceive(s_t,a_t,r_t,s_t1,done) # Add experience to replay buffer
+
+			except Exception as e:
+				print("Exception caught at port " + str(i) + str(e) )
+				ob = None
+				while ob is None:
+					try:
+						client = snakeoil3.Client(p=port, vision=False)  # Open new UDP in vtorcs
+						client.MAX_STEPS = np.inf
+						client.get_servers_input(0)  # Get the initial input from torcs
+						obs = client.S.d  # Get the current full-observation from torcs
+						ob = env.make_observation(obs)
+					except:
+						pass  
+					continue
 			total_reward += r_t
 			s_t = s_t1
 
@@ -113,12 +130,14 @@ def playGame(f_diagnostics, train_indicator, port=3101):    # 1 means Train, 0 m
 			if done:
 				break
 
-		# Saving the best model.
+
 		if ((save_indicator==1) and (train_indicator ==1 )):
-			if (total_reward >= best_reward):
-				print("Now we save model with reward " + str(total_reward) + " previous best reward was " + str(best_reward))
-				best_reward = total_reward
-				agent.saveNetwork()     
+                        # Uncomment this for saving only the best model.
+			#if (total_reward >= best_reward): 
+			#	print("Now we save model with reward " + str(total_reward) + " previous best reward was " + str(best_reward))
+			best_reward = total_reward
+			if(i%300 == 0):
+				agent.saveNetwork(i)     
 
 		running_avg_reward = running_average(running_avg_reward, i+1, total_reward)  
 
@@ -128,16 +147,31 @@ def playGame(f_diagnostics, train_indicator, port=3101):    # 1 means Train, 0 m
 		print("")
 
 		print(info)
-		if 'termination_cause' in info.keys() and info['termination_cause']=='hardReset':
-			print('Hard reset by some agent')
-			ob, client = env.reset(client=client) 
-		else:
-			ob, client = env.reset(client=client, relaunch=True) 
-		s_t = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm))
+		try:
+			if 'termination_cause' in info.keys() and info['termination_cause']=='hardReset':
+				print('Hard reset by some agent')
+				ob, client = env.reset(client=client, relaunch=True) 
+			else:
+				ob, client = env.reset(client=client, relaunch=True) 
+		except Exception as e:
+			print("Exception caught at point B at port " + str(i) + str(e) )
+			ob = None
+			while ob is None:
+				try:
+					client = snakeoil3.Client(p=port, vision=False)  # Open new UDP in vtorcs
+					client.MAX_STEPS = np.inf
+					client.get_servers_input(0)  # Get the initial input from torcs
+					obs = client.S.d  # Get the current full-observation from torcs
+					ob = env.make_observation(obs)
+				except:
+					print("Exception caught at at point C at port " + str(i) + str(e) )
+
+
+		s_t = np.hstack((ob.angle, ob.track, ob.trackPos, ob.speedX, ob.speedY,  ob.speedZ, ob.wheelSpinVel/100.0, ob.rpm, ob.opponents))
 
 		# document_episode(i, distance_traversed, speed_array, trackPos_array, info, running_avg_reward, f_diagnostics)
 
-	env.end()  # Shut down TORCS
+	env.end()  # This is for shutting down TORCS
 	print("Finish.")
 
 def document_episode(episode_no, distance_traversed, speed_array, trackPos_array, info, running_avg_reward, f_diagnostics):
@@ -176,13 +210,13 @@ def analyse_info(info, printing=True):
 
 	print('is_training : ' + str(is_training))
 	print('Starting best_reward : ' + str(start_reward))
-	print(total_explore)
-	print(max_eps)
-	print(max_steps_eps)
-	print(epsilon_start)
+	print( total_explore )
+	print( max_eps )
+	print( max_steps_eps )
+	print( epsilon_start )
 	print('config_file : ' + str(configFile))
 
 	# f_diagnostics = open('output_logs/diagnostics_for_window_' + sys.argv[1]+'_with_fixed_episode_length', 'w') #Add date and time to file name
 	f_diagnostics = ""
-	playGame(f_diagnostics, train_indicator=1, port=port)
+	playGame(f_diagnostics, train_indicator=0, port=port)
 	# f_diagnostics.close()

sample_DDPG_agent/actor_network.py

@@ -35,11 +35,9 @@ def __init__(self,sess,state_dim,action_dim):
     def create_training_method(self):
         self.q_gradient_input = tf.placeholder("float",[None,self.action_dim])
         self.parameters_gradients = tf.gradients(self.action_output,self.net,-self.q_gradient_input)
-        '''        
-        for i, grad in enumerate(self.parameters_gradients):
+        '''for i, grad in enumerate(self.parameters_gradients):
             if grad is not None:
-                self.parameters_gradients[i] = tf.clip_by_value(grad, -2.0,2.0)
-        '''
+                self.parameters_gradients[i] = tf.clip_by_value(grad, -2.0,2.0)'''
         self.optimizer = tf.train.AdamOptimizer(LEARNING_RATE).apply_gradients(zip(self.parameters_gradients,self.net))
 
     def create_network(self,state_dim,action_dim):
@@ -96,6 +94,9 @@ def create_target_network(self,state_dim,action_dim,net):
         action_output = tf.concat([steer, accel, brake], 1)
         return state_input,action_output,target_update,target_net
 
+
+
+
     def update_target(self):
         self.sess.run(self.target_update)
 

sample_DDPG_agent/configurations.py

@@ -9,17 +9,17 @@
 ################################################################### 
 
 visualize_after = 5
-is_training 	= 0
+is_training 	= 1
 
 # total_explore  	= 300000.0
 total_explore  	= 600000.0
-max_eps 		= 500
-max_steps_eps 	= 1000
+max_eps 	= 6000
+max_steps_eps 	= 10000
 
 wait_at_beginning 	= 0
 initial_wait_period = 200		# to give the other cars a headstart of these many steps
 
-epsilon_start  	= 0.5				# 
+epsilon_start  	= 1				# 
 start_reward 	= -10000			# these need to be changed if restarting the playGame.py script
 
 save_location = './'
@@ -29,6 +29,6 @@
 # save_location = 'saved_networks_our_traffic_noBrakes/'
 # save_location = 'saved_networks_scr_noTraffic/'				# test this playGame_old.py
 
-torcsPort 	= 3001
+torcsPort 	= 3101
 configFile 	= '~/.torcs/config/raceman/quickrace.xml'
-# configFile = '~/.torcs/config/raceman/practice.xml'
+# configFile = '~/.torcs/config/raceman/practice.xml'

sample_DDPG_agent/critic_network.py

@@ -41,16 +41,14 @@ def create_training_method(self):
         weight_decay = tf.add_n([L2 * tf.nn.l2_loss(var) for var in self.net])
         self.cost = tf.reduce_mean(tf.square(self.y_input - self.q_value_output)) + weight_decay
         self.optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize(self.cost)
-        '''
-	self.optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
+        '''self.optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
         self.parameters_gradients = self.optimizer.compute_gradients(self.cost)
 
         for i, (grad,var) in enumerate(self.parameters_gradients):
             if grad is not None:
                 self.parameters_gradients[i] = (tf.clip_by_value(grad, -5.0,5.0),var)
 
-        self.train_op = self.optimizer.apply_gradients(self.parameters_gradients)
-        '''
+        self.train_op = self.optimizer.apply_gradients(self.parameters_gradients)'''
         self.action_gradients = tf.gradients(self.q_value_output,self.action_input)
 
     def create_q_network(self,state_dim,action_dim):

sample_DDPG_agent/ddpg.py

@@ -13,7 +13,7 @@
 from critic_network import CriticNetwork 
 from actor_network import ActorNetwork
 from ReplayBuffer import ReplayBuffer
-from configurations import save_location
+#from configurations import save_location
 
 
 # Hyper Parameters:
@@ -26,14 +26,14 @@
 
 class DDPG:
     """docstring for DDPG"""
-    def __init__(self, env_name, state_dim,action_dim):
+    def __init__(self, env_name, state_dim,action_dim,save_location):
         self.name = 'DDPG' # name for uploading results
         self.env_name = env_name
         # Randomly initialize actor network and critic network
         # with both their target networks
         self.state_dim = state_dim
         self.action_dim = action_dim
-
+        self.save_location = save_location
         # Ensure action bound is symmetric
         self.time_step = 0 
         self.sess = tf.InteractiveSession()
@@ -48,8 +48,8 @@ def __init__(self, env_name, state_dim,action_dim):
         self.OU = OU()
 
         # loading networks
-        self.saver = tf.train.Saver()
-        checkpoint = tf.train.get_checkpoint_state(save_location)
+        self.saver = tf.train.Saver(max_to_keep=0)
+        checkpoint = tf.train.get_checkpoint_state(self.save_location)
         if checkpoint and checkpoint.model_checkpoint_path:
             self.saver.restore(self.sess, checkpoint.model_checkpoint_path)
             print("Successfully loaded:", checkpoint.model_checkpoint_path)
@@ -93,8 +93,8 @@ def train(self):
         self.actor_network.update_target()
         self.critic_network.update_target()
 
-    def saveNetwork(self):
-        self.saver.save(self.sess, save_location + self.env_name + 'network' + '-ddpg', global_step = self.time_step)
+    def saveNetwork(self,i):
+        self.saver.save(self.sess, self.save_location + 'network' + str(i)+'DDPG.ckpt', global_step = self.time_step,)
 
 
     def action(self,state):

scripts/autostart.sh

@@ -13,3 +13,10 @@ xte 'usleep 100000'
 xte 'key Return'
 xte 'usleep 100000'
 xte 'key Return'
+xte 'usleep 100000'
+xte 'key Return'
+# Uncomment following four lines to increase speed of TORCS simulator by 4x
+#xte 'usleep 100000'
+#xte 'keydown Shift_L'; xte 'key plus'; xte 'keyup Shift_L'
+#xte 'usleep 100000'
+#xte 'keydown Shift_L'; xte 'key plus'; xte 'keyup Shift_L'