first commit

Author: gissimo

.gitignore

@@ -0,0 +1,14 @@
+.DS_Store
+*.pyc
+*.pyo
+__pycache__
+*.h5
+*.json
+runs/
+check*
+omnet/*/out/
+omnet/*/networkRL
+omnet/*/messages/*.cc
+omnet/*/messages/*.h
+omnet/*/NetworkAll*
+traffic/

ActorNetwork.py

@@ -0,0 +1,69 @@
+"""
+ActorNetwork.py
+"""
+__author__ = "giorgio@ac.upc.edu"
+__credits__ = "https://github.com/yanpanlau"
+
+from keras.initializations import normal, glorot_normal
+from keras.activations import relu
+from keras.layers import Dense, Input, BatchNormalization
+from keras.models import Model
+from keras.regularizers import l2
+import keras.backend as K
+import tensorflow as tf
+
+from helper import selu
+
+
+class ActorNetwork(object):
+    def __init__(self, sess, state_size, action_size, DDPG_config):
+        self.HIDDEN1_UNITS = DDPG_config['HIDDEN1_UNITS']
+        self.HIDDEN2_UNITS = DDPG_config['HIDDEN2_UNITS']
+
+        self.sess = sess
+        self.BATCH_SIZE = DDPG_config['BATCH_SIZE']
+        self.TAU = DDPG_config['TAU']
+        self.LEARNING_RATE = DDPG_config['LRA']
+        self.ACTUM = DDPG_config['ACTUM']
+
+        if self.ACTUM == 'NEW':
+            self.acti = 'sigmoid'
+        elif self.ACTUM == 'DELTA':
+            self.acti = 'tanh'
+
+        self.h_acti = relu
+        if DDPG_config['HACTI'] == 'selu':
+            self.h_acti = selu
+
+        K.set_session(sess)
+
+        #Now create the model
+        self.model, self.weights, self.state = self.create_actor_network(state_size, action_size)
+        self.target_model, self.target_weights, self.target_state = self.create_actor_network(state_size, action_size)
+        self.action_gradient = tf.placeholder(tf.float32, [None, action_size])
+        self.params_grad = tf.gradients(self.model.output, self.weights, -self.action_gradient)
+        grads = zip(self.params_grad, self.weights)
+        self.optimize = tf.train.AdamOptimizer(self.LEARNING_RATE).apply_gradients(grads)
+        self.sess.run(tf.global_variables_initializer())
+
+    def train(self, states, action_grads):
+        self.sess.run(self.optimize, feed_dict={
+            self.state: states,
+            self.action_gradient: action_grads
+        })
+
+    def target_train(self):
+        actor_weights = self.model.get_weights()
+        actor_target_weights = self.target_model.get_weights()
+        for i in range(len(actor_weights)):
+            actor_target_weights[i] = self.TAU * actor_weights[i] + (1 - self.TAU)* actor_target_weights[i]
+        self.target_model.set_weights(actor_target_weights)
+
+    def create_actor_network(self, state_size, action_dim):
+        S = Input(shape=[state_size], name='a_S')
+        h0 = Dense(self.HIDDEN1_UNITS, activation=self.h_acti, init=glorot_normal, name='a_h0')(S)
+        h1 = Dense(self.HIDDEN2_UNITS, activation=self.h_acti, init=glorot_normal, name='a_h1')(h0)
+        # https://github.com/fchollet/keras/issues/374
+        V = Dense(action_dim, activation=self.acti, init=glorot_normal, name='a_V')(h1)
+        model = Model(input=S, output=V)
+        return model, model.trainable_weights, S

CriticNetwork.py

@@ -0,0 +1,67 @@
+"""
+CriticNetwork.py
+"""
+__author__ = "giorgio@ac.upc.edu"
+__credits__ = "https://github.com/yanpanlau"
+
+from keras.initializations import normal, glorot_normal
+from keras.activations import relu
+from keras.layers import Dense, Input, merge, BatchNormalization
+from keras.models import Model
+from keras.optimizers import Adam
+from keras.regularizers import l2
+import keras.backend as K
+import tensorflow as tf
+
+from helper import selu
+
+
+class CriticNetwork(object):
+    def __init__(self, sess, state_size, action_size, DDPG_config):
+        self.HIDDEN1_UNITS = DDPG_config['HIDDEN1_UNITS']
+        self.HIDDEN2_UNITS = DDPG_config['HIDDEN2_UNITS']
+
+        self.sess = sess
+        self.BATCH_SIZE = DDPG_config['BATCH_SIZE']
+        self.TAU = DDPG_config['TAU']
+        self.LEARNING_RATE = DDPG_config['LRC']
+        self.action_size = action_size
+
+        self.h_acti = relu
+        if DDPG_config['HACTI'] == 'selu':
+            self.h_acti = selu
+
+        K.set_session(sess)
+
+        #Now create the model
+        self.model, self.action, self.state = self.create_critic_network(state_size, action_size)
+        self.target_model, self.target_action, self.target_state = self.create_critic_network(state_size, action_size)
+        self.action_grads = tf.gradients(self.model.output, self.action)  #GRADIENTS for policy update
+        self.sess.run(tf.global_variables_initializer())
+
+    def gradients(self, states, actions):
+        return self.sess.run(self.action_grads, feed_dict={
+            self.state: states,
+            self.action: actions
+        })[0]
+
+    def target_train(self):
+        critic_weights = self.model.get_weights()
+        critic_target_weights = self.target_model.get_weights()
+        for i in range(len(critic_weights)):
+            critic_target_weights[i] = self.TAU * critic_weights[i] + (1 - self.TAU)* critic_target_weights[i]
+        self.target_model.set_weights(critic_target_weights)
+
+    def create_critic_network(self, state_size, action_dim):
+        S = Input(shape=[state_size], name='c_S')
+        A = Input(shape=[action_dim], name='c_A')
+        w1 = Dense(self.HIDDEN1_UNITS, activation=self.h_acti, init=glorot_normal, name='c_w1')(S)
+        a1 = Dense(self.HIDDEN2_UNITS, activation='linear', init=glorot_normal, name='c_a1')(A)
+        h1 = Dense(self.HIDDEN2_UNITS, activation='linear', init=glorot_normal, name='c_h1')(w1)
+        h2 = merge([h1, a1], mode='sum', name='c_h2')
+        h3 = Dense(self.HIDDEN2_UNITS, activation=self.h_acti, init=glorot_normal, name='c_h3')(h2)
+        V = Dense(action_dim, activation='linear', init=glorot_normal, name='c_V')(h3)
+        model = Model(input=[S, A], output=V)
+        adam = Adam(lr=self.LEARNING_RATE)
+        model.compile(loss='mse', optimizer=adam)
+        return model, A, S

DDPG.json

@@ -0,0 +1,28 @@
+{
+    "ACTIVE_NODES": 14,
+    "ACTUM": "NEW",
+    "BATCH_SIZE": 32,
+    "BN": "reward",
+    "BUFFER_SIZE": 1600,
+    "ENV": "label",
+    "EPISODE_COUNT": 100,
+    "EXPLORE": 0.8,
+    "GAMMA": 0.99,
+    "HACTI": "selu",
+    "HIDDEN1_UNITS": 91,
+    "HIDDEN2_UNITS": 42,
+    "LRA": 0.0001,
+    "LRC": 0.001,
+    "MAX_DELTA": 0.1,
+    "MAX_STEPS": 1000,
+    "MU": 0.0,
+    "PRAEMIUM": "AVG",
+    "PRINT": false,
+    "ROUTING": "Linkweight",
+    "RSEED": null,
+    "SIGMA": 0.4,
+    "STATUM": "T",
+    "TAU": 0.001,
+    "THETA": 0.2,
+    "TRAFFIC": "EXP"
+}