Extended testing for tensorset and modelrun (#320)

* [add] extended testing for tensorset and modelrun

* [fix] adding numpy version to make sure we have rng

* [add] updated numpy test requirements to 1.17 (use rng )

* [fix] proper naming for tensorflow modelrun added test

Author: filipecosta90

.circleci/config.yml

@@ -36,7 +36,7 @@ commands:
           name: Test
           command: |
             mkdir -p ~/workspace/tests
-            make -C opt test SHOW=1 VERBOSE=1
+            make -C opt test SHOW=1
       - run:
           name: Package
           command: make -C opt pack BRANCH="${CIRCLE_BRANCH//[^A-Za-z0-9._-]/_}" INTO=~/workspace/packages SHOW=1

test/includes.py

@@ -5,7 +5,7 @@
 import time
 from multiprocessing import Process
 import threading
-
+from numpy.random import default_rng
 import numpy as np
 from skimage.io import imread
 from skimage.transform import resize
@@ -84,6 +84,22 @@ def load_mobilenet_test_data():
 
     return model_pb, labels, img
 
+def load_creditcardfraud_data(env,max_tensors=10000):
+    test_data_path = os.path.join(os.path.dirname(__file__), 'test_data')
+    model_filename = os.path.join(test_data_path, 'creditcardfraud.pb')
+    creditcard_transaction_filename = os.path.join(test_data_path, 'creditcard_10K.csv')
+    rg = default_rng()
+
+    creditcard_transactions = np.genfromtxt(creditcard_transaction_filename, delimiter=',', dtype='float32', skip_header=1, usecols=range(0,30))
+
+    creditcard_referencedata = []
+    for tr in range(0,max_tensors):
+        creditcard_referencedata.append(rg.random((1,256), dtype='float32'))
+
+    with open(model_filename, 'rb') as f:
+        model_pb = f.read()
+
+    return model_pb, creditcard_transactions, creditcard_referencedata
 
 def run_mobilenet(con, img, input_var, output_var):
     time.sleep(0.5 * random.randint(0, 10))

test/test_data/creditcard_10K.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96ef0da3497866b726ac88928ed876d9647ca6a77b80d80f08513a8daaa38e07
+size 12251

test/test_data/creditcardfraud.pb

@@ -1,3 +1,3 @@
-numpy
+numpy>=1.17.1
 scikit-image
 redis-py-cluster

test/test_requirements.txt

@@ -61,23 +61,23 @@ def test_common_tensorset_error_replies(env):
     except Exception as e:
         exception = e
         env.assertEqual(type(exception), redis.exceptions.ResponseError)
-        env.assertEqual(exception.__str__(), "negative value found in tensor shape")
+        env.assertEqual("negative value found in tensor shape",exception.__str__())
 
-    # ERR unsupported data format
+    # ERR invalid argument found in tensor shape
     try:
         con.execute_command('AI.TENSORSET', 'z', 'INT32', 2, 'unsupported', 2, 3)
     except Exception as e:
         exception = e
         env.assertEqual(type(exception), redis.exceptions.ResponseError)
-        env.assertEqual(exception.__str__(), "invalid argument found in tensor shape")
+        env.assertEqual("invalid argument found in tensor shape",exception.__str__())
 
     # ERR invalid value
     try:
         con.execute_command('AI.TENSORSET', 'z', 'FLOAT', 2, 'VALUES', 2, 'A')
     except Exception as e:
         exception = e
         env.assertEqual(type(exception), redis.exceptions.ResponseError)
-        env.assertEqual(exception.__str__(), "invalid value")
+        env.assertEqual("invalid value",exception.__str__())
 
     # ERR invalid value
     try:
@@ -198,14 +198,14 @@ def test_common_tensorget_error_replies(env):
         env.assertEqual(exception.__str__(), "WRONGTYPE Operation against a key holding the wrong kind of value")
 
     # ERR unsupported data format
+    ret = con.execute_command('AI.TENSORSET', "T_FLOAT", "FLOAT", 2, 'VALUES', 1, 1)
+    env.assertEqual(ret, b'OK')
     try:
-        ret = con.execute_command('AI.TENSORSET', "T_FLOAT", "FLOAT", 2, 'VALUES', 1, 1)
-        env.assertEqual(ret, b'OK')
         con.execute_command('AI.TENSORGET', 'T_FLOAT', 'unsupported')
     except Exception as e:
         exception = e
         env.assertEqual(type(exception), redis.exceptions.ResponseError)
-        env.assertEqual(exception.__str__(), "unsupported data format")
+        env.assertEqual("unsupported data format",exception.__str__())
 
 
 def test_common_tensorset_multiproc(env):
@@ -219,6 +219,38 @@ def test_common_tensorset_multiproc(env):
     env.assertEqual(values, [b'2', b'3'])
 
 
+def test_common_tensorset_multiproc_blob(env):
+    con = env.getConnection()
+    tested_datatypes = ["FLOAT", "DOUBLE", "INT8", "INT16", "INT32", "INT64", "UINT8", "UINT16"]
+    tested_datatypes_map = {}
+    for datatype in tested_datatypes:
+        ret = con.execute_command('AI.TENSORSET', 'tensor_{0}'.format(datatype), datatype, 1, 256)
+        env.assertEqual(ret, b'OK')
+
+    ensureSlaveSynced(con, env)
+
+    # AI.TENSORGET in BLOB format and set in a new key
+    for datatype in tested_datatypes:
+        tensor_dtype, tensor_dim, tensor_blob = con.execute_command('AI.TENSORGET', 'tensor_{0}'.format(datatype),
+                                                                    'BLOB')
+        ret = con.execute_command('AI.TENSORSET', 'tensor_blob_{0}'.format(datatype), datatype, 1, 256, 'BLOB', tensor_blob)
+        tested_datatypes_map[datatype]=tensor_blob
+        env.assertEqual(ret, b'OK')
+
+    def funcname(env, blob, repetitions, same_key):
+        for _ in range(1,same_key):
+            for repetion in range(1,repetitions):
+                env.execute_command('AI.TENSORSET', 'tensor_{0}'.format(repetitions), 'FLOAT', 1, 256, 'BLOB', blob)
+    
+    tensor_blob = tested_datatypes_map["FLOAT"]
+    t = time.time()
+    run_test_multiproc(env, 10,
+                       lambda env: funcname(env,tensor_blob,10000,10) )
+    elapsed_time = time.time() - t
+    avg_ops_sec = 100000*10/elapsed_time
+    env.debugPrint("AI.TENSORSET elapsed time(sec) {:6.2f}\tAvg. ops/sec {:10.2f}".format(elapsed_time, avg_ops_sec), True)
+
+
 def test_tensorset_disconnect(env):
     red = env.getConnection()
     ret = send_and_disconnect(('AI.TENSORSET', 't_FLOAT', 'FLOAT', 2, 'VALUES', 2, 3), red)

test/tests_common.py

@@ -605,7 +605,7 @@ def test_tensorflow_modelrun_disconnect(env):
 
 
 @skip_if_no_TF
-def test_with_batch_and_minbatch(env):
+def test_tensorflow_modelrun_with_batch_and_minbatch(env):
     con = env.getConnection()
     batch_size = 2
     minbatch_size = 2
@@ -663,4 +663,36 @@ def run(name=model_name, output_name='output'):
 
     p3.terminate()
 
+@skip_if_no_TF
+def test_tensorflow_modelrun_financialNet(env):
+    con = env.getConnection()
+
+    model_pb, creditcard_transactions, creditcard_referencedata = load_creditcardfraud_data(env)
+
+    tensor_number = 1
+    for transaction_tensor in creditcard_transactions:
+        ret = con.execute_command('AI.TENSORSET', 'transactionTensor:{0}'.format(tensor_number),
+                                  'FLOAT', 1, 30,
+                                  'BLOB', transaction_tensor.tobytes())
+        env.assertEqual(ret, b'OK')
+        tensor_number = tensor_number + 1
+
+    tensor_number = 1
+    for reference_tensor in creditcard_referencedata:
+        ret = con.execute_command('AI.TENSORSET', 'referenceTensor:{0}'.format(tensor_number),
+                                  'FLOAT', 1, 256,
+                                  'BLOB', reference_tensor.tobytes())
+        env.assertEqual(ret, b'OK')
+        tensor_number = tensor_number + 1
+
+    ret = con.execute_command('AI.MODELSET', 'financialNet', 'TF', "CPU",
+                              'INPUTS', 'transaction', 'reference', 'OUTPUTS', 'output', model_pb)
+    env.assertEqual(ret, b'OK')
 
+    for tensor_number in range(1, 10001):
+        for repetition in range(0, 10):
+            ret = con.execute_command('AI.MODELRUN', 'financialNet', 'INPUTS',
+                                      'transactionTensor:{}'.format(tensor_number),
+                                      'referenceTensor:{}'.format(tensor_number), 'OUTPUTS',
+                                      'classificationTensor:{}_{}'.format(tensor_number, repetition))
+            env.assertEqual(ret, b'OK')