Merge pull request #150 from maxsch3/145-type-error-with-new-tensorflow-218

145 type error with new tensorflow 218

Author: maxsch3

docs/images/functional_diagram.svg

@@ -38,7 +38,7 @@ label_enc = LabelBinarizer().fit(df['label'])
 # define a batch generator
 train_gen = BatchGenerator(
     df,
-    x_structure=[('var1', var1_enc), ('var2', var2_enc)],
+    x_structure=(('var1', var1_enc), ('var2', var2_enc)),
     y_structure=('label', label_enc),
     batch_size=4,
     train_mode=True
@@ -49,7 +49,7 @@ The generator returns batches of format (x_structure, y_structure) and the shape
 
 ```python
 >>> train_gen.shape
-([(None, ), (None, )], (None, 3))
+(((None, ), (None, )), (None, 3))
 ``` 
 
 The first element is a x_structure and it is a list if two inputs. Both of them are outputs of LabelEncoders, that

docs/index.md

@@ -59,11 +59,11 @@ embark_encoder = LabelEncoder().fit(titanic_data['Embarked'])
 cabinclass_encoder = LabelEncoder().fit(titanic_data['Pclass'])
 
 train_generator = BatchGenerator(titanic_data, 
-                                 x_structure=[
+                                 x_structure=(
                                     ('Embarked', embark_encoder),
                                     ('Pclass', cabinclass_encoder),
-                                    ('Age', None)                    
-                                 ],
+                                    ('Age', None)
+                                 ),
                                  y_structure=('Survived', None),
                                  batch_size=32,
                                  shuffle=True

docs/user_guide.md

@@ -17,12 +17,17 @@ class BatchGenerator:
     **Parameters:**
 
     - **data** - a *Pandas dataframe* containing a dataset with both x and y
-    - **x_structure** - *tuple* or *list of tuples* - a structure describing mapping of dataframe columns to
-        pre-fitted encoders and to keras model inputs. When model has multiple inputs, keras expects
-        a list of numpy arrays as model X's. Each tuple is a mapping of a dataframe column to a relevant encoder.
-        It has format `('column name', encoder)`. If encoder is None, the column values will be converted to numpy
-        array and passed unchanged. If `(None, value)` is used, a new constant of value = `value` will be added
-        to Batch generator's output.
+    - **x_structure** - *tuple* or *tuple of tuples* - a structure describing mapping of dataframe columns to
+        pre-fitted encoders and to keras model inputs. When model has a single input x_structure will look like
+        `x_structure=('column_name', encoder)`. When model has multiple inputs, keras expects a tuple of numpy arrays
+        as model X's. The structure will look like `x_structure=(('column_name1', encoder1), ('column_name2', encoder2)`
+        If encoder is None, the column values will be converted to numpy array and passed unchanged. If you want to
+        add a constant to inputs or outputs, you can add tuples with `column_name = None` and constant value instead
+        of encoder, like so: `(None, value)`.
+        **Example:** `x_structure=(('column_name1', encoder1), ('column_name2', None), (None, 1)` - values in
+        column_name1 are encoded by encoder1, values from column_name2 are passed through unchanged, the third column
+        in the x structure will be a constant of 1. So the batch could be
+        `(np.array(...), np.array(...), np.array(1, 1, ...))`
     - **y_structure** - (optional) *tuple* or *list of tuples* - a structure describing mapping of dataframe columns to
         pre-fitted encoders and to keras model output. When model has multiple output, keras expects
         a list of numpy arrays as model Y's. **Default: None**. Same rules and same format applies (see x_structure)

keras_batchflow/base/batch_generators/batch_generator.py

@@ -97,10 +97,11 @@ def _walk_structure(self, data: pd.DataFrame, struc, func, **kwargs):
             ret = func(data=data, leaf=struc, **kwargs)
             return ret
         elif type(struc) in [list, tuple]:
-            ret = [self._walk_structure(data, s, func, **kwargs) for s in struc]
-            # we always return lists as tuples as tensorflow wants x and y to be tuples in case of multiple
-            # components
-            return tuple(ret)
+            walked_structure = [self._walk_structure(data, s, func, **kwargs) for s in struc]
+            if isinstance(struc, tuple):
+                return tuple(walked_structure)
+            else:
+                return walked_structure
         else:
             raise ValueError('Error: structure definition in {} class only supports lists and tuples, but {}'
                              'was found'.format(type(self).__name__, type(struc)))

keras_batchflow/base/batch_shapers/batch_shaper.py

@@ -165,7 +165,7 @@ def test_transform(self):
         le = LabelEncoder().fit(self.df['var2'])
         bg = BatchGenerator(
             self.df,
-            x_structure=[('var1', self.lb), ('var2', le)],
+            x_structure=(('var1', self.lb), ('var2', le)),
             y_structure=('label', self.le),
             shuffle=False,
             batch_size=3,
@@ -183,6 +183,19 @@ def test_transform(self):
         assert type(batch[0]) == tuple
         assert len(batch[0]) == 2
 
+        #test the same with x_structure a list
+        bg = BatchGenerator(
+            self.df,
+            x_structure=[('var1', self.lb), ('var2', le)],
+            y_structure=('label', self.le),
+            shuffle=False,
+            batch_size=3,
+        )
+        batch = bg.transform(self.df)
+        assert type(batch) == tuple
+        assert len(batch) == 2
+        assert type(batch[0]) == list
+
     def test_inverse_transform(self):
         # batch size equals to dataset size
         bg = BatchGenerator(
@@ -215,7 +228,7 @@ def test_shapes(self):
         le = LabelEncoder().fit(self.df['var2'])
         bg = BatchGenerator(
             self.df,
-            x_structure=[('var1', self.lb), ('var2', le)],
+            x_structure=(('var1', self.lb), ('var2', le)),
             y_structure=('label', self.le),
             shuffle=False,
         )
@@ -227,6 +240,17 @@ def test_shapes(self):
         assert sh[0][0] == (3,)
         assert sh[0][1] == (1,)
         assert sh[1] == (1,)
+        #test the same with x_structure a list
+        bg = BatchGenerator(
+            self.df,
+            x_structure=[('var1', self.lb), ('var2', le)],
+            y_structure=('label', self.le),
+            shuffle=False,
+        )
+        sh = bg.shapes
+        assert type(sh) == tuple
+        assert len(sh) == 2
+        assert type(sh[0]) == list
 
     def test_encoder_adaptor(self):
         """

tests/batch_generators/test_batch_generator.py

@@ -71,7 +71,7 @@ def test_fit_multiple_epochs(self):
         keras_model.fit(keras_bg, epochs=2)
 
     def test_multiple_inputs(self):
-        keras_bg = BatchGenerator(self.data, x_structure=[('x1', None), ('x2', None)],
+        keras_bg = BatchGenerator(self.data, x_structure=(('x1', None), ('x2', None)),
                                   y_structure=('y1', self.y1_enc))
         inp1 = keras.layers.Input(shape=(1,))
         inp2 = keras.layers.Input(shape=(1,))
@@ -82,8 +82,8 @@ def test_multiple_inputs(self):
         keras_model.fit(keras_bg, epochs=2)
 
     def test_multiple_inputs_outputs(self):
-        keras_bg = BatchGenerator(self.data, x_structure=[('x1', None), ('x2', None)],
-                                  y_structure=[('y1', self.y1_enc), ('y2', self.y2_enc)])
+        keras_bg = BatchGenerator(self.data, x_structure=(('x1', None), ('x2', None)),
+                                  y_structure=(('y1', self.y1_enc), ('y2', self.y2_enc)))
         inp1 = keras.layers.Input(shape=(1,))
         inp2 = keras.layers.Input(shape=(1,))
         x = keras.layers.Concatenate()([inp1, inp2])

tests/batch_generators/test_batch_generator_keras.py

@@ -76,7 +76,7 @@ def test_fit_multiple_epochs(self):
         tf_model.fit(tf_bg, epochs=2)
 
     def test_multiple_inputs(self):
-        keras_bg = BatchGeneratorTF(self.data, x_structure=[('x1', None), ('x2', None)],
+        keras_bg = BatchGeneratorTF(self.data, x_structure=(('x1', None), ('x2', None)),
                                   y_structure=('y1', self.y1_enc))
         inp1 = tf.keras.layers.Input(shape=(1,))
         inp2 = tf.keras.layers.Input(shape=(1,))
@@ -87,8 +87,8 @@ def test_multiple_inputs(self):
         keras_model.fit(keras_bg, epochs=2)
 
     def test_multiple_inputs_outputs(self):
-        keras_bg = BatchGeneratorTF(self.data, x_structure=[('x1', None), ('x2', None)],
-                                  y_structure=[('y1', self.y1_enc), ('y2', self.y2_enc)])
+        keras_bg = BatchGeneratorTF(self.data, x_structure=(('x1', None), ('x2', None)),
+                                  y_structure=(('y1', self.y1_enc), ('y2', self.y2_enc)))
         inp1 = tf.keras.layers.Input(shape=(1,))
         inp2 = tf.keras.layers.Input(shape=(1,))
         x = tf.keras.layers.Concatenate()([inp1, inp2])

tests/batch_generators/test_batch_generator_tf.py

@@ -69,7 +69,7 @@ def test_2d_transformer(self, data, one_hot_encoder, label_encoder):
 
     def test_many_x(self, data, label_binarizer, label_encoder):
         lb2 = LabelBinarizer().fit(data['var2'])
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('var2', lb2)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('var2', lb2)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         batch = bs.transform(data)
@@ -87,7 +87,7 @@ def test_many_x(self, data, label_binarizer, label_encoder):
     def test_many_y(self, data, label_binarizer, label_encoder):
         lb2 = LabelBinarizer().fit(data['var2'])
         bs = BatchShaper(x_structure=('var1', label_binarizer),
-                         y_structure=[('label', label_encoder), ('var2', lb2)],
+                         y_structure=(('label', label_encoder), ('var2', lb2)),
                          data_sample=data)
         batch = bs.transform(data)
         assert type(batch) == tuple
@@ -111,7 +111,7 @@ def test_predict_batch(self, data, label_binarizer, label_encoder):
         so for predict, the generator must return a tuple (x,), where x is a list of inputs
         """
         lb2 = LabelBinarizer().fit(data['var2'])
-        batch_shaper = BatchShaper(x_structure=[('var1', label_binarizer), ('var2', lb2)], data_sample=data)
+        batch_shaper = BatchShaper(x_structure=(('var1', label_binarizer), ('var2', lb2)), data_sample=data)
         batch = batch_shaper.transform(data)
         assert isinstance(batch, tuple)
         assert len(batch) == 1
@@ -146,7 +146,7 @@ def test_init_with_data_sample(self):
         pass
 
     def test_none_transformer(self, data, label_binarizer, label_encoder):
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('var2', None)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('var2', None)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         batch = bs.transform(data)
@@ -157,7 +157,7 @@ def test_none_transformer(self, data, label_binarizer, label_encoder):
         assert np.array_equal(batch[0][1], np.expand_dims(data['var2'].values, axis=-1))
 
     def test_const_component_int(self, data, label_binarizer, label_encoder):
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), (None, 0)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), (None, 0)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         batch = bs.transform(data)
@@ -169,7 +169,7 @@ def test_const_component_int(self, data, label_binarizer, label_encoder):
         assert batch[0][1].dtype == int
 
     def test_const_component_float(self, data, label_binarizer, label_encoder):
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), (None, 0.)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), (None, 0.)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         batch = bs.transform(data)
@@ -181,7 +181,7 @@ def test_const_component_float(self, data, label_binarizer, label_encoder):
         assert batch[0][1].dtype == float
 
     def test_const_component_str(self, data, label_binarizer, label_encoder):
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), (None, u'a')],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), (None, u'a')),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         batch = bs.transform(data)
@@ -194,7 +194,7 @@ def test_const_component_str(self, data, label_binarizer, label_encoder):
 
     def test_metadata(self, data, label_binarizer, label_encoder):
         VarShaper._dummy_constant_counter = 0
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), (None, 0.)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), (None, 0.)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         md = bs.metadata
@@ -228,7 +228,7 @@ def test_metadata(self, data, label_binarizer, label_encoder):
 
     def test_dummy_var_naming(self, data, label_binarizer, label_encoder):
         VarShaper._dummy_constant_counter = 0
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), (None, 0.), (None, 1.)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), (None, 0.), (None, 1.)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         md = bs.metadata
@@ -258,7 +258,7 @@ def inverse_transform(self, data):
                 return data
 
         a = A()
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('var1', a)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('var1', a)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         shapes = bs.shape
@@ -283,22 +283,22 @@ def inverse_transform(self, data):
                 return data
 
         a = A()
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('var1', a)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('var1', a)),
                          y_structure=('label', label_encoder), data_sample=data)
         n_classes = bs.n_classes
         pass
 
     def test_inverse_transform(self, data, label_binarizer, label_encoder):
         le2 = LabelEncoder().fit(data['var2'])
         bs = BatchShaper(x_structure=('var1', label_binarizer),
-                         y_structure=[('label', label_encoder), ('var2', le2)],
+                         y_structure=(('label', label_encoder), ('var2', le2)),
                          data_sample=data)
         batch = bs.transform(data)
         inverse = bs.inverse_transform(batch[1])
         assert inverse.equals(data[['label', 'var2']])
         # Check inverse transform when constant field is in the structure
         bs = BatchShaper(x_structure=('var1', label_binarizer),
-                         y_structure=[('label', label_encoder), ('var2', le2), (None, 0.)],
+                         y_structure=(('label', label_encoder), ('var2', le2), (None, 0.)),
                          data_sample=data)
         batch = bs.transform(data)
         # check that the constant field was added to the y output
@@ -309,7 +309,7 @@ def test_inverse_transform(self, data, label_binarizer, label_encoder):
         assert inverse.equals(data[['label', 'var2']])
         # Check inverse transform when direct mapping field is in the structure
         bs = BatchShaper(x_structure=('var1', label_binarizer),
-                         y_structure=[('label', label_encoder), ('var2', le2), ('var1', None)],
+                         y_structure=(('label', label_encoder), ('var2', le2), ('var1', None)),
                          data_sample=data)
         batch = bs.transform(data)
         # check that the constant field was added to the y output
@@ -366,7 +366,7 @@ def test_batch_forking(self, data, label_binarizer, label_encoder):
         # check that data is not modified
         assert data.equals(data_snapshot)
         assert data_xy_fork.columns.nlevels == 2
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('label', label_encoder)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('label', label_encoder)),
                          y_structure=('label', label_encoder),
                          data_sample=data)
         tr = bs.transform(data_xy_fork)
@@ -384,7 +384,7 @@ def test_batch_forking(self, data, label_binarizer, label_encoder):
         batch_fork_01 = BatchFork(levels=(0, 1))
         data_01_fork = batch_fork_01.transform(data)
         assert data_01_fork.columns.nlevels == 2
-        bs = BatchShaper(x_structure=[('var1', label_binarizer), ('label', label_encoder)],
+        bs = BatchShaper(x_structure=(('var1', label_binarizer), ('label', label_encoder)),
                          y_structure=('label', label_encoder),
                          multiindex_xy_keys=(0, 1),
                          data_sample=data)