From 636fe17bab0899a96ff9a9428d0cb9ba1e0813f2 Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 16:59:03 +0200 Subject: [PATCH 1/8] rm unused attribute --- mealy/error_tree.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mealy/error_tree.py b/mealy/error_tree.py index aa891b0..1688a6b 100644 --- a/mealy/error_tree.py +++ b/mealy/error_tree.py @@ -17,7 +17,6 @@ def __init__(self, error_decision_tree): self._difference = None self._total_error_fraction = None self._error_class_idx = None - self.correct_class_idx = None self._wrongly_predicted_leaves = None self._correctly_predicted_leaves = None From b3f047c5333e608141e280b98484092c13e53e9c Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 17:37:30 +0200 Subject: [PATCH 2/8] update docstring in analyzer --- mealy/error_analyzer.py | 130 ++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/mealy/error_analyzer.py b/mealy/error_analyzer.py index a0ff471..54f543f 100644 --- a/mealy/error_analyzer.py +++ b/mealy/error_analyzer.py @@ -20,25 +20,29 @@ class ErrorAnalyzer(BaseEstimator): - """ ErrorAnalyzer analyzes the errors of a prediction model on a test set. + """ErrorAnalyzer analyzes the errors of a prediction model on a test set. It uses model predictions and ground truth target to compute the model errors on the test set. It then trains a Decision Tree, called a Error Analyzer Tree, on the same test set by using the model error as target. The nodes of the decision tree are different segments of errors to be studied individually. Args: - primary_model (sklearn.base.BaseEstimator or sklearn.pipeline.Pipeline): a sklearn model to analyze. Either an estimator + primary_model (sklearn.base.BaseEstimator or sklearn.pipeline.Pipeline): A sklearn model to analyze. Either an estimator or a Pipeline containing a ColumnTransformer with the preprocessing steps and an estimator as last step. - feature_names (list): list of feature names, default=None. - max_num_row (int): maximum number of rows to process. - param_grid (dict): sklearn.tree.DecisionTree hyper-parameters values for grid search. - random_state (int): random seed. + feature_names (list): List of feature names, default=None. + max_num_row (int): Maximum number of rows to process. + param_grid (dict): The sklearn.tree.DecisionTree hyper-parameters values for grid search. + random_state (int): Random seed. Attributes: - total_error_fraction (numpy.ndarray): percentage of incorrectly predicted samples in leaves over the total number of - errors (used for ranking the nodes). - leaf_ids (numpy.ndarray): list of all leaves indices. - _error_tree (DecisionTreeClassifier): the estimator used to train the Error Analyzer Tree + primary_model (sklearn.base.BaseEstimator or sklearn.pipeline.Pipeline): A sklearn model to analyze. Either an estimator + or a Pipeline containing a ColumnTransformer with the preprocessing steps and an estimator as last step. + feature_names (list): List of feature names. + max_num_row (int): Maximum number of rows to process. + param_grid (dict): The sklearn.tree.DecisionTreeClassifier hyper-parameters values for grid search. + random_state (int): Random seed. + preprocessed_feature_names (list): List of preprocessed feature names. + error_tree (sklearn.tree.DecisionTreeClassifier): The estimator used to train the Error Analyzer Tree. """ def __init__(self, primary_model, @@ -134,16 +138,15 @@ def preprocessed_feature_names(self): return self._preprocessed_feature_names def fit(self, X, y): - """ - Fit the Error Analyzer Tree. + """Fit the Error Analyzer Tree. Trains the Error Analyzer Tree, a Decision Tree to discriminate between samples that are correctly predicted or wrongly predicted (errors) by a primary model. Args: - X (numpy.ndarray or pandas.DataFrame): feature data from a test set to evaluate the primary predictor and + X (numpy.ndarray or pandas.DataFrame): Feature data from a test set to evaluate the primary predictor and train a Error Analyzer Tree. - y (numpy.ndarray or pandas.DataFrame): target data from a test set to evaluate the primary predictor and + y (numpy.ndarray or pandas.DataFrame): Target data from a test set to evaluate the primary predictor and train a Error Analyzer Tree. """ logger.info("Preparing the Error Analyzer Tree...") @@ -179,7 +182,7 @@ def fit(self, X, y): #TODO rewrite this method using the ranking arrays def get_error_leaf_summary(self, leaf_selector=None, add_path_to_leaves=False, output_format='dict', rank_by='total_error_fraction'): - """ Return summary information regarding leaves. + """Return summary information regarding leaves. Args: leaf_selector (None, int or array-like): The leaves whose information will be returned @@ -195,7 +198,7 @@ def get_error_leaf_summary(self, leaf_selector=None, add_path_to_leaves=False, o in a node. Return: - dict or str: list of reports (as dictionary or string) with different information on each selected leaf. + dict or str: List of reports (as dictionary or string) with different information on each selected leaf. """ leaf_nodes = self._get_ranked_leaf_ids(leaf_selector=leaf_selector, rank_by=rank_by) @@ -235,19 +238,19 @@ def get_error_leaf_summary(self, leaf_selector=None, add_path_to_leaves=False, o return leaves_summary def evaluate(self, X, y, output_format='str'): - """ - Evaluate performance of ErrorAnalyzer on new the given test data and labels. + """Evaluate performance of ErrorAnalyzer on new the given test data and labels. + Print ErrorAnalyzer summary metrics regarding the Error Tree. Args: - X (numpy.ndarray or pandas.DataFrame): feature data from a test set to evaluate the primary predictor + X (numpy.ndarray or pandas.DataFrame): Feature data from a test set to evaluate the primary predictor and train a Error Analyzer Tree. - y (numpy.ndarray or pandas.DataFrame): target data from a test set to evaluate the primary predictor and + y (numpy.ndarray or pandas.DataFrame): Target data from a test set to evaluate the primary predictor and train a Error Analyzer Tree. output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'str'. Return: - dict or str: dictionary or string report storing different metrics regarding the Error Decision Tree. + dict or str: Dictionary or string report storing different metrics regarding the Error Decision Tree. """ prep_x, prep_y = self.pipeline_preprocessor.transform(X), np.array(y) y_true, _ = self._compute_primary_model_error(prep_x, prep_y) @@ -255,44 +258,32 @@ def evaluate(self, X, y, output_format='str'): return error_decision_tree_report(y_true, y_pred, output_format) def _compute_primary_model_error(self, X, y): - """ - Computes the errors of the primary model predictions and samples + """Computes the errors of the primary model predictions and samples. Args: - X: array-like of shape (n_samples, n_features) - Input samples. - - y: array-like of shape (n_samples,) - True target values for `X`. + X (numpy.ndarray): Input samples of shape `(n_samples, n_features)`. + y (numpy.ndarray): True target values for `X` of shape `(n_samples,)`. Returns: - sampled_X: ndarray - A sample of `X`. - - error_y: array of string of shape (n_sampled_X, ) - Boolean value of whether or not the primary model predicted correctly or incorrectly the samples in sampled_X. + error_y (numpy.ndarray): Array of booleans of shape `(len(y),)`, containing a boolean value of whether or + not the primary model got the prediction right. + error_rate (float): Accuracy of the primary model. """ y_pred = self._primary_model.predict(X) error_y, error_rate = self._evaluate_primary_model_predictions(y_true=y, y_pred=y_pred) return error_y, error_rate def _evaluate_primary_model_predictions(self, y_true, y_pred): - """ - Compute errors of the primary model on the test set + """Compute errors of the primary model on the test set. Args: - y_true: 1D array - True target values. + y_true (numpy.ndarray): True target values. + y_pred (numpy.ndarray): Predictions of the primary model. - y_pred: 1D array - Predictions of the primary model. - - Return: - error_y: array of string of len(y_true) - Boolean value of whether or not the primary model got the prediction right. - - error_rate: float - Accuracy of the primary model + Returns: + error_y (numpy.ndarray): Array of booleans of shape `(len(y),)`, containing a boolean value of whether or + not the primary model got the prediction right. + error_rate (float): Accuracy of the primary model. """ error_y = np.full_like(y_true, ErrorAnalyzerConstants.CORRECT_PREDICTION, dtype="O") @@ -317,18 +308,18 @@ def _get_ranked_leaf_ids(self, leaf_selector=None, rank_by='total_error_fraction """ Select error nodes and rank them by importance. Args: - leaf_selector (None, int or array-like): the leaves whose information will be returned + leaf_selector (None, int or array-like): The leaves whose information will be returned * int: Only return information of the leaf with the corresponding id * array-like: Only return information of the leaves corresponding to these ids * None (default): Return information of all the leaves - rank_by (str): ranking criterion for the leaves. Valid values are: + rank_by (str): Ranking criterion for the leaves. Valid values are: * 'total_error_fraction': rank by the fraction of total error in the node * 'purity': rank by the purity (ratio of wrongly predicted samples over the total number of node samples) * 'class_difference': rank by the difference of number of wrongly and correctly predicted samples in a node. - Return: - list or numpy.ndarray: list of selected leaves indices. + Returns: + list or numpy.ndarray: List of selected leaves indices. """ apply_leaf_selector = self._get_leaf_selector(leaf_selector) @@ -348,20 +339,20 @@ def _get_ranked_leaf_ids(self, leaf_selector=None, rank_by='total_error_fraction #TODO leaf_selector is taking too many different types of data ? def _get_leaf_selector(self, leaf_selector): - """ - Return a function that select rows of provided arrays. Arrays must be of shape (1, number of leaves) - Args: - leaf_selector: None, int or array-like - How to select the rows of the array - * int: Only keep the row corresponding to this leaf id - * array-like: Only keep the rows corresponding to these leaf ids - * None (default): Keep the whole array of leaf ids - - Return: - A function with one argument array as a selector of leaf ids + """Return a function that select rows of provided arrays. + + Arrays must be of shape (1, number of leaves). + + Args: + leaf_selector (None, int or array-like): How to select the rows of the array + * int: Only keep the row corresponding to this leaf id + * array-like: Only keep the rows corresponding to these leaf ids + * None (default): Keep the whole array of leaf ids + + Returns: + A function with one argument `array` as a selector of leaf ids. Args: - array: numpy array of shape (1, number of leaves) - An array of which we only want to keep some rows + array (numpy.array): array of shape (1, number of leaves) of which we only want to keep some rows. """ if leaf_selector is None: return lambda array: array @@ -376,7 +367,14 @@ def _get_leaf_selector(self, leaf_selector): return lambda array: array[leaf_selector] def _get_path_to_node(self, node_id): - """ Return path to node as a list of split steps from the nodes of the sklearn Tree object """ + """ Return path to node as a list of split steps from the nodes of the sklearn Tree object. + + Args: + node_id (int): Node identifier. + + Returns: + path_to_node (str): Text describing the path from the root to the input node. + """ feature_names = self.pipeline_preprocessor.get_original_feature_names() children_left = list(self._error_tree.estimator_.tree_.children_left) children_right = list(self._error_tree.estimator_.tree_.children_right) @@ -421,7 +419,7 @@ def _inverse_transform_features(self): indicate what features are used to split the training set at each node. See https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html. - Return: + Returns: list or numpy.ndarray: indices of features of the Error Analyzer Tree, possibly mapped back to the original unprocessed feature space. @@ -443,7 +441,7 @@ def _inverse_transform_thresholds(self): the decision tree. The thresholds of a decision tree are the feature values used to split the training set at each node. See https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html. - Return: + Returns: numpy.ndarray: thresholds of the Error Tree, possibly with preprocessing undone. """ From 9fe5ea4f3f0c64e7a2ed1a1c7d41ae093aa29c60 Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 17:53:24 +0200 Subject: [PATCH 3/8] add error tree docstrings --- mealy/error_analyzer.py | 2 +- mealy/error_tree.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mealy/error_analyzer.py b/mealy/error_analyzer.py index 54f543f..28c2885 100644 --- a/mealy/error_analyzer.py +++ b/mealy/error_analyzer.py @@ -42,7 +42,7 @@ class ErrorAnalyzer(BaseEstimator): param_grid (dict): The sklearn.tree.DecisionTreeClassifier hyper-parameters values for grid search. random_state (int): Random seed. preprocessed_feature_names (list): List of preprocessed feature names. - error_tree (sklearn.tree.DecisionTreeClassifier): The estimator used to train the Error Analyzer Tree. + error_tree (ErrorTree): the Error Tree. """ def __init__(self, primary_model, diff --git a/mealy/error_tree.py b/mealy/error_tree.py index 1688a6b..6bce4fa 100644 --- a/mealy/error_tree.py +++ b/mealy/error_tree.py @@ -6,8 +6,32 @@ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO, format='mealy | %(levelname)s - %(message)s') + class ErrorTree(object): + """ ErrorTree analyzes the errors of a prediction model on a test set. + + It uses model predictions and ground truth target to compute the model errors on the test set. + It then trains a Decision Tree, called a Error Analyzer Tree, on the same test set by using the model error + as target. The nodes of the decision tree are different segments of errors to be studied individually. + + Args: + error_decision_tree (sklearn.tree.DecisionTreeClassifier): The estimator used to train the Error Tree. + + Attributes: + estimator_ (sklearn.tree.DecisionTreeClassifier): The estimator used to train the Error Tree. + impurity (numpy.ndarray): Impurity of leaves. + quantized_impurity (numpy.ndarray): Impurity of leaves quantized into mealy.constants.NUMBER_PURITY_LEVELS + levels. + difference (numpy.ndarray): Difference of number of wrongly and correctly predicted samples in leaves. + total_error_fraction (numpy.ndarray): Percentage of incorrectly predicted samples in leaves over the total + number of errors (used to rank the nodes). + error_class_idx (int): Index of class of wrongly predicted samples in the Error Tree. + n_total_errors (int): Number of total errors. + wrongly_predicted_leaves (numpy.ndarray): Array of number of wrongly predicted samples in leaves. + correctly_predicted_leaves (numpy.ndarray): Array of number of correctly predicted samples in leaves. + leaf_ids (numpy.ndarray): List of all leaves indices. + """ def __init__(self, error_decision_tree): self._estimator = error_decision_tree From 27e65af83a4d465c39587d37a86c10eb29443a7e Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 21:27:55 +0200 Subject: [PATCH 4/8] update more docstring to google style --- mealy/error_analysis_utils.py | 42 ++++++++++++----- mealy/error_analyzer.py | 4 +- mealy/error_tree.py | 2 +- mealy/error_visualizer.py | 5 +- mealy/metrics.py | 2 +- mealy/preprocessing.py | 88 +++++++++++++++++------------------ 6 files changed, 81 insertions(+), 62 deletions(-) diff --git a/mealy/error_analysis_utils.py b/mealy/error_analysis_utils.py index d5fe5a0..ef0ce26 100644 --- a/mealy/error_analysis_utils.py +++ b/mealy/error_analysis_utils.py @@ -6,14 +6,15 @@ def get_epsilon(difference): - """ - Compute the threshold used to decide whether a prediction is wrong or correct (for regression tasks). + """Compute the threshold used to decide whether a prediction is wrong or correct (for regression tasks). Args: - difference (1D-array): The absolute differences between the true target values and the predicted ones (by the primary model). + difference (numpy.ndarray): The absolute differences between the true target values and the predicted ones + (by the primary model). - Return: - epsilon (float): The value of the threshold used to decide whether the prediction for a regression task is wrong or correct + Returns: + epsilon (float): The value of the threshold used to decide whether the prediction for a regression task + is wrong or correct """ epsilon_range = np.linspace(min(difference), max(difference), num=ErrorAnalyzerConstants.NUMBER_EPSILON_VALUES) cdf_error = [] @@ -23,7 +24,18 @@ def get_epsilon(difference): cdf_error.append(np.count_nonzero(correct_predictions) / float(n_samples)) return KneeLocator(epsilon_range, cdf_error).knee + def get_feature_list_from_column_transformer(ct_preprocessor): + """Get list of feature names and categorical feature names from a ColumnTransformer preprocessor. + + Args: + ct_preprocessor (sklearn.compose.ColumnTransformer): ColumnTransformer containing separate feature + preprocessing steps. + + Returns: + all_features (list): list of feature names. + categorical_features (list): list of categorical feature names. + """ all_features, categorical_features = [], [] for transformer_name, transformer, transformer_feature_names in ct_preprocessor.transformers_: if transformer_name == 'remainder' and transformer == 'drop': @@ -42,20 +54,28 @@ def get_feature_list_from_column_transformer(ct_preprocessor): def check_lists_having_same_elements(list_A, list_B): + """Check two lists have the same unique elements.""" return set(list_A) == set(list_B) + def check_enough_data(df, min_len): - """ - Compare length of dataframe to minimum lenght of the test data. + """Compare length of dataframe to minimum length of the test data. + Used in the relevance of the measure. - :param df: Input dataframe - :param min_len: - :return: + Args: + df (pandas.DataFrame): Input dataframe + min_len (int): Minimum number of rows required. + + Raises: + ValueError: If df does not have the required minimum number of rows. """ if df.shape[0] < min_len: - raise ValueError('The original dataset is too small ({} rows) to have stable result, it needs to have at least {} rows'.format(df.shape[0], min_len)) + raise ValueError( + 'The original dataset is too small ({} rows) to have stable result, it needs to have at least {} rows'.format( + df.shape[0], min_len)) def rank_features_by_error_correlation(feature_importances): + """Rank features by feature importance according to the error model.""" return np.argsort(- feature_importances) diff --git a/mealy/error_analyzer.py b/mealy/error_analyzer.py index 28c2885..0610b05 100644 --- a/mealy/error_analyzer.py +++ b/mealy/error_analyzer.py @@ -197,7 +197,7 @@ def get_error_leaf_summary(self, leaf_selector=None, add_path_to_leaves=False, o * 'class_difference': rank by the difference of number of wrongly and correctly predicted samples in a node. - Return: + Returns: dict or str: List of reports (as dictionary or string) with different information on each selected leaf. """ @@ -249,7 +249,7 @@ def evaluate(self, X, y, output_format='str'): train a Error Analyzer Tree. output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'str'. - Return: + Returns: dict or str: Dictionary or string report storing different metrics regarding the Error Decision Tree. """ prep_x, prep_y = self.pipeline_preprocessor.transform(X), np.array(y) diff --git a/mealy/error_tree.py b/mealy/error_tree.py index 6bce4fa..47b11e6 100644 --- a/mealy/error_tree.py +++ b/mealy/error_tree.py @@ -20,7 +20,7 @@ class ErrorTree(object): Attributes: estimator_ (sklearn.tree.DecisionTreeClassifier): The estimator used to train the Error Tree. impurity (numpy.ndarray): Impurity of leaves. - quantized_impurity (numpy.ndarray): Impurity of leaves quantized into mealy.constants.NUMBER_PURITY_LEVELS + quantized_impurity (numpy.ndarray): Impurity of leaves quantized into ErrorAnalyzerConstants.NUMBER_PURITY_LEVELS levels. difference (numpy.ndarray): Difference of number of wrongly and correctly predicted samples in leaves. total_error_fraction (numpy.ndarray): Percentage of incorrectly predicted samples in leaves over the total diff --git a/mealy/error_visualizer.py b/mealy/error_visualizer.py index da8ac38..bcb16df 100644 --- a/mealy/error_visualizer.py +++ b/mealy/error_visualizer.py @@ -68,8 +68,7 @@ def _plot_feature_distribution(x_ticks, feature_is_numerical, leaf_data, root_da class ErrorVisualizer(_BaseErrorVisualizer): - """ - ErrorVisualizer provides visual utilities to analyze the Error Tree in ErrorAnalyzer + """ErrorVisualizer provides visual utilities to analyze the Error Tree in ErrorAnalyzer. Args: error_analyzer (ErrorAnalyzer): fitted ErrorAnalyzer representing the performance of a primary model. @@ -97,7 +96,7 @@ def plot_error_tree(self, size=None): Args: size (tuple): size of the output plot. - Return: + Returns: graphviz.Source: graph of the Error Analyzer Tree. """ diff --git a/mealy/metrics.py b/mealy/metrics.py index 1922bb5..cb7f865 100644 --- a/mealy/metrics.py +++ b/mealy/metrics.py @@ -45,7 +45,7 @@ def error_decision_tree_report(y_true, y_pred, output_format='str'): [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. - Return: + Returns: dict or str: dictionary or string report storing different metrics regarding the Error Decision Tree. """ diff --git a/mealy/preprocessing.py b/mealy/preprocessing.py index d8ae157..e51555c 100644 --- a/mealy/preprocessing.py +++ b/mealy/preprocessing.py @@ -12,19 +12,19 @@ class FeatureNameTransformer(object): - """ Transformer of feature names and indices. + """Transformer of feature names and indices. - A FeatureNameTransformer parses an input Pipeline preprocessor and generate - a mapping between the input unprocessed feature names/indices and the output - preprocessed feature names/indices. + A FeatureNameTransformer parses an input Pipeline preprocessor and generate + a mapping between the input unprocessed feature names/indices and the output + preprocessed feature names/indices. - Args: - ct_preprocessor (sklearn.compose.ColumnTransformer): preprocessor - orig_feats (list): list of original unpreprocessed feature names, default=None. + Args: + ct_preprocessor (sklearn.compose.ColumnTransformer): Preprocessor. + original_features (list): List of original unpreprocessed feature names, default=None. - Attributes: - original_feature_names (list): list of original unpreprocessed feature names. - preprocessed_feature_names (list): list of preprocessed feature names + Attributes: + original_feature_names (list): List of original unpreprocessed feature names. + preprocessed_feature_names (list): List of preprocessed feature names. """ def __init__(self, ct_preprocessor, original_features=None): @@ -51,17 +51,18 @@ def __init__(self, ct_preprocessor, original_features=None): self._create_feature_mapping(ct_preprocessor) def get_original_feature_names(self): + """Get the list of original unpreprocessed feature names.""" return self.original_feature_names def get_preprocessed_feature_names(self): + """Get the list of preprocessed feature names.""" return self.preprocessed_feature_names def _create_feature_mapping(self, ct_preprocessor): - """ - Update the dicts of input <-> output feature id mapping: self.original2preprocessed and self.preprocessed2original + """Update the dicts of input <-> output feature id mapping: original2preprocessed and preprocessed2original. Args: - ct_preprocessor: a ColumnTransformer object + ct_preprocessor (sklearn.compose.ColumnTransformer): a ColumnTransformer object. """ for i, (transformer_name, transformer, transformer_feature_names) in enumerate(ct_preprocessor.transformers_): orig_feats_ids = np.where(np.in1d(self.original_feature_names, transformer_feature_names))[0] @@ -116,12 +117,15 @@ def _update_feature_mapping_dict_using_output_names(self, single_transformer, tr self.len_preproc += len(part_out_feature_names) def transform_feature_id(self, index=None, name=None): - """ + """Get the preprocessed feature names corresponding to an original feature. + + The input feature can be specified either by index or name. + Args: - index: int - name: str + index (int): Feature index. + name (str): Feature name. - Returns: index of output feature(s) generated by the requested feature + Returns: index of output feature(s) generated after preprocessing of the input feature """ if index is not None: return self.original2preprocessed[index] @@ -139,10 +143,10 @@ def inverse_transform_feature_id(self, index=None, name=None): back into the original unprocessed feature name or index. Args: - index (int): feature index. - name (str): feature name. + index (int): Feature index. + name (str): Feature name. - Return: + Returns: int or str: index (resp. name) of the unprocessed feature corresponding to the input preprocessed feature index (resp.name). If both index and name are provided, the index is retained and an output index is returned. @@ -160,10 +164,10 @@ def is_categorical(self, index=None, name=None): """Check whether an unprocessed feature at a given index or with a given name is categorical. Args: - index (int): feature index. - name (str): feature name. + index (int): Feature index. + name (str): Feature name. - Return: + Returns: bool: True if the input feature is categorical, else False. If both index and name are provided, the index is retained. """ @@ -177,20 +181,15 @@ def is_categorical(self, index=None, name=None): class PipelinePreprocessor(FeatureNameTransformer): - """ Transformer of feature values from the original values to preprocessed ones. + """Transformer of feature values from the original values to preprocessed ones. - A PipelinePreprocessor parses an input Pipeline preprocessor and generate - a mapping between the input unprocessed feature values and the output - preprocessed feature values. - - Args: - ct_preprocessor (sklearn.compose.ColumnTransformer): preprocessing steps - orig_feats (list): list of original unpreprocessed feature names, default=None. - - Attributes: - fn_transformer (FeatureNameTransformer): transformer managing the mapping between original and - preprocessed feature names. + A PipelinePreprocessor parses an input Pipeline preprocessor and generate + a mapping between the input unprocessed feature values and the output + preprocessed feature values. + Args: + ct_preprocessor (sklearn.compose.ColumnTransformer): Preprocessing steps + original_features (list): List of original unpreprocessed feature names, default=None. """ def __init__(self, ct_preprocessor, original_features=None): @@ -200,10 +199,10 @@ def transform(self, x): """Transform the input feature values according to the preprocessing pipeline. Args: - x (numpy.ndarray or pandas.DataFrame): input feature values. + x (numpy.ndarray or pandas.DataFrame): Input feature values. - Return: - numpy.ndarray: transformed feature values + Returns: + numpy.ndarray: Transformed feature values. """ return self.ct_preprocessor.transform(x) @@ -223,10 +222,10 @@ def inverse_transform(self, preprocessed_x): """Invert the preprocessing pipeline and inverse transform feature values. Args: - preprocessed_x (numpy.ndarray): preprocessed feature values. + preprocessed_x (numpy.ndarray): Preprocessed feature values. - Return: - numpy.ndarray: feature values without preprocessing + Returns: + numpy.ndarray: Feature values without preprocessing. """ def _inverse_single_step(single_step, step_output): @@ -274,12 +273,13 @@ def __init__(self, model_performance_predictor_features): self.model_performance_predictor_features = model_performance_predictor_features def transform(self, x): - """ + """ Identity transformation of the input. Args: - x: dataframe or ndarray + x (numpy.ndarray or pandas.DataFrame): Input feature values. + Returns: - ndarray + numpy.ndarray: Transformed feature values. """ if isinstance(x, pd.DataFrame): return x.values From d8ae9ba632cb1bc09a16b9d14aa1ff7a0aa89a03 Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 21:34:20 +0200 Subject: [PATCH 5/8] rm non existing input argument --- examples/plot_mealy.py | 2 +- examples/plot_mealy_pipeline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/plot_mealy.py b/examples/plot_mealy.py index 5a76687..cea84c1 100644 --- a/examples/plot_mealy.py +++ b/examples/plot_mealy.py @@ -87,7 +87,7 @@ ############################################################################## # Print the details regarding the decision tree nodes containing the majority of errors. -error_analyzer.get_error_leaf_summary(leaf_selector=None, add_path_to_leaves=True, print_summary=True); +error_analyzer.get_error_leaf_summary(leaf_selector=None, add_path_to_leaves=True); ############################################################################## # Plot the feature distributions of samples in the leaf containing the majority of errors. diff --git a/examples/plot_mealy_pipeline.py b/examples/plot_mealy_pipeline.py index d30fada..5cf17b3 100644 --- a/examples/plot_mealy_pipeline.py +++ b/examples/plot_mealy_pipeline.py @@ -132,7 +132,7 @@ ############################################################################## # Print the details regarding the decision tree nodes containing the majority of errors. -error_analyzer.get_error_leaf_summary(leaf_selector=None, add_path_to_leaves=True, print_summary=True); +error_analyzer.get_error_leaf_summary(leaf_selector=None, add_path_to_leaves=True); ############################################################################## # Plot the feature distributions of samples in the leaf containing the majority of errors. From 2474fcfb29bb1bb7fab7d532f053a038623eae56 Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Wed, 14 Apr 2021 22:02:07 +0200 Subject: [PATCH 6/8] more pydoc fixes --- mealy/error_analysis_utils.py | 4 ++-- mealy/error_analyzer.py | 4 ++-- mealy/metrics.py | 2 +- mealy/preprocessing.py | 9 ++++----- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mealy/error_analysis_utils.py b/mealy/error_analysis_utils.py index ef0ce26..620f256 100644 --- a/mealy/error_analysis_utils.py +++ b/mealy/error_analysis_utils.py @@ -13,8 +13,8 @@ def get_epsilon(difference): (by the primary model). Returns: - epsilon (float): The value of the threshold used to decide whether the prediction for a regression task - is wrong or correct + float: The value of the threshold used to decide whether the prediction for a regression task + is wrong or correct. """ epsilon_range = np.linspace(min(difference), max(difference), num=ErrorAnalyzerConstants.NUMBER_EPSILON_VALUES) cdf_error = [] diff --git a/mealy/error_analyzer.py b/mealy/error_analyzer.py index 0610b05..5617c15 100644 --- a/mealy/error_analyzer.py +++ b/mealy/error_analyzer.py @@ -190,7 +190,7 @@ def get_error_leaf_summary(self, leaf_selector=None, add_path_to_leaves=False, o * array-like: Only return information of the leaves corresponding to these ids * None (default): Return information of all the leaves add_path_to_leaves (bool): Whether to add information of the path across the tree till the selected node. Defaults to False. - output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'dict'. + output_format (str): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'dict'. rank_by (str): Ranking criterion for the leaves. Valid values are: * 'total_error_fraction' (default): rank by the fraction of total error in the node * 'purity': rank by the purity (ratio of wrongly predicted samples over the total number of node samples) @@ -247,7 +247,7 @@ def evaluate(self, X, y, output_format='str'): and train a Error Analyzer Tree. y (numpy.ndarray or pandas.DataFrame): Target data from a test set to evaluate the primary predictor and train a Error Analyzer Tree. - output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'str'. + output_format (str): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'str'. Returns: dict or str: Dictionary or string report storing different metrics regarding the Error Decision Tree. diff --git a/mealy/metrics.py b/mealy/metrics.py index cb7f865..2f5d1ec 100644 --- a/mealy/metrics.py +++ b/mealy/metrics.py @@ -43,7 +43,7 @@ def error_decision_tree_report(y_true, y_pred, output_format='str'): Expected values in [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. y_pred (numpy.ndarray): Estimated targets as returned by the error tree. Expected values in [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. - output_format (string): Return format used for the report. Valid values are 'dict' or 'str'. + output_format (str): Return format used for the report. Valid values are 'dict' or 'str'. Returns: dict or str: dictionary or string report storing different metrics regarding the Error Decision Tree. diff --git a/mealy/preprocessing.py b/mealy/preprocessing.py index e51555c..44e59e2 100644 --- a/mealy/preprocessing.py +++ b/mealy/preprocessing.py @@ -125,7 +125,7 @@ def transform_feature_id(self, index=None, name=None): index (int): Feature index. name (str): Feature name. - Returns: index of output feature(s) generated after preprocessing of the input feature + Returns: index of output feature(s) generated after preprocessing of the input feature. """ if index is not None: return self.original2preprocessed[index] @@ -148,8 +148,8 @@ def inverse_transform_feature_id(self, index=None, name=None): Returns: int or str: index (resp. name) of the unprocessed feature corresponding to the input preprocessed feature - index (resp.name). If both index and name are provided, the index is retained and an output index is - returned. + index (resp.name). If both index and name are provided, the index is retained and an output index is + returned. """ if index is not None: return self.preprocessed2original[index] @@ -169,7 +169,7 @@ def is_categorical(self, index=None, name=None): Returns: bool: True if the input feature is categorical, else False. If both index and name are provided, the index - is retained. + is retained. """ if index is not None: name = self.original_feature_names[index] @@ -226,7 +226,6 @@ def inverse_transform(self, preprocessed_x): Returns: numpy.ndarray: Feature values without preprocessing. - """ def _inverse_single_step(single_step, step_output): inverse_transform_function_available = getattr(single_step, "inverse_transform", None) From f9a073acd71784877afd0dc00fbf14cf9d297888 Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Thu, 15 Apr 2021 16:39:44 +0200 Subject: [PATCH 7/8] more docstrings in metrics --- mealy/error_analyzer.py | 2 +- mealy/metrics.py | 51 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/mealy/error_analyzer.py b/mealy/error_analyzer.py index 5617c15..6f41490 100644 --- a/mealy/error_analyzer.py +++ b/mealy/error_analyzer.py @@ -250,7 +250,7 @@ def evaluate(self, X, y, output_format='str'): output_format (str): Return format used for the report. Valid values are 'dict' or 'str'. Defaults to 'str'. Returns: - dict or str: Dictionary or string report storing different metrics regarding the Error Decision Tree. + dict or str: Dictionary or string report storing different metrics regarding the Error Tree. """ prep_x, prep_y = self.pipeline_preprocessor.transform(X), np.array(y) y_true, _ = self._compute_primary_model_error(prep_x, prep_y) diff --git a/mealy/metrics.py b/mealy/metrics.py index 2f5d1ec..e4af1a3 100644 --- a/mealy/metrics.py +++ b/mealy/metrics.py @@ -5,6 +5,19 @@ def compute_confidence_decision(primary_model_true_accuracy, primary_model_predicted_accuracy): + """Return fidelity of the Error Tree and decision regarding its reliability. + + Args: + primary_model_true_accuracy (numpy.ndarray): Ground truth values of wrong/correct predictions of the error tree + primary model. Expected values in [ErrorAnalyzerConstants.WRONG_PREDICTION, + ErrorAnalyzerConstants.CORRECT_PREDICTION]. + primary_model_predicted_accuracy (numpy.ndarray): Estimated targets as returned by the error tree. Expected + values in [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + + Returns: + fidelity (float): Fidelity score, measuring how well the Error Tree represents the original model errors. + decision (bool): Decision regarding whether to trust the Error Tree. + """ difference_true_pred_accuracy = np.abs(primary_model_true_accuracy - primary_model_predicted_accuracy) decision = difference_true_pred_accuracy <= ErrorAnalyzerConstants.TREE_ACCURACY_TOLERANCE @@ -15,15 +28,36 @@ def compute_confidence_decision(primary_model_true_accuracy, primary_model_predi def compute_accuracy_score(y_true, y_pred): + """Return the accuracy of predictions with respect to true values.""" return accuracy_score(y_true, y_pred) def compute_primary_model_accuracy(y): + """Return fidelity of the Error Tree and decision regarding its reliability. + + Args: + y (numpy.ndarray): Array indicating whether the model is correct for each sample. Expected values in + [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + + Returns: + float: Estimated accuracy of the primary model. + """ n_test_samples = y.shape[0] return float(np.count_nonzero(y == ErrorAnalyzerConstants.CORRECT_PREDICTION)) / n_test_samples def compute_fidelity_score(y_true, y_pred): + """Return fidelity of the Error Tree and decision regarding its reliability. + + Args: + y_true (numpy.ndarray): Ground truth values of wrong/correct predictions of the error tree primary model. + Expected values in [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + y_pred (numpy.ndarray): Estimated targets as returned by the error tree. Expected values in + [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + + Returns: + fidelity (float): Fidelity score, measuring how well the Error Tree represents the original model errors. + """ difference_true_pred_accuracy = np.abs(compute_primary_model_accuracy(y_true) - compute_primary_model_accuracy(y_pred)) fidelity = 1. - difference_true_pred_accuracy @@ -32,11 +66,22 @@ def compute_fidelity_score(y_true, y_pred): def fidelity_balanced_accuracy_score(y_true, y_pred): + """Return a custom metrics, as the sum of the fidelity and the balanced accuracy of the Error Tree. + + Args: + y_true (numpy.ndarray): Ground truth values of wrong/correct predictions of the error tree primary model. + Expected values in [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + y_pred (numpy.ndarray): Estimated targets as returned by the error tree. Expected values in + [ErrorAnalyzerConstants.WRONG_PREDICTION, ErrorAnalyzerConstants.CORRECT_PREDICTION]. + + Returns: + dict or str: Dictionary or string report storing different metrics regarding the Error Tree. + """ return compute_fidelity_score(y_true, y_pred) + balanced_accuracy_score(y_true, y_pred) def error_decision_tree_report(y_true, y_pred, output_format='str'): - """Return a report showing the main Error Decision Tree metrics. + """Return a report showing the main Error Tree metrics. Args: y_true (numpy.ndarray): Ground truth values of wrong/correct predictions of the error tree primary model. @@ -46,7 +91,7 @@ def error_decision_tree_report(y_true, y_pred, output_format='str'): output_format (str): Return format used for the report. Valid values are 'dict' or 'str'. Returns: - dict or str: dictionary or string report storing different metrics regarding the Error Decision Tree. + dict or str: Dictionary or string report storing different metrics regarding the Error Tree. """ tree_accuracy_score = compute_accuracy_score(y_true, y_pred) @@ -67,7 +112,7 @@ def error_decision_tree_report(y_true, y_pred, output_format='str'): if output_format == 'str': - report = 'The Error Decision Tree was trained with accuracy %.2f%% and balanced accuracy %.2f%%.' % (tree_accuracy_score * 100, tree_balanced_accuracy * 100) + report = 'The Error Tree was trained with accuracy %.2f%% and balanced accuracy %.2f%%.' % (tree_accuracy_score * 100, tree_balanced_accuracy * 100) report += '\n' report += 'The Decision Tree estimated the primary model''s accuracy to %.2f%%.' % \ (primary_model_predicted_accuracy * 100) From b29ce29669ca2452c875cbfb9648e0692fb01a7e Mon Sep 17 00:00:00 2001 From: Simona Maggio Date: Thu, 22 Apr 2021 08:40:28 +0200 Subject: [PATCH 8/8] fix function description --- mealy/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mealy/metrics.py b/mealy/metrics.py index e4af1a3..8cf004e 100644 --- a/mealy/metrics.py +++ b/mealy/metrics.py @@ -33,7 +33,7 @@ def compute_accuracy_score(y_true, y_pred): def compute_primary_model_accuracy(y): - """Return fidelity of the Error Tree and decision regarding its reliability. + """Return accuracy of the primary model. Args: y (numpy.ndarray): Array indicating whether the model is correct for each sample. Expected values in @@ -47,7 +47,7 @@ def compute_primary_model_accuracy(y): def compute_fidelity_score(y_true, y_pred): - """Return fidelity of the Error Tree and decision regarding its reliability. + """Return fidelity of the Error Tree. Args: y_true (numpy.ndarray): Ground truth values of wrong/correct predictions of the error tree primary model.