| # coding: utf-8 |
| # pylint: disable=no-member, too-many-lines |
| |
| """Online evaluation metric module.""" |
| from __future__ import absolute_import |
| import math |
| from collections import OrderedDict |
| |
| import numpy |
| |
| from .base import numeric_types, string_types |
| from . import ndarray |
| from . import registry |
| |
| |
| def check_label_shapes(labels, preds, shape=0): |
| if shape == 0: |
| label_shape, pred_shape = len(labels), len(preds) |
| else: |
| label_shape, pred_shape = labels.shape, preds.shape |
| |
| if label_shape != pred_shape: |
| raise ValueError("Shape of labels {} does not match shape of " |
| "predictions {}".format(label_shape, pred_shape)) |
| |
| |
| class EvalMetric(object): |
| """Base class for all evaluation metrics. |
| |
| .. note:: |
| |
| This is a base class that provides common metric interfaces. |
| One should not use this class directly, but instead create new metric |
| classes that extend it. |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| """ |
| def __init__(self, name, output_names=None, |
| label_names=None, **kwargs): |
| self.name = str(name) |
| self.output_names = output_names |
| self.label_names = label_names |
| self._kwargs = kwargs |
| self.reset() |
| |
| def __str__(self): |
| return "EvalMetric: {}".format(dict(self.get_name_value())) |
| |
| def get_config(self): |
| """Save configurations of metric. Can be recreated |
| from configs with metric.create(**config) |
| """ |
| config = self._kwargs.copy() |
| config.update({ |
| 'metric': self.__class__.__name__, |
| 'name': self.name, |
| 'output_names': self.output_names, |
| 'label_names': self.label_names}) |
| return config |
| |
| def update_dict(self, label, pred): |
| """Update the internal evaluation with named label and pred |
| |
| Parameters |
| ---------- |
| labels : OrderedDict of str -> NDArray |
| name to array mapping for labels. |
| |
| preds : list of NDArray |
| name to array mapping of predicted outputs. |
| """ |
| if self.output_names is not None: |
| pred = [pred[name] for name in self.output_names] |
| else: |
| pred = list(pred.values()) |
| |
| if self.label_names is not None: |
| label = [label[name] for name in self.label_names] |
| else: |
| label = list(label.values()) |
| |
| self.update(label, pred) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| raise NotImplementedError() |
| |
| def reset(self): |
| """Resets the internal evaluation result to initial state.""" |
| self.num_inst = 0 |
| self.sum_metric = 0.0 |
| |
| def get(self): |
| """Gets the current evaluation result. |
| |
| Returns |
| ------- |
| names : list of str |
| Name of the metrics. |
| values : list of float |
| Value of the evaluations. |
| """ |
| if self.num_inst == 0: |
| return (self.name, float('nan')) |
| else: |
| return (self.name, self.sum_metric / self.num_inst) |
| |
| def get_name_value(self): |
| """Returns zipped name and value pairs. |
| |
| Returns |
| ------- |
| list of tuples |
| A (name, value) tuple list. |
| """ |
| name, value = self.get() |
| if not isinstance(name, list): |
| name = [name] |
| if not isinstance(value, list): |
| value = [value] |
| return list(zip(name, value)) |
| |
| # pylint: disable=invalid-name |
| register = registry.get_register_func(EvalMetric, 'metric') |
| alias = registry.get_alias_func(EvalMetric, 'metric') |
| _create = registry.get_create_func(EvalMetric, 'metric') |
| # pylint: enable=invalid-name |
| |
| |
| def create(metric, *args, **kwargs): |
| """Creates evaluation metric from metric names or instances of EvalMetric |
| or a custom metric function. |
| |
| Parameters |
| ---------- |
| metric : str or callable |
| Specifies the metric to create. |
| This argument must be one of the below: |
| |
| - Name of a metric. |
| - An instance of `EvalMetric`. |
| - A list, each element of which is a metric or a metric name. |
| - An evaluation function that computes custom metric for a given batch of |
| labels and predictions. |
| *args : list |
| Additional arguments to metric constructor. |
| Only used when metric is str. |
| **kwargs : dict |
| Additional arguments to metric constructor. |
| Only used when metric is str |
| |
| Examples |
| -------- |
| >>> def custom_metric(label, pred): |
| ... return np.mean(np.abs(label - pred)) |
| ... |
| >>> metric1 = mx.metric.create('acc') |
| >>> metric2 = mx.metric.create(custom_metric) |
| >>> metric3 = mx.metric.create([metric1, metric2, 'rmse']) |
| """ |
| if callable(metric): |
| return CustomMetric(metric, *args, **kwargs) |
| elif isinstance(metric, list): |
| composite_metric = CompositeEvalMetric() |
| for child_metric in metric: |
| composite_metric.add(create(child_metric, *args, **kwargs)) |
| return composite_metric |
| |
| return _create(metric, *args, **kwargs) |
| |
| |
| @register |
| @alias('composite') |
| class CompositeEvalMetric(EvalMetric): |
| """Manages multiple evaluation metrics. |
| |
| Parameters |
| ---------- |
| metrics : list of EvalMetric |
| List of child metrics. |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] |
| >>> labels = [mx.nd.array([0, 1, 1])] |
| >>> eval_metrics_1 = mx.metric.Accuracy() |
| >>> eval_metrics_2 = mx.metric.F1() |
| >>> eval_metrics = mx.metric.CompositeEvalMetric() |
| >>> for child_metric in [eval_metrics_1, eval_metrics_2]: |
| >>> eval_metrics.add(child_metric) |
| >>> eval_metrics.update(labels = labels, preds = predicts) |
| >>> print eval_metrics.get() |
| (['accuracy', 'f1'], [0.6666666666666666, 0.8]) |
| """ |
| |
| def __init__(self, metrics=None, name='composite', |
| output_names=None, label_names=None): |
| super(CompositeEvalMetric, self).__init__( |
| 'composite', output_names=output_names, label_names=label_names) |
| if metrics is None: |
| metrics = [] |
| self.metrics = [create(i) for i in metrics] |
| |
| def add(self, metric): |
| """Adds a child metric. |
| |
| Parameters |
| ---------- |
| metric |
| A metric instance. |
| """ |
| self.metrics.append(create(metric)) |
| |
| def get_metric(self, index): |
| """Returns a child metric. |
| |
| Parameters |
| ---------- |
| index : int |
| Index of child metric in the list of metrics. |
| """ |
| try: |
| return self.metrics[index] |
| except IndexError: |
| return ValueError("Metric index {} is out of range 0 and {}".format( |
| index, len(self.metrics))) |
| |
| def update_dict(self, labels, preds): # pylint: disable=arguments-differ |
| if self.label_names is not None: |
| labels = OrderedDict([i for i in labels.items() |
| if i[0] in self.label_names]) |
| if self.output_names is not None: |
| preds = OrderedDict([i for i in preds.items() |
| if i[0] in self.output_names]) |
| |
| for metric in self.metrics: |
| metric.update_dict(labels, preds) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| for metric in self.metrics: |
| metric.update(labels, preds) |
| |
| def reset(self): |
| """Resets the internal evaluation result to initial state.""" |
| try: |
| for metric in self.metrics: |
| metric.reset() |
| except AttributeError: |
| pass |
| |
| def get(self): |
| """Returns the current evaluation result. |
| |
| Returns |
| ------- |
| names : list of str |
| Name of the metrics. |
| values : list of float |
| Value of the evaluations. |
| """ |
| names = [] |
| values = [] |
| for metric in self.metrics: |
| name, value = metric.get() |
| if isinstance(name, string_types): |
| name = [name] |
| if isinstance(value, numeric_types): |
| value = [value] |
| names.extend(name) |
| values.extend(value) |
| return (names, values) |
| |
| def get_config(self): |
| config = super(CompositeEvalMetric, self).get_config() |
| config.update({'metrics': [i.get_config() for i in self.metrics]}) |
| return config |
| |
| |
| ######################## |
| # CLASSIFICATION METRICS |
| ######################## |
| |
| |
| @register |
| @alias('acc') |
| class Accuracy(EvalMetric): |
| """Computes accuracy classification score. |
| |
| The accuracy score is defined as |
| |
| .. math:: |
| \\text{accuracy}(y, \\hat{y}) = \\frac{1}{n} \\sum_{i=0}^{n-1} |
| \\text{1}(\\hat{y_i} == y_i) |
| |
| Parameters |
| ---------- |
| axis : int, default=1 |
| The axis that represents classes |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] |
| >>> labels = [mx.nd.array([0, 1, 1])] |
| >>> acc = mx.metric.Accuracy() |
| >>> acc.update(preds = predicts, labels = labels) |
| >>> print acc.get() |
| ('accuracy', 0.6666666666666666) |
| """ |
| def __init__(self, axis=1, name='accuracy', |
| output_names=None, label_names=None): |
| super(Accuracy, self).__init__( |
| name, axis=axis, |
| output_names=output_names, label_names=label_names) |
| self.axis = axis |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred_label in zip(labels, preds): |
| if pred_label.shape != label.shape: |
| pred_label = ndarray.argmax(pred_label, axis=self.axis) |
| pred_label = pred_label.asnumpy().astype('int32') |
| label = label.asnumpy().astype('int32') |
| |
| check_label_shapes(label, pred_label) |
| |
| self.sum_metric += (pred_label.flat == label.flat).sum() |
| self.num_inst += len(pred_label.flat) |
| |
| |
| @register |
| @alias('top_k_accuracy', 'top_k_acc') |
| class TopKAccuracy(EvalMetric): |
| """Computes top k predictions accuracy. |
| |
| `TopKAccuracy` differs from Accuracy in that it considers the prediction |
| to be ``True`` as long as the ground truth label is in the top K |
| predicated labels. |
| |
| If `top_k` = ``1``, then `TopKAccuracy` is identical to `Accuracy`. |
| |
| Parameters |
| ---------- |
| top_k : int |
| Whether targets are in top k predictions. |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> np.random.seed(999) |
| >>> top_k = 3 |
| >>> labels = [mx.nd.array([2, 6, 9, 2, 3, 4, 7, 8, 9, 6])] |
| >>> predicts = [mx.nd.array(np.random.rand(10, 10))] |
| >>> acc = mx.metric.TopKAccuracy(top_k=top_k) |
| >>> acc.update(labels, predicts) |
| >>> print acc.get() |
| ('top_k_accuracy', 0.3) |
| """ |
| |
| def __init__(self, top_k=1, name='top_k_accuracy', |
| output_names=None, label_names=None): |
| super(TopKAccuracy, self).__init__( |
| name, top_k=top_k, |
| output_names=output_names, label_names=label_names) |
| self.top_k = top_k |
| assert(self.top_k > 1), 'Please use Accuracy if top_k is no more than 1' |
| self.name += '_%d' % self.top_k |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred_label in zip(labels, preds): |
| assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims' |
| pred_label = numpy.argsort(pred_label.asnumpy().astype('float32'), axis=1) |
| label = label.asnumpy().astype('int32') |
| check_label_shapes(label, pred_label) |
| num_samples = pred_label.shape[0] |
| num_dims = len(pred_label.shape) |
| if num_dims == 1: |
| self.sum_metric += (pred_label.flat == label.flat).sum() |
| elif num_dims == 2: |
| num_classes = pred_label.shape[1] |
| top_k = min(num_classes, self.top_k) |
| for j in range(top_k): |
| self.sum_metric += (pred_label[:, num_classes - 1 - j].flat == label.flat).sum() |
| self.num_inst += num_samples |
| |
| |
| @register |
| class F1(EvalMetric): |
| """Computes the F1 score of a binary classification problem. |
| |
| The F1 score is equivalent to weighted average of the precision and recall, |
| where the best value is 1.0 and the worst value is 0.0. The formula for F1 score is:: |
| |
| F1 = 2 * (precision * recall) / (precision + recall) |
| |
| The formula for precision and recall is:: |
| |
| precision = true_positives / (true_positives + false_positives) |
| recall = true_positives / (true_positives + false_negatives) |
| |
| .. note:: |
| |
| This F1 score only supports binary classification. |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array([[0.3, 0.7], [0., 1.], [0.4, 0.6]])] |
| >>> labels = [mx.nd.array([0., 1., 1.])] |
| >>> acc = mx.metric.F1() |
| >>> acc.update(preds = predicts, labels = labels) |
| >>> print acc.get() |
| ('f1', 0.8) |
| """ |
| |
| def __init__(self, name='f1', |
| output_names=None, label_names=None): |
| super(F1, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred in zip(labels, preds): |
| pred = pred.asnumpy() |
| label = label.asnumpy().astype('int32') |
| pred_label = numpy.argmax(pred, axis=1) |
| |
| check_label_shapes(label, pred) |
| if len(numpy.unique(label)) > 2: |
| raise ValueError("F1 currently only supports binary classification.") |
| |
| true_positives, false_positives, false_negatives = 0., 0., 0. |
| |
| for y_pred, y_true in zip(pred_label, label): |
| if y_pred == 1 and y_true == 1: |
| true_positives += 1. |
| elif y_pred == 1 and y_true == 0: |
| false_positives += 1. |
| elif y_pred == 0 and y_true == 1: |
| false_negatives += 1. |
| |
| if true_positives + false_positives > 0: |
| precision = true_positives / (true_positives + false_positives) |
| else: |
| precision = 0. |
| |
| if true_positives + false_negatives > 0: |
| recall = true_positives / (true_positives + false_negatives) |
| else: |
| recall = 0. |
| |
| if precision + recall > 0: |
| f1_score = 2 * precision * recall / (precision + recall) |
| else: |
| f1_score = 0. |
| |
| self.sum_metric += f1_score |
| self.num_inst += 1 |
| |
| |
| @register |
| class Perplexity(EvalMetric): |
| """Computes perplexity. |
| |
| Perplexity is a measurement of how well a probability distribution |
| or model predicts a sample. A low perplexity indicates the model |
| is good at predicting the sample. |
| |
| The perplexity of a model q is defined as |
| |
| .. math:: |
| b^{\\big(-\\frac{1}{N} \\sum_{i=1}^N \\log_b q(x_i) \\big)} |
| = \\exp \\big(-\\frac{1}{N} \\sum_{i=1}^N \\log q(x_i)\\big) |
| |
| where we let `b = e`. |
| |
| :math:`q(x_i)` is the predicted value of its ground truth |
| label on sample :math:`x_i`. |
| |
| For example, we have three samples :math:`x_1, x_2, x_3` and their labels |
| are :math:`[0, 1, 1]`. |
| Suppose our model predicts :math:`q(x_1) = p(y_1 = 0 | x_1) = 0.3` |
| and :math:`q(x_2) = 1.0`, |
| :math:`q(x_3) = 0.6`. The perplexity of model q is |
| :math:`exp\\big(-(\\log 0.3 + \\log 1.0 + \\log 0.6) / 3\\big) = 1.77109762852`. |
| |
| Parameters |
| ---------- |
| ignore_label : int or None |
| Index of invalid label to ignore when |
| counting. By default, sets to -1. |
| If set to `None`, it will include all entries. |
| axis : int (default -1) |
| The axis from prediction that was used to |
| compute softmax. By default use the last |
| axis. |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] |
| >>> labels = [mx.nd.array([0, 1, 1])] |
| >>> perp = mx.metric.Perplexity(ignore_label=None) |
| >>> perp.update(labels, predicts) |
| >>> print perp.get() |
| ('Perplexity', 1.7710976285155853) |
| """ |
| def __init__(self, ignore_label, axis=-1, name='perplexity', |
| output_names=None, label_names=None): |
| super(Perplexity, self).__init__( |
| name, ignore_label=ignore_label, |
| output_names=output_names, label_names=label_names) |
| self.ignore_label = ignore_label |
| self.axis = axis |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| assert len(labels) == len(preds) |
| loss = 0. |
| num = 0 |
| for label, pred in zip(labels, preds): |
| assert label.size == pred.size/pred.shape[-1], \ |
| "shape mismatch: %s vs. %s"%(label.shape, pred.shape) |
| label = label.as_in_context(pred.context).reshape((label.size,)) |
| pred = ndarray.pick(pred, label.astype(dtype='int32'), axis=self.axis) |
| if self.ignore_label is not None: |
| ignore = label == self.ignore_label |
| num -= ndarray.sum(ignore).asscalar() |
| pred = pred*(1-ignore) + ignore |
| loss -= ndarray.sum(ndarray.log(ndarray.maximum(1e-10, pred))).asscalar() |
| num += pred.size |
| self.sum_metric += loss |
| self.num_inst += num |
| |
| def get(self): |
| """Returns the current evaluation result. |
| |
| Returns |
| ------- |
| Tuple of (str, float) |
| Representing name of the metric and evaluation result. |
| """ |
| return (self.name, math.exp(self.sum_metric/self.num_inst)) |
| |
| #################### |
| # REGRESSION METRICS |
| #################### |
| |
| |
| @register |
| class MAE(EvalMetric): |
| """Computes Mean Absolute Error (MAE) loss. |
| |
| The mean absolute error is given by |
| |
| .. math:: |
| \\frac{\\sum_i^n |y_i - \\hat{y}_i|}{n} |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] |
| >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] |
| >>> mean_absolute_error = mx.metric.MAE() |
| >>> mean_absolute_error.update(labels = labels, preds = predicts) |
| >>> print mean_absolute_error.get() |
| ('mae', 0.5) |
| """ |
| |
| def __init__(self, name='mae', |
| output_names=None, label_names=None): |
| super(MAE, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred in zip(labels, preds): |
| label = label.asnumpy() |
| pred = pred.asnumpy() |
| |
| if len(label.shape) == 1: |
| label = label.reshape(label.shape[0], 1) |
| |
| self.sum_metric += numpy.abs(label - pred).mean() |
| self.num_inst += 1 # numpy.prod(label.shape) |
| |
| |
| @register |
| class MSE(EvalMetric): |
| """Computes Mean Squared Error (MSE) loss. |
| |
| The mean squared error is given by |
| |
| .. math:: |
| \\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n} |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] |
| >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] |
| >>> mean_squared_error = mx.metric.MSE() |
| >>> mean_squared_error.update(labels = labels, preds = predicts) |
| >>> print mean_squared_error.get() |
| ('mse', 0.375) |
| """ |
| def __init__(self, name='mse', |
| output_names=None, label_names=None): |
| super(MSE, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred in zip(labels, preds): |
| label = label.asnumpy() |
| pred = pred.asnumpy() |
| |
| if len(label.shape) == 1: |
| label = label.reshape(label.shape[0], 1) |
| |
| self.sum_metric += ((label - pred)**2.0).mean() |
| self.num_inst += 1 # numpy.prod(label.shape) |
| |
| |
| @register |
| class RMSE(EvalMetric): |
| """Computes Root Mean Squred Error (RMSE) loss. |
| |
| The root mean squared error is given by |
| |
| .. math:: |
| \\sqrt{\\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n}} |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] |
| >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] |
| >>> root_mean_squared_error = mx.metric.RMSE() |
| >>> root_mean_squared_error.update(labels = labels, preds = predicts) |
| >>> print root_mean_squared_error.get() |
| ('rmse', 0.612372457981) |
| """ |
| def __init__(self, name='rmse', |
| output_names=None, label_names=None): |
| super(RMSE, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred in zip(labels, preds): |
| label = label.asnumpy() |
| pred = pred.asnumpy() |
| |
| if len(label.shape) == 1: |
| label = label.reshape(label.shape[0], 1) |
| |
| self.sum_metric += numpy.sqrt(((label - pred)**2.0).mean()) |
| self.num_inst += 1 |
| |
| |
| @register |
| @alias('ce') |
| class CrossEntropy(EvalMetric): |
| """Computes Cross Entropy loss. |
| |
| The cross entropy is given by |
| |
| .. math:: |
| -y\\log \\hat{y} + (1-y)\\log (1-\\hat{y}) |
| |
| Parameters |
| ---------- |
| eps : float |
| Cross Entropy loss is undefined for predicted value is 0 or 1, |
| so predicted values are added with the small constant. |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] |
| >>> labels = [mx.nd.array([0, 1, 1])] |
| >>> ce = mx.metric.CrossEntropy() |
| >>> ce.update(labels, predicts) |
| >>> print ce.get() |
| ('cross-entropy', 0.57159948348999023) |
| """ |
| def __init__(self, eps=1e-8, name='cross-entropy', |
| output_names=None, label_names=None): |
| super(CrossEntropy, self).__init__( |
| name, eps=eps, |
| output_names=output_names, label_names=label_names) |
| self.eps = eps |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| check_label_shapes(labels, preds) |
| |
| for label, pred in zip(labels, preds): |
| label = label.asnumpy() |
| pred = pred.asnumpy() |
| |
| label = label.ravel() |
| assert label.shape[0] == pred.shape[0] |
| |
| prob = pred[numpy.arange(label.shape[0]), numpy.int64(label)] |
| self.sum_metric += (-numpy.log(prob + self.eps)).sum() |
| self.num_inst += label.shape[0] |
| |
| |
| @register |
| class Loss(EvalMetric): |
| """Dummy metric for directly printing loss. |
| |
| Parameters |
| ---------- |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| """ |
| def __init__(self, name='loss', |
| output_names=None, label_names=None): |
| super(Loss, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| def update(self, _, preds): |
| for pred in preds: |
| self.sum_metric += ndarray.sum(pred).asscalar() |
| self.num_inst += pred.size |
| |
| |
| @register |
| class Torch(Loss): |
| """Dummy metric for torch criterions.""" |
| def __init__(self, name='torch', |
| output_names=None, label_names=None): |
| super(Torch, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| |
| @register |
| class Caffe(Loss): |
| """Dummy metric for caffe criterions.""" |
| def __init__(self, name='caffe', |
| output_names=None, label_names=None): |
| super(Caffe, self).__init__( |
| name, output_names=output_names, label_names=label_names) |
| |
| |
| @register |
| class CustomMetric(EvalMetric): |
| """Computes a customized evaluation metric. |
| |
| The `feval` function can return a `tuple` of (sum_metric, num_inst) or return |
| an `int` sum_metric. |
| |
| Parameters |
| ---------- |
| feval : callable(label, pred) |
| Customized evaluation function. |
| name : str, optional |
| The name of the metric. (the default is None). |
| allow_extra_outputs : bool, optional |
| If true, the prediction outputs can have extra outputs. |
| This is useful in RNN, where the states are also produced |
| in outputs for forwarding. (the default is False). |
| name : str |
| Name of this metric instance for display. |
| output_names : list of str, or None |
| Name of predictions that should be used when updating with update_dict. |
| By default include all predictions. |
| label_names : list of str, or None |
| Name of labels that should be used when updating with update_dict. |
| By default include all labels. |
| |
| Examples |
| -------- |
| >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] |
| >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] |
| >>> feval = lambda x, y : (x + y).mean() |
| >>> eval_metrics = mx.metric.CustomMetric(feval=feval) |
| >>> eval_metrics.update(labels, predicts) |
| >>> print eval_metrics.get() |
| ('custom(<lambda>)', 6.0) |
| """ |
| def __init__(self, feval, name=None, allow_extra_outputs=False, |
| output_names=None, label_names=None): |
| if name is None: |
| name = feval.__name__ |
| if name.find('<') != -1: |
| name = 'custom(%s)' % name |
| super(CustomMetric, self).__init__( |
| name, feval=feval, |
| allow_extra_outputs=allow_extra_outputs, |
| output_names=output_names, label_names=label_names) |
| self._feval = feval |
| self._allow_extra_outputs = allow_extra_outputs |
| |
| def update(self, labels, preds): |
| """Updates the internal evaluation result. |
| |
| Parameters |
| ---------- |
| labels : list of `NDArray` |
| The labels of the data. |
| |
| preds : list of `NDArray` |
| Predicted values. |
| """ |
| if not self._allow_extra_outputs: |
| check_label_shapes(labels, preds) |
| |
| for pred, label in zip(preds, labels): |
| label = label.asnumpy() |
| pred = pred.asnumpy() |
| |
| reval = self._feval(label, pred) |
| if isinstance(reval, tuple): |
| (sum_metric, num_inst) = reval |
| self.sum_metric += sum_metric |
| self.num_inst += num_inst |
| else: |
| self.sum_metric += reval |
| self.num_inst += 1 |
| |
| def get_config(self): |
| raise NotImplementedError("CustomMetric cannot be serialized") |
| |
| |
| # pylint: disable=invalid-name |
| def np(numpy_feval, name=None, allow_extra_outputs=False): |
| """Creates a custom evaluation metric that receives its inputs as numpy arrays. |
| |
| Parameters |
| ---------- |
| numpy_feval : callable(label, pred) |
| Custom evaluation function that receives labels and predictions for a minibatch |
| as numpy arrays and returns the corresponding custom metric as a floating point number. |
| name : str, optional |
| Name of the custom metric. |
| allow_extra_outputs : bool, optional |
| Whether prediction output is allowed to have extra outputs. This is useful in cases |
| like RNN where states are also part of output which can then be fed back to the RNN |
| in the next step. By default, extra outputs are not allowed. |
| |
| Returns |
| ------- |
| float |
| Custom metric corresponding to the provided labels and predictions. |
| |
| Example |
| ------- |
| >>> def custom_metric(label, pred): |
| ... return np.mean(np.abs(label-pred)) |
| ... |
| >>> metric = mx.metric.np(custom_metric) |
| """ |
| def feval(label, pred): |
| """Internal eval function.""" |
| return numpy_feval(label, pred) |
| feval.__name__ = numpy_feval.__name__ |
| return CustomMetric(feval, name, allow_extra_outputs) |
| # pylint: enable=invalid-name |
