!9137 Add Beta distribution

From: @peixu_ren
Reviewed-by: @zichun_ye,@sunnybeike
Signed-off-by: @sunnybeike

mindspore/nn/probability/distribution/__init__.py

@@ -19,6 +19,7 @@ Distributions are the high-level components used to construct the probabilistic
 from .distribution import Distribution
 from .transformed_distribution import TransformedDistribution
 from .bernoulli import Bernoulli
+from .beta import Beta
 from .categorical import Categorical
 from .cauchy import Cauchy
 from .exponential import Exponential
@@ -34,6 +35,7 @@ from .uniform import Uniform
 __all__ = ['Distribution',
            'TransformedDistribution',
            'Bernoulli',
+           'Beta',
            'Categorical',
            'Cauchy',
            'Exponential',

mindspore/nn/probability/distribution/gamma.py

@@ -81,12 +81,12 @@ class Gamma(Distribution):
         ...         ans = self.g2.prob(value, concentration_a, rate_a)
         ...
         ...
-        ...         # Functions `concentration`, `rate`, `mean`, `sd`, `var`, and `entropy` have the same arguments.
+        ...         # Functions `mean`, `sd`, `mode`, `var`, and `entropy` have the same arguments.
         ...         # Args:
         ...         #     concentration (Tensor): the concentration of the distribution. Default: self._concentration.
         ...         #     rate (Tensor): the rate of the distribution. Default: self._rate.
         ...
-        ...         # Example of `concentration`, `rate`, `mean`. `sd`, `var`, and `entropy` are similar.
+        ...         # Example of `mean`, `sd`, `mode`, `var`, and `entropy` are similar.
         ...         ans = self.g1.concentration() # return 1.0
         ...         ans = self.g1.concentration(concentration_b, rate_b) # return concentration_b
         ...         # `concentration` and `rate` must be passed in during function calls.

mindspore/nn/probability/distribution/poisson.py

@@ -76,11 +76,11 @@ class Poisson(Distribution):
         ...         ans = self.p2.prob(value, rate_a)
         ...
         ...
-        ...         # Functions `mean`, `sd`, and 'var' have the same arguments as follows.
+        ...         # Functions `mean`, `mode`, `sd`, and 'var' have the same arguments as follows.
         ...         # Args:
         ...         #     rate (Tensor): the rate of the distribution. Default: self.rate.
         ...
-        ...         # Examples of `mean`. `sd`, `var`, and `entropy` are similar.
+        ...         # Examples of `mean`, `sd`, `mode`, `var`, and `entropy` are similar.
         ...         ans = self.p1.mean() # return 2
         ...         ans = self.p1.mean(rate_b) # return 1 / rate_b
         ...         # `rate` must be passed in during function calls.

tests/st/probability/distribution/test_beta.py

@@ -0,0 +1,245 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""test cases for Beta distribution"""
+import numpy as np
+from scipy import stats
+from scipy import special
+import mindspore.context as context
+import mindspore.nn as nn
+import mindspore.nn.probability.distribution as msd
+from mindspore import Tensor
+from mindspore import dtype
+
+context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
+
+class Prob(nn.Cell):
+    """
+    Test class: probability of Beta distribution.
+    """
+    def __init__(self):
+        super(Prob, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+
+    def construct(self, x_):
+        return self.b.prob(x_)
+
+def test_pdf():
+    """
+    Test pdf.
+    """
+    beta_benchmark = stats.beta(np.array([3.0]), np.array([1.0]))
+    expect_pdf = beta_benchmark.pdf([0.25, 0.75]).astype(np.float32)
+    pdf = Prob()
+    output = pdf(Tensor([0.25, 0.75], dtype=dtype.float32))
+    tol = 1e-6
+    assert (np.abs(output.asnumpy() - expect_pdf) < tol).all()
+
+class LogProb(nn.Cell):
+    """
+    Test class: log probability of Beta distribution.
+    """
+    def __init__(self):
+        super(LogProb, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+
+    def construct(self, x_):
+        return self.b.log_prob(x_)
+
+def test_log_likelihood():
+    """
+    Test log_pdf.
+    """
+    beta_benchmark = stats.beta(np.array([3.0]), np.array([1.0]))
+    expect_logpdf = beta_benchmark.logpdf([0.25, 0.75]).astype(np.float32)
+    logprob = LogProb()
+    output = logprob(Tensor([0.25, 0.75], dtype=dtype.float32))
+    tol = 1e-6
+    assert (np.abs(output.asnumpy() - expect_logpdf) < tol).all()
+
+class KL(nn.Cell):
+    """
+    Test class: kl_loss of Beta distribution.
+    """
+    def __init__(self):
+        super(KL, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([4.0]), dtype=dtype.float32)
+
+    def construct(self, x_, y_):
+        return self.b.kl_loss('Beta', x_, y_)
+
+def test_kl_loss():
+    """
+    Test kl_loss.
+    """
+    concentration1_a = np.array([3.0]).astype(np.float32)
+    concentration0_a = np.array([4.0]).astype(np.float32)
+
+    concentration1_b = np.array([1.0]).astype(np.float32)
+    concentration0_b = np.array([1.0]).astype(np.float32)
+
+    total_concentration_a = concentration1_a + concentration0_a
+    total_concentration_b = concentration1_b + concentration0_b
+    log_normalization_a = np.log(special.beta(concentration1_a, concentration0_a))
+    log_normalization_b = np.log(special.beta(concentration1_b, concentration0_b))
+    expect_kl_loss = (log_normalization_b - log_normalization_a) \
+                     - (special.digamma(concentration1_a) * (concentration1_b - concentration1_a)) \
+                     - (special.digamma(concentration0_a) * (concentration0_b - concentration0_a)) \
+                     + (special.digamma(total_concentration_a) * (total_concentration_b - total_concentration_a))
+
+    kl_loss = KL()
+    concentration1 = Tensor(concentration1_b, dtype=dtype.float32)
+    concentration0 = Tensor(concentration0_b, dtype=dtype.float32)
+    output = kl_loss(concentration1, concentration0)
+    tol = 1e-6
+    assert (np.abs(output.asnumpy() - expect_kl_loss) < tol).all()
+
+class Basics(nn.Cell):
+    """
+    Test class: mean/sd/mode of Beta distribution.
+    """
+    def __init__(self):
+        super(Basics, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([3.0]), dtype=dtype.float32)
+
+    def construct(self):
+        return self.b.mean(), self.b.sd(), self.b.mode()
+
+def test_basics():
+    """
+    Test mean/standard deviation/mode.
+    """
+    basics = Basics()
+    mean, sd, mode = basics()
+    beta_benchmark = stats.beta(np.array([3.0]), np.array([3.0]))
+    expect_mean = beta_benchmark.mean().astype(np.float32)
+    expect_sd = beta_benchmark.std().astype(np.float32)
+    expect_mode = [0.5]
+    tol = 1e-6
+    assert (np.abs(mean.asnumpy() - expect_mean) < tol).all()
+    assert (np.abs(mode.asnumpy() - expect_mode) < tol).all()
+    assert (np.abs(sd.asnumpy() - expect_sd) < tol).all()
+
+class Sampling(nn.Cell):
+    """
+    Test class: sample of Beta distribution.
+    """
+    def __init__(self, shape, seed=0):
+        super(Sampling, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([1.0]), seed=seed, dtype=dtype.float32)
+        self.shape = shape
+
+    def construct(self, concentration1=None, concentration0=None):
+        return self.b.sample(self.shape, concentration1, concentration0)
+
+def test_sample():
+    """
+    Test sample.
+    """
+    shape = (2, 3)
+    seed = 10
+    concentration1 = Tensor([2.0], dtype=dtype.float32)
+    concentration0 = Tensor([2.0, 2.0, 2.0], dtype=dtype.float32)
+    sample = Sampling(shape, seed=seed)
+    output = sample(concentration1, concentration0)
+    assert output.shape == (2, 3, 3)
+
+class EntropyH(nn.Cell):
+    """
+    Test class: entropy of Beta distribution.
+    """
+    def __init__(self):
+        super(EntropyH, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+
+    def construct(self):
+        return self.b.entropy()
+
+def test_entropy():
+    """
+    Test entropy.
+    """
+    beta_benchmark = stats.beta(np.array([3.0]), np.array([1.0]))
+    expect_entropy = beta_benchmark.entropy().astype(np.float32)
+    entropy = EntropyH()
+    output = entropy()
+    tol = 1e-6
+    assert (np.abs(output.asnumpy() - expect_entropy) < tol).all()
+
+class CrossEntropy(nn.Cell):
+    """
+    Test class: cross entropy between Beta distributions.
+    """
+    def __init__(self):
+        super(CrossEntropy, self).__init__()
+        self.b = msd.Beta(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+
+    def construct(self, x_, y_):
+        entropy = self.b.entropy()
+        kl_loss = self.b.kl_loss('Beta', x_, y_)
+        h_sum_kl = entropy + kl_loss
+        cross_entropy = self.b.cross_entropy('Beta', x_, y_)
+        return h_sum_kl - cross_entropy
+
+def test_cross_entropy():
+    """
+    Test cross_entropy.
+    """
+    cross_entropy = CrossEntropy()
+    concentration1 = Tensor([3.0], dtype=dtype.float32)
+    concentration0 = Tensor([2.0], dtype=dtype.float32)
+    diff = cross_entropy(concentration1, concentration0)
+    tol = 1e-6
+    assert (np.abs(diff.asnumpy() - np.zeros(diff.shape)) < tol).all()
+
+class Net(nn.Cell):
+    """
+    Test class: expand single distribution instance to multiple graphs
+    by specifying the attributes.
+    """
+
+    def __init__(self):
+        super(Net, self).__init__()
+        self.beta = msd.Beta(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+
+    def construct(self, x_, y_):
+        kl = self.beta.kl_loss('Beta', x_, y_)
+        prob = self.beta.prob(kl)
+        return prob
+
+def test_multiple_graphs():
+    """
+    Test multiple graphs case.
+    """
+    prob = Net()
+    concentration1_a = np.array([3.0]).astype(np.float32)
+    concentration0_a = np.array([1.0]).astype(np.float32)
+    concentration1_b = np.array([2.0]).astype(np.float32)
+    concentration0_b = np.array([1.0]).astype(np.float32)
+    ans = prob(Tensor(concentration1_b), Tensor(concentration0_b))
+
+    total_concentration_a = concentration1_a + concentration0_a
+    total_concentration_b = concentration1_b + concentration0_b
+    log_normalization_a = np.log(special.beta(concentration1_a, concentration0_a))
+    log_normalization_b = np.log(special.beta(concentration1_b, concentration0_b))
+    expect_kl_loss = (log_normalization_b - log_normalization_a) \
+                     - (special.digamma(concentration1_a) * (concentration1_b - concentration1_a)) \
+                     - (special.digamma(concentration0_a) * (concentration0_b - concentration0_a)) \
+                     + (special.digamma(total_concentration_a) * (total_concentration_b - total_concentration_a))
+
+    beta_benchmark = stats.beta(np.array([3.0]), np.array([1.0]))
+    expect_prob = beta_benchmark.pdf(expect_kl_loss).astype(np.float32)
+
+    tol = 1e-6
+    assert (np.abs(ans.asnumpy() - expect_prob) < tol).all()

tests/st/probability/distribution/test_gamma.py

@@ -298,11 +298,11 @@ class Net(nn.Cell):
 
     def __init__(self):
         super(Net, self).__init__()
-        self.Gamma = msd.Gamma(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
+        self.get_flags = msd.Gamma(np.array([3.0]), np.array([1.0]), dtype=dtype.float32)
 
     def construct(self, x_, y_):
-        kl = self.Gamma.kl_loss('Gamma', x_, y_)
+        kl = self.g.kl_loss('Gamma', x_, y_)
-        prob = self.Gamma.prob(kl)
+        prob = self.g.prob(kl)
         return prob
 
 def test_multiple_graphs():

tests/ut/python/nn/probability/distribution/test_beta.py

@@ -0,0 +1,212 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Test nn.probability.distribution.Gamma.
+"""
+import numpy as np
+import pytest
+
+import mindspore.nn as nn
+import mindspore.nn.probability.distribution as msd
+from mindspore import dtype
+from mindspore import Tensor
+
+def test_gamma_shape_errpr():
+    """
+    Invalid shapes.
+    """
+    with pytest.raises(ValueError):
+        msd.Gamma([[2.], [1.]], [[2.], [3.], [4.]], dtype=dtype.float32)
+
+def test_type():
+    with pytest.raises(TypeError):
+        msd.Gamma(0., 1., dtype=dtype.int32)
+
+def test_name():
+    with pytest.raises(TypeError):
+        msd.Gamma(0., 1., name=1.0)
+
+def test_seed():
+    with pytest.raises(TypeError):
+        msd.Gamma(0., 1., seed='seed')
+
+def test_concentration1():
+    with pytest.raises(ValueError):
+        msd.Gamma(0., 1.)
+    with pytest.raises(ValueError):
+        msd.Gamma(-1., 1.)
+
+def test_concentration0():
+    with pytest.raises(ValueError):
+        msd.Gamma(1., 0.)
+    with pytest.raises(ValueError):
+        msd.Gamma(1., -1.)
+
+def test_arguments():
+    """
+    args passing during initialization.
+    """
+    g = msd.Gamma()
+    assert isinstance(g, msd.Distribution)
+    g = msd.Gamma([3.0], [4.0], dtype=dtype.float32)
+    assert isinstance(g, msd.Distribution)
+
+
+class GammaProb(nn.Cell):
+    """
+    Gamma distribution: initialize with concentration1/concentration0.
+    """
+    def __init__(self):
+        super(GammaProb, self).__init__()
+        self.gamma = msd.Gamma([3.0, 4.0], [1.0, 1.0], dtype=dtype.float32)
+
+    def construct(self, value):
+        prob = self.gamma.prob(value)
+        log_prob = self.gamma.log_prob(value)
+        return prob + log_prob
+
+def test_gamma_prob():
+    """
+    Test probability functions: passing value through construct.
+    """
+    net = GammaProb()
+    value = Tensor([0.5, 1.0], dtype=dtype.float32)
+    ans = net(value)
+    assert isinstance(ans, Tensor)
+
+
+class GammaProb1(nn.Cell):
+    """
+    Gamma distribution: initialize without concentration1/concentration0.
+    """
+    def __init__(self):
+        super(GammaProb1, self).__init__()
+        self.gamma = msd.Gamma()
+
+    def construct(self, value, concentration1, concentration0):
+        prob = self.gamma.prob(value, concentration1, concentration0)
+        log_prob = self.gamma.log_prob(value, concentration1, concentration0)
+        return prob + log_prob
+
+def test_gamma_prob1():
+    """
+    Test probability functions: passing concentration1/concentration0, value through construct.
+    """
+    net = GammaProb1()
+    value = Tensor([0.5, 1.0], dtype=dtype.float32)
+    concentration1 = Tensor([2.0, 3.0], dtype=dtype.float32)
+    concentration0 = Tensor([1.0], dtype=dtype.float32)
+    ans = net(value, concentration1, concentration0)
+    assert isinstance(ans, Tensor)
+
+class GammaKl(nn.Cell):
+    """
+    Test class: kl_loss of Gamma distribution.
+    """
+    def __init__(self):
+        super(GammaKl, self).__init__()
+        self.g1 = msd.Gamma(np.array([3.0]), np.array([4.0]), dtype=dtype.float32)
+        self.g2 = msd.Gamma(dtype=dtype.float32)
+
+    def construct(self, concentration1_b, concentration0_b, concentration1_a, concentration0_a):
+        kl1 = self.g1.kl_loss('Gamma', concentration1_b, concentration0_b)
+        kl2 = self.g2.kl_loss('Gamma', concentration1_b, concentration0_b, concentration1_a, concentration0_a)
+        return kl1 + kl2
+
+def test_kl():
+    """
+    Test kl_loss.
+    """
+    net = GammaKl()
+    concentration1_b = Tensor(np.array([1.0]).astype(np.float32), dtype=dtype.float32)
+    concentration0_b = Tensor(np.array([1.0]).astype(np.float32), dtype=dtype.float32)
+    concentration1_a = Tensor(np.array([2.0]).astype(np.float32), dtype=dtype.float32)
+    concentration0_a = Tensor(np.array([3.0]).astype(np.float32), dtype=dtype.float32)
+    ans = net(concentration1_b, concentration0_b, concentration1_a, concentration0_a)
+    assert isinstance(ans, Tensor)
+
+class GammaCrossEntropy(nn.Cell):
+    """
+    Test class: cross_entropy of Gamma distribution.
+    """
+    def __init__(self):
+        super(GammaCrossEntropy, self).__init__()
+        self.g1 = msd.Gamma(np.array([3.0]), np.array([4.0]), dtype=dtype.float32)
+        self.g2 = msd.Gamma(dtype=dtype.float32)
+
+    def construct(self, concentration1_b, concentration0_b, concentration1_a, concentration0_a):
+        h1 = self.g1.cross_entropy('Gamma', concentration1_b, concentration0_b)
+        h2 = self.g2.cross_entropy('Gamma', concentration1_b, concentration0_b, concentration1_a, concentration0_a)
+        return h1 + h2
+
+def test_cross_entropy():
+    """
+    Test cross entropy between Gamma distributions.
+    """
+    net = GammaCrossEntropy()
+    concentration1_b = Tensor(np.array([1.0]).astype(np.float32), dtype=dtype.float32)
+    concentration0_b = Tensor(np.array([1.0]).astype(np.float32), dtype=dtype.float32)
+    concentration1_a = Tensor(np.array([2.0]).astype(np.float32), dtype=dtype.float32)
+    concentration0_a = Tensor(np.array([3.0]).astype(np.float32), dtype=dtype.float32)
+    ans = net(concentration1_b, concentration0_b, concentration1_a, concentration0_a)
+    assert isinstance(ans, Tensor)
+
+class GammaBasics(nn.Cell):
+    """
+    Test class: basic mean/sd function.
+    """
+    def __init__(self):
+        super(GammaBasics, self).__init__()
+        self.g = msd.Gamma(np.array([3.0, 4.0]), np.array([4.0, 6.0]), dtype=dtype.float32)
+
+    def construct(self):
+        mean = self.g.mean()
+        sd = self.g.sd()
+        mode = self.g.mode()
+        return mean + sd + mode
+
+def test_bascis():
+    """
+    Test mean/sd/mode/entropy functionality of Gamma.
+    """
+    net = GammaBasics()
+    ans = net()
+    assert isinstance(ans, Tensor)
+
+class GammaConstruct(nn.Cell):
+    """
+    Gamma distribution: going through construct.
+    """
+    def __init__(self):
+        super(GammaConstruct, self).__init__()
+        self.gamma = msd.Gamma([3.0], [4.0])
+        self.gamma1 = msd.Gamma()
+
+    def construct(self, value, concentration1, concentration0):
+        prob = self.gamma('prob', value)
+        prob1 = self.gamma('prob', value, concentration1, concentration0)
+        prob2 = self.gamma1('prob', value, concentration1, concentration0)
+        return prob + prob1 + prob2
+
+def test_gamma_construct():
+    """
+    Test probability function going through construct.
+    """
+    net = GammaConstruct()
+    value = Tensor([0.5, 1.0], dtype=dtype.float32)
+    concentration1 = Tensor([0.0], dtype=dtype.float32)
+    concentration0 = Tensor([1.0], dtype=dtype.float32)
+    ans = net(value, concentration1, concentration0)
+    assert isinstance(ans, Tensor)