xsf/scipy_special/logsumexp.rs
1//! Translated from `scipy/special/_logsumexp.py`
2
3const NINF: f64 = f64::NEG_INFINITY;
4
5/// Return the maximum value in a slice of f64, propagating NaNs
6#[inline]
7fn fmax(xs: &[f64]) -> f64 {
8 xs.iter()
9 .fold(NINF, |a, &b| if a >= b || a.is_nan() { a } else { b })
10}
11
12/// Compute the log of the sum of exponentials of input elements
13///
14/// Pure Rust translation of [`scipy.special.logsumexp`][scipy].
15///
16/// [scipy]: https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.logsumexp.html
17///
18/// # Definition
19///
20/// For a sequence of real values $\vec{\bm{x}}$, `logsumexp` is defined as
21///
22/// $$ \ln \sum_{x \in \vec{\bm{x}}} e^x $$
23///
24/// The implementation uses a numerically stable equivalent algorithm which factors out the maximum
25/// value.
26///
27/// # Notes
28///
29/// Unlike the SciPy version, this implementation only supports 1-dimensional slices, and doesn't
30/// support complex numbers at the moment.
31///
32/// # See also
33/// - [`logaddexp`](crate::logaddexp)
34/// - [`softmax`]
35#[must_use]
36#[allow(clippy::missing_panics_doc)]
37pub fn logsumexp(xs: &[f64]) -> f64 {
38 let x_max = fmax(xs);
39
40 // fast-path for -/+inf and NaN
41 if !x_max.is_finite() {
42 return x_max;
43 }
44
45 // for precision, these terms are separated out of the main sum.
46 let m: f64 = xs
47 .iter()
48 .fold(0.0, |acc, &x| if x == x_max { acc + 1.0 } else { acc });
49 let xs = xs.iter().map(|&x| if x == x_max { NINF } else { x });
50
51 // Shift, exponentiate, and sum
52 let s = xs.map(|x| (x - x_max).exp()).sum::<f64>();
53
54 // Take log and undo shift
55 (s / m).ln_1p() + m.ln() + x_max
56}
57
58/// Compute the softmax function
59///
60/// Pure Rust translation of [`scipy.special.softmax`][scipy].
61///
62/// [scipy]: https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.softmax.html
63///
64/// # Definition
65///
66/// The formula for the softmax function $\sigma(\bm{x})$ for a vector
67/// $\bm{x} = \begin{bmatrix} x_0 & x_1 & \cdots & x_{n-1} \end{bmatrix}$ is
68///
69/// $$
70/// \sigma(\bm{x})_j = {e^{x_j} \over \sum_k e^{x_k}}
71/// $$
72///
73/// The `softmax` function is the gradient of [`logsumexp`].
74///
75/// The implementation uses shifting to improve numerical stability.
76///
77/// # See also
78/// - [`log_softmax`]
79/// - [`logsumexp`]
80#[must_use]
81pub fn softmax(xs: &[f64]) -> Vec<f64> {
82 let x_max = fmax(xs);
83 let exs0 = xs.iter().map(|&x| (x - x_max).exp()).collect::<Vec<f64>>();
84 let s = exs0.iter().sum::<f64>().recip();
85 exs0.iter().map(|ex| ex * s).collect()
86}
87
88/// Compute the logarithm of the [`softmax`] function
89///
90/// Pure Rust translation of [`scipy.special.log_softmax`][scipy].
91///
92/// [scipy]: https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.log_softmax.html
93///
94/// # Notes
95///
96/// `log_softmax` is more accurate than taking the log of [`softmax`] with inputs that make
97/// [`softmax`] saturate.
98///
99/// # See also
100/// - [`softmax`]
101/// - [`logsumexp`]
102#[must_use]
103pub fn log_softmax(xs: &[f64]) -> Vec<f64> {
104 let x_max = fmax(xs);
105 let x_max = if x_max.is_finite() { x_max } else { 0.0 };
106
107 let xs0 = xs.iter().map(|&x| x - x_max).collect::<Vec<f64>>();
108 // Unlike in SciPy, `logsumexp` takes `ln_1p` of the terms below the maximum, as the result
109 // for the maximum is about minus their sum, which `ln` of the whole sum rounds away
110 let s = logsumexp(&xs0);
111 xs0.iter().map(|&x| x - s).collect()
112}