fix(signal): refuse filter designs whose coefficients overflow

Assisted-by: GLM 5.3 Flash

signal/filter.go

@@ -360,6 +360,9 @@ func butterworth(order int, fs, cutoff float64, highpass bool) (b, a []float64,
 			}
 		}
 		k := polyEvalAtMinusOne(a) / math.Pow(2, float64(order))
+		if err := designCoefficientsFinite(name, order, k, b, a); err != nil {
+			return nil, nil, err
+		}
 		for i := range b {
 			b[i] *= k
 		}
@@ -367,12 +370,35 @@ func butterworth(order int, fs, cutoff float64, highpass bool) (b, a []float64,
 	}
 	b = binomialCoeffs(order)
 	k := polyEvalAtOne(a) / math.Pow(2, float64(order))
+	if err := designCoefficientsFinite(name, order, k, b, a); err != nil {
+		return nil, nil, err
+	}
 	for i := range b {
 		b[i] *= k
 	}
 	return b, a, nil
 }
 
+// designCoefficientsFinite refuses a design whose arithmetic left the
+// float64 range. Past an order of about a thousand the gain divides by
+// an infinite 2^order and the binomial numerator overflows with it,
+// answers that would otherwise ship as a numerator of zeros or NaN
+// presented as a filter: the gain must be finite and non-zero, and
+// every coefficient of both polynomials finite.
+func designCoefficientsFinite(name string, order int, gain float64, b, a []float64) error {
+	if math.IsNaN(gain) || math.IsInf(gain, 0) || gain == 0 {
+		return base.Errf("%s: order %d overflows the coefficient arithmetic; use a lower order", name, order)
+	}
+	for _, poly := range [2][]float64{b, a} {
+		for _, v := range poly {
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return base.Errf("%s: order %d overflows the coefficient arithmetic; use a lower order", name, order)
+			}
+		}
+	}
+	return nil
+}
+
 // mulPolyReal multiplies two real polynomials in u = z^{-1} (index m
 // is the coefficient of u^m).
 func mulPolyReal(p, q []float64) []float64 {