fix(stats): keep regression inference alive when squared deviations underflow

stats/regression.go

@@ -177,6 +177,7 @@ func LinearRegression(x, y *core.Array) (*LinearRegressionResult, error) {
 	rss := 0.0
 	tss := 0.0
 	uncentred := 0.0
+	maxRes, maxDev, maxY := 0.0, 0.0, 0.0
 	mean := 0.0
 	if fy != nil {
 		for _, v := range fy[:n] {
@@ -212,26 +213,85 @@ func LinearRegression(x, y *core.Array) (*LinearRegressionResult, error) {
 		rss += res * res
 		tss += (yv - mean) * (yv - mean)
 		uncentred += yv * yv
+		if a := math.Abs(res); a > maxRes {
+			maxRes = a
+		}
+		if a := math.Abs(yv - mean); a > maxDev {
+			maxDev = a
+		}
+		if a := math.Abs(yv); a > maxY {
+			maxY = a
+		}
 	}
 	if !hasConstant {
 		// The null model is y = 0, so the uncentred total is what the
 		// model has to beat, and it carries n degrees of freedom.
 		tss = uncentred
 	}
+	// The factored sums of squares: a response on a scale whose squared
+	// deviations fall below the subnormal floor reads as a zero sum while
+	// its deviations are live, and the statistics below would report the
+	// evidence backwards (an exact fit the t statistics cannot support,
+	// an F of zero beside them). Each pair keeps the largest deviation as
+	// the scale and the scaled sum as the unit, so scale²·unit is the
+	// true sum wherever the plain product underflows; the unit stays 1
+	// whenever the plain sum already holds.
+	rssScale, rssUnit := 1.0, rss
+	if rss == 0 && maxRes > 0 {
+		rssScale, rssUnit = maxRes, 0.0
+		for _, res := range out.Residuals {
+			d := res / maxRes
+			rssUnit += d * d
+		}
+	}
+	tssScale, tssUnit := 1.0, tss
+	if tss == 0 {
+		devScale := maxDev
+		if !hasConstant {
+			devScale = maxY
+		}
+		if devScale > 0 {
+			tssScale = devScale
+			tssUnit = 0.0
+			for r := range n {
+				var yv, dev float64
+				if fy != nil {
+					yv = fy[r]
+				} else {
+					yv = y.FloatAt(r)
+				}
+				if !hasConstant {
+					dev = yv
+				} else {
+					dev = yv - mean
+				}
+				d := dev / devScale
+				tssUnit += d * d
+			}
+		}
+	}
 	dof := n - p
 	out.ResidualVariance = rss / float64(dof)
-	if tss == 0 {
+	tssDOF := n - 1
+	if !hasConstant {
+		tssDOF = n
+	}
+	if tss == 0 && tssScale == 1 {
 		// A constant response reproduced exactly: R² is 1 by the
 		// perfect-fit convention, not the 1 − 0/0 NaN every consumer
 		// would propagate. The same guard the F statistic below has.
 		out.RSquared = 1
 		out.AdjustedRSquared = 1
+	} else if tss == 0 {
+		// The total underflowed while the response varies: the ratio of
+		// the factored forms, the scale factors divided out one at a
+		// time. Both R² measures round back to 1 here, but the F
+		// statistic below reads the same factored pieces and does not.
+		ratio := rssUnit / tssUnit * (rssScale / tssScale) * (rssScale / tssScale)
+		out.RSquared = 1 - ratio
+		out.AdjustedRSquared = 1 - ratio*float64(tssDOF)/float64(dof)
 	} else {
 		out.RSquared = 1 - rss/tss
-		tssDOF := n - 1
-		if !hasConstant {
-			tssDOF = n
-		}
 		out.AdjustedRSquared = 1 - (rss/float64(dof))/(tss/float64(tssDOF))
 	}
 	out.DModel = p - 1
@@ -271,6 +331,24 @@ func LinearRegression(x, y *core.Array) (*LinearRegressionResult, error) {
 			}
 			out.PValues[j] = pv
 		case v == 0:
+			// The residual sum of squares may have underflowed while the
+			// residuals live: the factored standard error is representable
+			// where the squared one is not, and the t test then reports
+			// the evidence it actually holds instead of an unearned
+			// infinity.
+			if rssScale != 1 && inv[j][j] > 0 {
+				se := rssScale * math.Sqrt(rssUnit*inv[j][j]/float64(dof))
+				if se > 0 {
+					out.StandardErrors[j] = se
+					out.TStatistics[j] = beta[j] / se
+					pv, err := twoSidedT(out.TStatistics[j], dof)
+					if err != nil {
+						return nil, base.Errf("%s: %w", name, err)
+					}
+					out.PValues[j] = pv
+					continue
+				}
+			}
 			// An exact fit: the coefficient is infinitely many standard
 			// errors from zero, and the evidence is total. Reporting
 			// t = 0 next to p = 0 would contradict itself. A zero
@@ -300,6 +378,19 @@ func LinearRegression(x, y *core.Array) (*LinearRegressionResult, error) {
 			explained = 0 // rounding only, and a negative F is meaningless
 		}
 		out.FStatistic = explained / float64(out.DModel) / out.ResidualVariance
+		if (math.IsInf(out.FStatistic, 0) || math.IsNaN(out.FStatistic)) && (rssScale != 1 || tssScale != 1) {
+			// An underflowed sum of squares drove the quotient to Inf or
+			// 0/0 while the factored pieces live: F from the factored
+			// forms, every scale factor applied one division at a time so
+			// no intermediate leaves the representable range before the
+			// answer does. rssUnit 0 is the exact fit, whose F is
+			// genuinely infinite.
+			out.FStatistic = (tssUnit*tssScale/rssScale/rssScale*tssScale - rssUnit) *
+				float64(dof) / (float64(out.DModel) * rssUnit)
+			if out.FStatistic < 0 {
+				out.FStatistic = 0
+			}
+		}
 		if math.IsNaN(out.FStatistic) {
 			// 0/0: a response with no variation at all, reproduced
 			// exactly by the fit. There is no evidence of a model, so
@@ -484,6 +575,7 @@ func WeightedLinearRegression(x, y, w *core.Array) (*LinearRegressionResult, err
 	// regression and reports the uncentred conventions whenever the
 	// weights vary, so the four model-level fields are overwritten here.
 	sumW, sumWY, rssW := 0.0, 0.0, 0.0
+	maxResW := 0.0
 	for r := range n {
 		var wr float64
 		if fw != nil {
@@ -500,8 +592,12 @@ func WeightedLinearRegression(x, y, w *core.Array) (*LinearRegressionResult, err
 		sumW += wr
 		sumWY += wr * yv
 		rssW += wr * out.Residuals[r] * out.Residuals[r]
+		if a := math.Abs(out.Residuals[r]); a > maxResW {
+			maxResW = a
+		}
 	}
 	tssW := 0.0
+	maxDevW := 0.0
 	if hasConstant {
 		meanW := sumWY / sumW
 		for r := range n {
@@ -518,6 +614,9 @@ func WeightedLinearRegression(x, y, w *core.Array) (*LinearRegressionResult, err
 			}
 			d := yv - meanW
 			tssW += wr * d * d
+			if a := math.Abs(d); a > maxDevW {
+				maxDevW = a
+			}
 		}
 	} else {
 		// Without an intercept the null model is zero, so Σw·y² is the
@@ -535,16 +634,85 @@ func WeightedLinearRegression(x, y, w *core.Array) (*LinearRegressionResult, err
 				yv = y.FloatAt(r)
 			}
 			tssW += wr * yv * yv
+			if a := math.Abs(yv); a > maxDevW {
+				maxDevW = a
+			}
+		}
+	}
+	// The weighted sums of squares carry the same factored form the
+	// unweighted fit keeps: a response scale whose weighted squared
+	// deviations fall below the subnormal floor reads as a zero sum
+	// while the deviations live, and the F below would report zero
+	// evidence beside the t statistics' infinity.
+	rssScaleW, rssUnitW := 1.0, rssW
+	if rssW == 0 && maxResW > 0 {
+		rssScaleW = maxResW
+		rssUnitW = 0.0
+		for r := range n {
+			var wr float64
+			if fw != nil {
+				wr = fw[r]
+			} else {
+				wr = w.FloatAt(r)
+			}
+			d := out.Residuals[r] / maxResW
+			rssUnitW += wr * d * d
+		}
+	}
+	tssScaleW, tssUnitW := 1.0, tssW
+	if tssW == 0 && maxDevW > 0 {
+		tssScaleW = maxDevW
+		tssUnitW = 0.0
+		if hasConstant {
+			meanW := sumWY / sumW
+			for r := range n {
+				var wr, yv float64
+				if fw != nil {
+					wr = fw[r]
+				} else {
+					wr = w.FloatAt(r)
+				}
+				if fy != nil {
+					yv = fy[r]
+				} else {
+					yv = y.FloatAt(r)
+				}
+				d := (yv - meanW) / maxDevW
+				tssUnitW += wr * d * d
+			}
+		} else {
+			for r := range n {
+				var wr, yv float64
+				if fw != nil {
+					wr = fw[r]
+				} else {
+					wr = w.FloatAt(r)
+				}
+				if fy != nil {
+					yv = fy[r]
+				} else {
+					yv = y.FloatAt(r)
+				}
+				d := yv / maxDevW
+				tssUnitW += wr * d * d
+			}
 		}
 	}
 	tssDOF := n - 1
 	if !hasConstant {
 		tssDOF = n
 	}
-	if tssW == 0 {
+	if tssW == 0 && tssScaleW == 1 {
 		// Constant weighted response, exact fit: 1, as above.
 		out.RSquared = 1
 		out.AdjustedRSquared = 1
+	} else if tssW == 0 {
+		// The weighted total underflowed while the weighted response
+		// varies: the factored ratio, both R² measures rounding back
+		// to 1 while the F below reads the same pieces and does not.
+		ratio := rssUnitW / tssUnitW * (rssScaleW / tssScaleW) * (rssScaleW / tssScaleW)
+		out.RSquared = 1 - ratio
+		out.AdjustedRSquared = 1 - ratio*float64(tssDOF)/float64(out.DResidual)
 	} else {
 		out.RSquared = 1 - rssW/tssW
 		out.AdjustedRSquared = 1 - (rssW/float64(out.DResidual))/(tssW/float64(tssDOF))
@@ -559,6 +727,15 @@ func WeightedLinearRegression(x, y, w *core.Array) (*LinearRegressionResult, err
 			explained = 0 // rounding only, and a negative F is meaningless
 		}
 		out.FStatistic = explained / float64(out.DModel) / out.ResidualVariance
+		if (math.IsInf(out.FStatistic, 0) || math.IsNaN(out.FStatistic)) && (rssScaleW != 1 || tssScaleW != 1) {
+			// The factored F, as in the unweighted path: every scale
+			// factor divided out one step at a time.
+			out.FStatistic = (tssUnitW*tssScaleW/rssScaleW/rssScaleW*tssScaleW - rssUnitW) *
+				float64(out.DResidual) / (float64(out.DModel) * rssUnitW)
+			if out.FStatistic < 0 {
+				out.FStatistic = 0
+			}
+		}
 		if math.IsNaN(out.FStatistic) {
 			// 0/0, as in the unweighted path: nothing to test, p = 1.
 			out.FStatistic = 0