Information Theory for ML

// Entropy: H(X) = -sum(p(x) * log2(p(x)))
// It measures the average surprise of events from a distribution

function entropy(probs: number[]): number {
return -probs.reduce((sum, p) => {
  if (p === 0) return sum;  // 0 * log(0) = 0 by convention
  return sum + p * Math.log2(p);
}, 0);
}

// Fair coin: maximum uncertainty
console.log('Fair coin:', entropy([0.5, 0.5]).toFixed(4));
// 1.0000 bit: you need exactly 1 bit (0 or 1) to encode each flip

// Biased coin (90% heads)
console.log('Biased coin:', entropy([0.9, 0.1]).toFixed(4));
// 0.4690 bits: less surprise, more compressible

// Certain outcome (100% heads)
console.log('Certain:', entropy([1.0, 0.0]).toFixed(4));
// 0.0000 bits: no surprise at all, nothing to encode

// Uniform distribution over 8 classes
const uniform8 = Array(8).fill(1/8);
console.log('Uniform 8-class:', entropy(uniform8).toFixed(4));
// 3.0000 bits: need 3 bits to encode 8 equally likely outcomes

// Connection to gzip:
// High-entropy file (random bytes) -> poor compression ratio
// Low-entropy file (repeated patterns) -> great compression ratio
// Entropy IS the compression limit

import * as tf from '@tensorflow/tfjs';

function crossEntropy(trueProbs: number[], predictedProbs: number[]): number {
return -trueProbs.reduce((sum, p, i) => {
  if (p === 0) return sum;
  return sum + p * Math.log2(predictedProbs[i]);
}, 0);
}

// True distribution: cat with 100% certainty
const trueLabel = [1.0, 0.0, 0.0]; // [cat, dog, bird]

// Good prediction
const goodPred = [0.9, 0.05, 0.05];
console.log('Good prediction CE:', crossEntropy(trueLabel, goodPred).toFixed(4));
// 0.1520 bits: low cost, model is close

// Bad prediction
const badPred = [0.1, 0.6, 0.3];
console.log('Bad prediction CE:', crossEntropy(trueLabel, badPred).toFixed(4));
// 3.3219 bits: high cost, model is very wrong

// Terrible prediction (almost certain it's NOT a cat)
const terriblePred = [0.01, 0.90, 0.09];
console.log('Terrible prediction CE:', crossEntropy(trueLabel, terriblePred).toFixed(4));
// 6.6439 bits: massive cost for confident wrong answer

// This is why cross-entropy penalizes confident wrong predictions
// so harshly: the log makes small probabilities very expensive

// TensorFlow.js built-in (uses natural log, not log2).
// softmaxCrossEntropy expects LOGITS, the raw pre-softmax scores,
// not probabilities. Log-probabilities work as logits here because
// softmax(ln(p)) recovers p exactly when p sums to 1.
const labels = tf.tensor2d([[1, 0, 0]]);
const logits = tf.tensor2d([
[Math.log(0.9), Math.log(0.05), Math.log(0.05)]
]);
const ce = tf.losses.softmaxCrossEntropy(labels, logits);
console.log('TF cross-entropy:', await ce.array());
// 0.1054 nats. Divide by ln(2) and you get 0.1520 bits,
// the same number we computed by hand for goodPred above.

function klDivergence(p: number[], q: number[]): number {
return p.reduce((sum, pi, i) => {
  if (pi === 0) return sum;
  return sum + pi * Math.log2(pi / q[i]);
}, 0);
}

const p = [0.7, 0.2, 0.1]; // true distribution
const q = [0.3, 0.4, 0.3]; // model's distribution

console.log('KL(p || q):', klDivergence(p, q).toFixed(4));
// Positive: q is not a perfect model of p

console.log('KL(q || p):', klDivergence(q, p).toFixed(4));
// Different value! KL divergence is ASYMMETRIC
// KL(p||q) != KL(q||p)
// This asymmetry has practical consequences:
// - Minimizing KL(p||q) = minimizing cross-entropy (what we do in training)
// - Minimizing KL(q||p) = mode-seeking (used in variational inference)

// KL divergence of p from itself
console.log('KL(p || p):', klDivergence(p, p).toFixed(4));
// 0.0000: zero when distributions match

// Why cross-entropy loss works for classification:
// Minimizing CE(true, predicted) is equivalent to minimizing
// KL(true || predicted), because the entropy of the true labels
// is constant. We're directly minimizing the information gap.

// Consider a confident wrong prediction: true=[1,0], pred=[0.01, 0.99]
const trueLabel = 1.0;

// MSE gradient at pred=0.01:
// d/dp (1 - p)^2 = -2(1-p) = -2(0.99) = -1.98
const mseGrad = -2 * (trueLabel - 0.01);

// Cross-entropy gradient at pred=0.01:
// d/dp -log(p) = -1/p = -1/0.01 = -100
const ceGrad = -1 / 0.01;

console.log('MSE gradient:', mseGrad.toFixed(2));    // -1.98
console.log('CE gradient:', ceGrad.toFixed(2));        // -100.00

// Cross-entropy produces a MUCH stronger gradient for confident
// wrong predictions. This is why CE trains faster for classification:
// it screams "you're wrong!" when the model is confidently incorrect,
// while MSE just whispers.

// Information-theoretic reason: CE directly minimizes the information
// gap. MSE minimizes squared distance, which has no information
// theoretic justification for probability distributions.