math writeup first draft

.gitignore

@@ -4,3 +4,5 @@ build/
 /output/
 /.vscode/
 /.pixi/
+*.aux
+*.log

docs/fmb-math.qmd

@@ -0,0 +1,190 @@
+\newcommand{\L}{\mathcal{L}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\SE}{\mathrm{SE}}
+\newcommand{\argmax}{\operatorname*{argmax}}
+
+# Mathematical Description of Fuzzy Metaball Rendering
+
+## Notation
+
+$\N_n$ denotes the set $\{1, 2, \ldots, n\}$. $\SE(d)$ denotes the group of
+$d$-dimensional rigid transformations (hence $\SE(3)$ is the group of poses).
+$S^d$ denotes the $d$-dimensional sphere, i.e. the set of $d + 1$ dimensional
+unit vectors.
+
+The notation $\mathrm{symbol} := \mathrm{expression}$ offers the definition of "symbol" in terms of the "expression."
+
+Greek letters are unknowns, Latin letters are known
+
+Writing $(\tau, \rho) := \pi $ for $\pi \in \SE(3)$ means that $\pi$'s translational
+component is $\tau$ and it's rotational component is $\rho$. For $v \in \R^d$
+we write $\pi\cdot v$ to denote $v$ transformed by $\pi$.
+
+| Symbol | Type | Meaning |
+|:------:|:----:|:--------|
+| $N$    | $\N$ | number of fuzzy metaballs |
+| $T$    | $\N$ | number of time steps (frames) |
+| $H$    | $\N$ | height of each observed frame in pixels |
+| $W$    | $\N$ | width of each observed frame in pixels |
+| $i$    | $\N_H$ | index ranging over the rows in each observed frame |
+| $j$    | $\N_W$ | index ranging over the columns in each observed frame |
+| $k$    | $\N_N$ | index ranging over the fuzzy metaballs |
+| $t$    | $\N_T$ | index ranging over frames |
+| $v_{ij}$ | $S^2$ | direction of the ray corresponding to pixel $(i, j)$ in the camera frame |
+| $\pi^{(t)}$ | $\SE(3)$ | camera pose in world frame at frame $t$ |
+| $\mu_k$ | $\R^3$ | the mean vector of the $k$-the fuzzy metaball |
+| $\Sigma_k$ | $\R^{3\times 3}_{\succ 0}$ | the covariance matrix of the $k$-the fuzzy metaball |
+| $\lambda_k$ | $\R$ | log-weight of the $k$-the fuzzy metaball |
+| $d_{ijk}^{(t)}$    | $\R$ | intersection depth of ray corresponding to pixel $(i, j)$ at frame $t$ with the $k$-th fuzzy metaball |
+| $q_{ijk}^{(t)}$    | $\R$ | the value of the quadratic form of $k$-th fuzzy metaball at the intersection of ray $(i, j)$ at frame $t$ |
+| $w_{ijk}^{(t)}$    | $\R$ | depth blending weight of the $k$-th fuzzy metaball for pixel $(i, j)$ at frame $t$ |
+| $\bar{d}_{ij}^{(t)}$    | $\R$ | depth value of pixel $(i, j)$ at frame $t$ |
+| $c_{ij}^{(t)}$    | $[0, 1]$ | confidence value of pixel $(i, j)$ at frame $t$ |
+| $\L_c$    | $\R$ | contour loss  (a function of $c_{ij}^{(t)}$) |
+| $\L_d$    | $\R$ | depth loss  (a function of $d_{ij}^{(t)}$) |
+
+
+## Forward Pass
+
+\begin{align*}
+%\argmax_{d \in \R} (\mu_k - \Delta x^{(t)} - dv_{ij}^{(t)})^\top
+%\Sigma^{-1} (\mu_k - \Delta x^{(t)} - dv_{ij}^{(t)}) \\
+(\tau^{(t)}, \rho^{(t)}) &:= \pi^{(t)} \\
+v_{ij}^{(t)} &:= \rho^{(t)}v_{ij} \\
+d_{ijk}^{(t)} &:= \frac{(\mu_k - \tau^{(t)})^\top\Sigma_k^{-1}v_{ij}^{(t)}}
+    {{v_{ij}^{(t)}}^\top\Sigma_k^{-1}v_{ij}^{(t)}} \\
+q_{ijk}^{(t)} &:= (\mu_k - \tau^{(t)} - d_{ijk}^{(t)}v_{ij}^{(t)})^\top
+    \Sigma_k^{-1} (\mu_k - \tau^{(t)} - d_{ijk}^{(t)}v_{ij}^{(t)}) \\
+\tilde{w}_{ijk}^{(t)} &:=
+    \exp\left(\beta_1(-q_{ijk}^{(t)}/2 + \lambda_k) - \beta_2 d_{ijk}^{(t)}\right) \\
+w_{ijk}^{(t)} &:= \frac{\tilde{w}_{ijk}^{(t)}}{\sum_{k=1}^N \tilde{w}_{ijk}^{(t)}} \\
+\bar{d}_{ij}^{(t)} &:= \sum_{k=1}^N w_{ijk}^{(t)} d_{ijk}^{(t)} \\
+c_{ij}^{(t)} &:= 1 - \exp\left(\sum_{k=1} \exp(q_{ijk}^{(t)}/2 - \lambda_k)\right)
+\end{align*}
+
+![Dependency graph of the variables in the forward pass using plate
+notation](plate_diagram.png){width=540}
+
+## Depth Backward Pass
+
+\begin{align*}
+\frac{\partial \L_d}{\partial \tilde{w}_{ijk}^{(t)}} &= 
+    \frac{\partial \L_d}{\partial \bar{d}_{ij}^{(t)}}
+    \frac{\partial \bar{d}_{ij}^{(t)}}{\partial \tilde{w}_{ijk}^{(t)}}\\
+\frac{\partial \L_d}{\partial q_{ijk}^{(t)}} &= 
+    \frac{\partial \L_d}{\partial \tilde{w}_{ijk}^{(t)}}
+    \frac{\partial \tilde{w}_{ijk}^{(t)}}{\partial q_{ijk}^{(t)}}\\
+\frac{\partial \L_d}{\partial d_{ijk}^{(t)}} &= 
+    \frac{\partial \L_d}{\partial q_{ijk}^{(t)}}
+    \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+    + \frac{\partial \L_d}{\partial \tilde{w}_{ijk}^{(t)}}
+    \frac{\partial \tilde{w}_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+    + \frac{\partial \L_d}{\partial \bar{d}_{ij}^{(t)}}
+    \frac{\partial \bar{d}_{ij}^{(t)}}{\partial d_{ijk}^{(t)}} \\
+\frac{\partial \L_d}{\partial \tau^{(t)}} &=
+    \sum_{i,j}\sum_k \frac{\partial \L_d}{\partial q_{ijk}^{(t)}}
+    \frac{\partial q_{ijk}^{(t)}}{\partial \tau^{(t)}}
+    + \sum_{i,j}\sum_k \frac{\partial \L_d}{\partial d_{ijk}^{(t)}}
+    \frac{\partial d_{ijk}^{(t)}}{\partial \tau^{(t)}} \\
+\frac{\partial \L_d}{\partial v_{ij}^{(t)}} &=
+    \sum_k \frac{\partial \L_d}{\partial q_{ijk}^{(t)}}
+    \frac{\partial q_{ijk}^{(t)}}{\partial v_{ij}^{(t)}}
+    + \sum_k \frac{\partial \L_d}{\partial d_{ijk}^{(t)}}
+    \frac{\partial d_{ijk}^{(t)}}{\partial v_{ij}^{(t)}} \\
+\frac{\partial \L_d}{\partial \rho^{(t)}} &=
+    \sum_{i,j} \frac{\partial \L_d}{\partial v_{ij}^{(t)}}
+           \frac{\partial v_{ij}^{(t)}}{\partial \rho^{(t)}} \\
+\frac{\partial \L_d}{\partial \lambda_k} &= 
+    \sum_t \sum_{i, j}
+    \frac{\partial \L_d}{\partial \tilde{w}_{ijk}^{(t)}}
+    \frac{\partial \tilde{w}_{ijk}^{(t)}}{\partial \lambda_k} \\
+\frac{\partial \L_d}{\partial \mu_k} &= 
+    \sum_t \sum_{i, j} \frac{\partial \L_d}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial \mu_k}
+    + \sum_t \sum_{i, j} \frac{\partial \L_d}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial \mu_k} \\
+\frac{\partial \L_d}{\partial \Sigma_k} &= 
+    \sum_t \sum_{i, j} \frac{\partial \L_d}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial \Sigma_k}
+    + \sum_t \sum_{i, j} \frac{\partial \L_d}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial \Sigma_k} \\
+\end{align*}
+
+
+## Confidence Backward Pass
+
+\begin{align*}
+\frac{\partial \L_c}{\partial d_{ijk}^{(t)}} &= 
+    \frac{\partial \L_c}{\partial q_{ijk}^{(t)}}
+    \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}\\
+\frac{\partial \L_c}{\partial q_{ijk}^{(t)}} &= 
+    \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+    \frac{\partial c_{ij}^{(t)}}{\partial q_{ijk}^{(t)}}\\
+\frac{\partial \L_c}{\partial \tau^{(t)}} &= 
+    \sum_{i,j}\sum_k \frac{\partial \L_c}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial \tau^{(t)}}
+    + \sum_{i,j}\sum_k \frac{\partial \L_c}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial \tau^{(t)}}\\
+\frac{\partial \L_c}{\partial v_{ij}^{(t)}} &= 
+    \sum_{k} \frac{\partial \L_c}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial v_{ij}^{(t)}}
+    + \sum_{k} \frac{\partial \L_c}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial v_{ij}^{(t)}} \\
+\frac{\partial \L_c}{\partial \rho^{(t)}} &= 
+    \sum_{i,j} \frac{\partial \L_c}{\partial v_{ij}^{(t)}}
+        \frac{\partial v_{ij}^{(t)}}{\partial \rho^{(t)}} \\
+\frac{\partial \L_c}{\partial \lambda_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+        \frac{\partial c_{ij}^{(t)}}{\partial \lambda_k} \\
+\frac{\partial \L_c}{\partial \mu_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial \mu_k}
+    + \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial \mu_k} \\
+\frac{\partial \L_c}{\partial \Sigma_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial q_{ijk}^{(t)}}
+        \frac{\partial q_{ijk}^{(t)}}{\partial \Sigma_k}
+    + \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial \Sigma_k}
+\end{align*}
+
+We can simplify the backward pass by inlining those derivatives that depend on
+fixed $i$, $j$, and $t$. We get:
+
+\begin{align*}
+\frac{\partial \L_c}{\partial \tau^{(t)}} &= 
+    \sum_{i,j}\sum_k \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+        \frac{\partial c_{ij}^{(t)}}{\partial q_{ijk}^{(t)}} \left(
+            \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+            \frac{\partial d_{ijk}^{(t)}}{\partial \tau^{(t)}}
+            + \frac{\partial q_{ijk}^{(t)}}{\partial \tau^{(t)}}
+      \right)\\
+\frac{\partial \L_c}{\partial \rho^{(t)}} &= 
+    \sum_{i,j} \sum_{k} \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+    \frac{\partial c_{ij}^{(t)}}{\partial q_{ijk}^{(t)}} \left(
+        \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+        \frac{\partial d_{ijk}^{(t)}}{\partial v_{ij}^{(t)}}
+        + \frac{\partial q_{ijk}^{(t)}}{\partial v_{ij}^{(t)}}
+    \right) 
+        \frac{\partial v_{ij}^{(t)}}{\partial \rho^{(t)}} \\
+\frac{\partial \L_c}{\partial \lambda_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+        \frac{\partial c_{ij}^{(t)}}{\partial \lambda_k} \\
+\frac{\partial \L_c}{\partial \mu_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+        \frac{\partial c_{ij}^{(t)}}{\partial q_{ijk}^{(t)}} \left(
+        \frac{\partial q_{ijk}^{(t)}}{\partial \mu_k}
+        + \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+          \frac{\partial d_{ijk}^{(t)}}{\partial \mu_k}
+    \right) \\
+\frac{\partial \L_c}{\partial \Sigma_k} &= 
+    \sum_{t}\sum_{i,j} \frac{\partial \L_c}{\partial c_{ij}^{(t)}}
+        \frac{\partial c_{ij}^{(t)}}{\partial q_{ijk}^{(t)}} \left(
+        \frac{\partial q_{ijk}^{(t)}}{\partial \Sigma_k}
+        + \frac{\partial q_{ijk}^{(t)}}{\partial d_{ijk}^{(t)}}
+          \frac{\partial d_{ijk}^{(t)}}{\partial \Sigma_k}
+    \right) \\
+\end{align*}
+
+## 

docs/make_fig.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+pdflatex plate_diagram.tex
+convert -verbose -density 300 -trim plate_diagram.pdf -size 1080x1080 -quality 100 -flatten -sharpen 0x1.0 plate_diagram.png

docs/plate_diagram.tex

@@ -0,0 +1,77 @@
+\documentclass[border=10pt]{standalone}
+\usepackage{tikz}
+\usepackage{amsmath,amssymb}
+
+\usetikzlibrary{arrows.meta, positioning, fit, backgrounds}
+
+\begin{document}
+% if plate p binds index i and variable v appears in p, then v should vary over
+% i
+\begin{tikzpicture}[
+    node distance=1cm and 1cm,
+    >={Stealth[length=2mm]},
+    every node/.style={font=\small},
+    every edge/.style = {draw, -latex}
+]
+
+% Define coordinates for nodes
+\node (lambdak) at ( -1, 5) {$\lambda_k$};
+\node (muk)  at (  0, 5) {$\mu_k$};
+\node (Sigmak)     at (  1, 5) {$\Sigma_k$};
+
+\node (vij)     at (2.5, 4) {$v_{ij}^{(t)}$};
+\node (rho)     at (  4, 4) {$\rho^{(t)}$};
+
+\node (tau)     at (  4, 3.5) {$\tau^{(t)}$};
+
+\node (qijk)    at (  0, 2.5) {$q_{ijk}^{(t)}$};
+\node (dijk)    at (  1, 2.5) {$d_{ijk}^{(t)}$};
+
+\node (wijk)    at (  0, 1) {$\tilde{w}_{ijk}^{(t)}$};
+
+\node (cij)     at ( -1, -0.2) {$c_{ij}^{(t)}$};
+\node (dijbar)  at (  1, -0.2) {$\bar{d}_{ij}^{(t)}$};
+
+\node (Lc)      at ( -1, -1.4) {$\mathcal{L}_c$};
+\node (Ld)      at (  1, -1.4) {$\mathcal{L}_d$};
+
+
+\draw (Sigmak.south) edge (qijk.north) edge (dijk.north)
+      (muk.south) edge (qijk.north) edge (dijk.north)
+      (vij.south) edge (qijk.north) edge (dijk.north)
+      (rho.west) edge (vij.east)
+      (tau.west) edge (qijk.north) edge (dijk.north)
+      (lambdak.south) edge (wijk.north) edge (cij.north)
+      (qijk.south) edge (wijk.north) edge (cij.north)
+      (dijk.south) edge (wijk.north) edge (dijbar.north)
+      (dijk.west) edge (qijk.east)
+      (wijk.south) edge (dijbar.north)
+      (cij) edge (Lc)
+      (dijbar) edge (Ld);
+
+% Inner plate k
+\begin{scope}[on background layer]
+    \node[draw, rectangle, rounded corners=2pt, 
+          inner sep=4pt,
+          fit=(Sigmak)(muk)(lambdak)(wijk)(dijk),
+          label={[anchor=south east, font=\small]south east:$k$}] (plate_k) {};
+\end{scope}
+
+% Middle plate (i,j)
+\begin{scope}[on background layer]
+    \node[draw, rectangle, rounded corners=2pt,
+          inner sep=4pt,
+          fit=(wijk)(dijk)(vij)(cij)(dijbar),
+          label={[anchor=south east, font=\small]south east:$(i,j)$}] (plate_ij) {};
+\end{scope}
+
+% Outer plate t
+\begin{scope}[on background layer]
+    \node[draw, rectangle, rounded corners=2pt,
+          inner sep=4pt,
+          fit=(plate_ij)(rho)(tau),
+          label={[anchor=south east, font=\small]south east:$t$}] (plate_t) {};
+\end{scope}
+
+\end{tikzpicture}
+\end{document}