created stochastic control

Mathematics/5th/Stochastic_calculus/Stochastic_calculus.tex

@@ -137,7 +137,7 @@
   \begin{proof}
     By \mnameref{SC:doob_maximal} applied to $M^n-M^m$ we have that for fixed $t\geq 0$ and $k\in\NN$:
     $$
-      \Prob\left(\sup_{0\leq s\leq t}\abs{M_s^n-M_s^m}\geq \frac{1}{kmn }\right)\leq k^2\Exp({(M_t^n-M_t^m)}^2)\leq \frac{1}{k^2}
+      \Prob\left(\sup_{0\leq s\leq t}\abs{M_s^n-M_s^m}\geq \frac{1}{k^2}\right)\leq k^2\Exp({(M_t^n-M_t^m)}^2)\leq \frac{1}{k^2}
     $$
     where in the last inequality we have used that ${(M^n)}$ converges in $L^2$ and so we have chosen $n,m$ large enough so that the inequality holds. Thus, there is an increasing sequence $(n_k)$ such that:
     $$
@@ -808,7 +808,7 @@
     $$
   \end{definition}
   \subsubsection{Existence and uniqueness of solutions}
-  \begin{lemma}[Gronwall's lemma]
+  \begin{lemma}[Gronwall's lemma]\label{SC:gronwall}
     Let ${(x_t)}_{t\in[0,T]}$ be a non-negative function in $L^1([0,T])$ satisfying that $\forall t\in[0,T]$:
     $$
       x_t\leq \alpha+\beta\int_0^t x_s\dd{s}

Mathematics/5th/Stochastic_control/Stochastic_control.tex

@@ -0,0 +1,142 @@
+\documentclass[../../../main_math.tex]{subfiles}
+
+\begin{document}
+\changecolor{SCO}
+\begin{multicols}{2}[\section{Stochastic control}]
+  \subsection{SDEs and Feynman-Kac formula}
+  In this section we will work in a filtered space $(\Omega, \mathcal{F}, \Prob, {(\mathcal{F}_t)}_{t\geq 0})$. We will denote by $\vf{X}={(\vf{X}_t)}_{t\geq 0}$ a $d$-dimensional Itô process
+  $$
+    \vf{X}_t=\vf{X}_0+\int_0^t \vf{b}(s,\vf{X}_s)ds+\int_0^t \vf{\sigma}(s,\vf{X}_s)d\vf{B}_s
+  $$
+  with $\vf{b}:[0,T]\times\RR^d\to \RR^d$, $\vf{\sigma}:[0,T]\times\RR^d\to \RR^{d\times m}$ be progressively measurable with respect to ${(\mathcal{F}_t)}_{t\geq 0}$ and  $\vf{b}\in\MM^1_{\text{loc}}(\RR^d)$, $\vf{\sigma}\in\MM^2_{\text{loc}}(\RR^d)$.
+  \subsubsection{Itô's formula}
+  \begin{theorem}[Itô's formula]\label{SCO:ito_formula}
+    For all $\varphi:\RR_{\geq 0}\times\RR^d\to\RR$ of class $\mathcal{C}^{1,2}$, we have:
+    \begin{multline*}
+      \varphi(t,\vf{X}_t)=\varphi(0,\vf{X}_0)+\\+\int_0^t \left(\pdv{\varphi}{s}+\vf{b}\cdot\grad\varphi+\frac{1}{2}\tr\left[\vf{\sigma}\transpose{\vf\sigma}\vf{D}\varphi\right]\right)\dd{s}+\\+\int_0^t \grad\varphi\cdot\vf{\sigma}\dd{\vf{B}_s}
+    \end{multline*}
+  \end{theorem}
+  \subsubsection{SDEs}
+  Suppose now that $\vf{b}$ and $\vf\sigma$ are Lipschitz-continuous in the second variable and consider the following SDE:
+  \begin{equation}\label{SCO:SDE}
+    \begin{cases}
+      d\vf{X}_t=\vf{b}(t,\vf{X}_t)\dd{t}+\vf{\sigma}(t,\vf{X}_t)\dd{\vf{B}_t} \\
+      \vf{X}_{t_0}=\vf{x}_0
+    \end{cases}
+  \end{equation}
+  with $t_0\geq 0$.
+  \begin{proposition}
+    Let ${(\vf{X}_t^{t_0,\vf{x}_0})}_{t\geq t_0}$ be the solution of \eqref{SCO:SDE}. Then, for all $T>0$ $\exists C_T>0$ such that:
+    \begin{enumerate}
+      \item
+            $$
+              \Exp\left(\sup_{s\in [t_0,T]}\abs{\vf{X}_s^{t_0,\vf{x}_0}-\vf{X}_s^{t_0,\vf{y}_0}}^2\right)\leq C_T\abs{\vf{x}_0-\vf{y}_0}^2
+            $$
+      \item $$
+              \Exp\left(\sup_{s\in [t_0,T]}\abs{\vf{X}_s^{t_0,\vf{x}_0}}^2\right)\leq C_T(1+\abs{\vf{x}_0}^2)
+            $$
+      \item For all $t_0\leq t_1<t_2$:
+            $$
+              \Exp\left(\abs{\vf{X}_{t_2}^{t_0,\vf{x}_0}-\vf{X}_{t_1}^{t_0,\vf{x}_0}}^2\right)\leq C_T\abs{t_2-t_1}
+            $$
+    \end{enumerate}
+  \end{proposition}
+  \begin{proof}
+    We only proof the first point. The others are similar. We have:
+    \begin{multline*}
+      \abs{\vf{X}_s^{t_0,\vf{x}_0}-\vf{X}_s^{t_0,\vf{y}_0}}^2\leq \abs{\vf{x}_0-\vf{y}_0}^2+\\+\int_{t_0}^s \abs{\vf{b}(r,\vf{X}_r^{t_0,\vf{x}_0})-\vf{b}(r,\vf{X}_r^{t_0,\vf{y}_0})}^2\dd{r}+\\+\abs{\int_{t_0}^s \vf{\sigma}(r,\vf{X}_r^{t_0,\vf{x}_0})-\vf{\sigma}(r,\vf{X}_r^{t_0,\vf{y}_0})\dd{\vf{B}_r}}^2
+    \end{multline*}
+    In second term we use the Lipschitz-continuity. For the third term, once we take supremum and expectation, we can use \mnameref{SC:doob_maximal}. And finally, we will need to use \mnameref{SC:gronwall}.
+  \end{proof}
+  \begin{proposition}[Flow property]
+    Let ${(\vf{X}_t^{t_0,\vf{x}_0})}_{t\geq t_0}$ be the solution of \eqref{SCO:SDE}. For all $t_0\leq t_1\leq t_2$ we have: $$\vf{X}_{t_2}^{t_0,\vf{x}_0}=\vf{X}_{t_2}^{t_1,\vf{X}_{t_1}^{t_0,\vf{x}_0}}$$
+  \end{proposition}
+  \subsubsection{Feynman-Kac formula}
+  \begin{definition}
+    Let ${(\vf{X}_t^{t_0,\vf{x}_0})}_{t\geq t_0}$ be the solution of \eqref{SCO:SDE}, $T>0$ and $g:\RR^d\to\RR$ be continuous and bounded. We define $u(t_0,\vf{x}_0)=\Exp(g(\vf{X}_T^{t_0,\vf{x}_0}))$.
+  \end{definition}
+  \begin{proposition}
+    Let ${(\mathcal{F}_t)}_{t\geq 0}$ be the filtration generated by the Brownian motion. If $t_0\leq t_1\leq T$, then:
+    $$
+      u(t_0,\vf{x}_0)=\Exp\left(g\left(\vf{X}_T^{t_1,\vf{X}_{t_1}^{t_0,\vf{x}_0}}\right)\mid \mathcal{F}_{t_1}\right)
+    $$
+  \end{proposition}
+  \begin{theorem}[Dynamic programming principle]\label{SCO:dynamic_programming}
+    If $t_0\leq t_1\leq T$, we have that $u:[0,T]\times\RR^d\to\RR$ satisfies:
+    $$
+      u(t_0,\vf{x}_0)=\Exp\left(u(t_1,\vf{X}_{t_1}^{t_0,\vf{x}_0})\right)
+    $$
+  \end{theorem}
+  \begin{corollary}
+    Let $\tau\geq t_0$ be a stopping time. Then, $$u(t_0,\vf{x}_0)=\Exp\left(g(\vf{X}_\tau^{t_0,\vf{x}_0})\right)$$
+  \end{corollary}
+  \begin{theorem}\hfill
+    \begin{enumerate}
+      \item If $u\in\mathcal{C}^{1,2}$ and $\partial_t u,\grad u, \vf{D}^2u,\vf\sigma,\vf{b}$ are bounded, then $u$ solves the \emph{Fokker-Planck equation}:
+            $$
+              \begin{cases}
+                \partial_tu+\frac{1}{2}\tr(\vf{\sigma}\transpose{\vf\sigma}\vf{D}^2u)-\vf{b}\cdot\grad u=0 & \!\text{in }(0,T)\!\times\!\RR^d \\
+                u(T,\vf{x})=g(\vf{x})                                                                      & \!\text{in }\RR^d
+              \end{cases}
+            $$
+      \item If $v\in \mathcal{C}^{1,2}$ solves the Fokker-Planck equation and it is bounded, then $u=v$.
+    \end{enumerate}
+  \end{theorem}
+  \begin{proof}\hfill
+    \begin{enumerate}
+      \item Fix $(t,\vf{x})\in (0,T)\times\RR^d$. By the \mnameref{SCO:dynamic_programming} we have that $\forall h\in (0,T)$ we have:
+            \begin{multline*}
+              u(t,\vf{x})=\Exp\left(u(t+h,\vf{X}_{t+h}^{t,\vf{x}})\right)=\\=\Exp\Bigg(u(t,\vf{x})+\\\left.\int_t^{t+h}\left[\partial_t u + \vf{b}\cdot\grad u + \frac{1}{2}\tr(\vf{\sigma}\transpose{\vf\sigma}\vf{D}^2u)\right]\!(r,\vf{X}_r^{t,\vf{x}})\dd{r}\!\right)\!+\\+\Exp\left(\int_t^{t+h}\grad u(r,\vf{X}_r^{t,\vf{x}})\cdot\vf{\sigma}(r,\vf{X}_r^{t,\vf{x}})\dd{\vf{B}_r}\right)
+            \end{multline*}
+            where we have used \mnameref{SCO:ito_formula}. Note that the last term is zero because it is a martingale. Now dividing by $h$ we have:
+            $$
+              \Exp\left(\frac{1}{h}\int_t^{t+h}\partial_t u + \vf{b}\cdot\grad u + \frac{1}{2}\tr(\vf{\sigma}\transpose{\vf\sigma}\vf{D}^2u)  \dd{r}\right)=0
+            $$
+            And now use \mnameref{P:dominated}.
+      \item Assume $v$ solves the Fokker-Planck equation. We have:
+            \begin{equation*}
+              g(\vf{X}_T^{t,\vf{x}})=v(T,\vf{X}_T^{t,\vf{x}})=v(t,\vf{X}_t^{t,\vf{x}})+\int_t^T(\transpose{\vf\sigma} \grad v) \dd{\vf{B}_r}
+            \end{equation*}
+            the second term is a local a bounded (by hypothesis) local martingale. Hence, it is a martingale, and taking expectations we get the result.
+    \end{enumerate}
+  \end{proof}
+  \begin{remark}
+    Note that the Fokker-Planck equation becomes the backward in time heat equation if $\vf{b}=0$ and $\vf{\sigma}=\sqrt{2}\vf{I}$.
+  \end{remark}
+  \subsection{Dynamic programming}
+  \subsubsection{Optimal control problem}
+  \begin{definition}
+    Consider the following SDE:
+    \begin{equation}\label{SCO:SDE_control}
+      \begin{cases}
+        \dd{\vf{X}_t}=\vf{b}_t(\vf{X}_t,\alpha_t)\dd{t}+\vf{\sigma}_t(\vf{X}_t,\alpha_t)\dd{\vf{B}_t} \\
+        \vf{X}_{t_0}=\vf{x}_0
+      \end{cases}
+    \end{equation}
+    where $\vf{b}:\RR_{\geq 0}\times\RR^d\times A\to\RR^d$, $\vf{\sigma}:\RR_{\geq 0}\times\RR^d\times A\to\RR^{d\times m}$ are continuous, $A$ is a compact metric space and $\alpha_t\in \text{ct}_t:=\{\rho:[0,t]\times\Omega\to A:\rho\text{ is progressively measurable}\}$ is a \emph{control parameter}.
+  \end{definition}
+  From here on we will assume that both $\vf{b}$ and $\vf{\sigma}$ are uniformly Lipschitz-continuous in the second variable.
+  \begin{theorem}
+    For all $\alpha\in \text{ct}_{t_0}$ and all $(t_0,\vf{x}_0)\in [0,T]\times\RR^d$ there exists a unique solution ${(\vf{X}_t^{t_0,\vf{x}_0,\alpha})}_{t\geq t_0}$ of \mcref{SCO:SDE_control}.
+  \end{theorem}
+  \begin{lemma}
+    Let ${(\vf{X}_t^{t_0,\vf{x}_0,\alpha})}_{t\geq t_0}$ be the solution of \mcref{SCO:SDE_control}. Then, for all $t_2>t_1\geq t_0$ and all $\alpha\in \text{ct}_{t_0}$ we have:
+    $$
+      \vf{X}_{t_2}^{t_0,\vf{x}_0,\alpha}=\vf{X}_{t_2}^{t_1,\vf{X}_{t_1}^{t_0,\vf{x}_0,\alpha},\alpha}
+    $$
+  \end{lemma}
+  \begin{definition}[Finite horizon problem]
+    Let $T>0$, $g:\RR^d\to\RR$ be continuous and bounded and $\ell:[0,T]\times\RR^d\times A\to\RR$ be continuous and bounded. We define the following problem:
+    $$
+      \inf_{\alpha\in \text{ct}_{t_0}}\Exp\left(\int_{t_0}^T \ell(r,\vf{X}_r^{t_0,\vf{x}_0,\alpha},\alpha_r)\dd{r}+g(\vf{X}_T^{t_0,\vf{x}_0,\alpha})\right)
+    $$
+    The first term in the expectation is called \emph{running cost} and the second one \emph{terminal cost}.
+  \end{definition}
+  \begin{definition}[Infinte horizon problem]
+    Let $g:\RR^d\to\RR$ be continuous and bounded and $\ell:[0,\infty)\times\RR^d\times A\to\RR$ be continuous and bounded. We define the following problem:
+    $$
+      \inf_{\alpha\in \text{ct}_{t_0}}\Exp\left(\int_{t_0}^\infty \exp{-\tau r}\ell(r,\vf{X}_r^{t_0,\vf{x}_0,\alpha},\alpha_r)\dd{r}\right)
+    $$
+  \end{definition}
+\end{multicols}
+\end{document}