Matrix-Cookbook.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "import matplotlib.pyplot as plt\n",
      "import numpy as np"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### This an implementation of the useful matrix cookbook techniques and formulas for linear algebra from Petersen & Pedersen"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Notation and Nomeclature\n",
      "<hr>\n",
      "\n",
      "$\\mathbf{A}\\rightarrow\\;$ Matrix\n",
      "\n",
      "$\\mathbf{A}_{ij}\\rightarrow\\;$ Matrix indexed for some purpose\n",
      "\n",
      "$\\mathbf{A}_{i}\\rightarrow\\;$ Matrix indexed for some purpose\n",
      "\n",
      "$\\mathbf{A}^{ij}\\rightarrow\\;$ Matrix indexed for some purpose\n",
      "\n",
      "$\\mathbf{A}^{n}\\rightarrow\\;$ Matrix indexed for some purpose or the the $n^{th}$ power of a square matrix\n",
      "\n",
      "$\\mathbf{A}^{-1}\\rightarrow\\;$ The inverse matrix of matrix $\\mathbf{A}$\n",
      "\n",
      "$\\mathbf{A}^{\\dagger}\\rightarrow\\;$ The pseudo inverse matrix of the matrix $\\mathbf{A}$\n",
      "\n",
      "$\\mathbf{A}^{1/2}\\rightarrow\\;$ The square root of a matrix (if unique), not elementwise\n",
      "\n",
      "$(\\mathbf{A})_{i,j}\\rightarrow\\;$ The $(i,j)^{th}$ entry of the matrix $\\mathbf{A}$\n",
      "\n",
      "$\\mathbf{A}_{i,j}\\rightarrow\\;$ The $(i,j)^{th}$ entry of the matrix $\\mathbf{A}$\n",
      "\n",
      "$[\\mathbf{A}]_{i,j}\\rightarrow\\;$ The $ij$-suubmatrix, i.e. $\\mathbf{A}$ with $i^{th}$ row and $j^{th}$ column deleted\n",
      "\n",
      "$\\vec{a}\\rightarrow\\;$ Vector (column-vector)\n",
      "\n",
      "$\\vec{a}_{i}\\rightarrow\\;$ Vector indexed for some purpose\n",
      "\n",
      "$\\alpha_{i}\\rightarrow\\;$ The $i^{th}$ element of vector $\\vec{a}$\n",
      "\n",
      "$\\alpha\\rightarrow\\;$  Scalar\n",
      "\n",
      "<hr>\n",
      "\n",
      "$\\Re z\\rightarrow\\;$ Real part of a scalar\n",
      "\n",
      "$\\Re \\mathbf{z}\\rightarrow\\;$ Real part of a vectorS\n",
      "\n",
      "$\\Re \\mathbf{Z}\\rightarrow\\;$ Real part of a matrix\n",
      "\n",
      "$\\Im z\\rightarrow\\;$ Imaginary part of a scalar\n",
      "\n",
      "$\\Im \\mathbf{z}\\rightarrow\\;$ Imaginary part of a vectorS\n",
      "\n",
      "$\\Im \\mathbf{Z}\\rightarrow\\;$ Imaginary part of a matrix\n",
      "\n",
      "<hr>\n",
      "\n",
      "$det(\\mathbf{A})\\rightarrow\\;$ Determinant of $\\mathbf{A}$\n",
      "\n",
      "\n",
      "$Tr(\\mathbf{A})\\rightarrow\\;$ Trace of the matrix $\\mathbf{A}$\n",
      "\n",
      "$diag(\\mathbf{A})\\rightarrow\\;$ Diagonal matrix of the matrix $\\mathbf{A}$, i.e. $(diag(\\mathbf{A}))_{ij} = \\delta_{ij}\\mathbf{A}_{ij}$\n",
      "\n",
      "$eig(\\mathbf{A})\\rightarrow\\;$ Eigenvalues of the matrix $\\mathbf{A}$\n",
      "\n",
      "$vec(\\mathbf{A})\\rightarrow\\;$ The vector-version of the matrix $\\mathbf{A}$\n",
      "\n",
      "$sup\\rightarrow\\;$ Supremum of a set\n",
      "\n",
      "$\\Vert\\mathbf{A}\\Vert\\rightarrow\\;$ Matrix norm (subscript if any denotes what norm)\n",
      "\n",
      "$\\mathbf{A}^{T}\\rightarrow\\;$ Transposed matrix\n",
      "\n",
      "$\\mathbf{A}^{-T}\\rightarrow\\;$ The inverse of the transposed and vice versa, $\\mathbf{A}^{-T} = (\\mathbf{A}^{-1})^{T} = (\\mathbf{A}^{T})^{-1}$\n",
      "\n",
      "$\\mathbf{A}^{\\star}\\rightarrow\\;$ Complex conjugated matrix\n",
      "\n",
      "$\\mathbf{A}^{H}\\rightarrow\\;$ Transposed and complex conjugated matrix (Hermitian)\n",
      "\n",
      "<hr>\n",
      "\n",
      "$\\mathbf{A}\\circ\\mathbf{B}$ Hadamard (elementwise) product\n",
      "\n",
      "$\\mathbf{A}\\otimes\\mathbf{B}$ Kronecker product\n",
      "\n",
      "<hr>\n",
      "\n",
      "$\\mathbf{0}\\rightarrow\\;$ The null matrix. Zero in all entries.\n",
      "\n",
      "$\\mathbf{I}\\rightarrow\\;$ The idnentity matrix\n",
      "\n",
      "$\\mathbf{J}^{ij}\\rightarrow\\;$ The single entry matrix, 1 at (i,j) and zero elsewhere\n",
      "\n",
      "$\\mathbf{\\Sigma}\\rightarrow\\;$ A posisitive definite matrix\n",
      "\n",
      "$\\mathbf{\\Lambda}\\rightarrow\\;$ A diagonal matrix\n",
      "\n",
      "<hr>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Basics\n",
      "<hr>\n",
      "\n",
      "\\begin{align}\n",
      "\\label{eqn:1}\n",
      "(\\mathbf{A}\\mathbf{B})^{-1} &= \\mathbf{B}^{-1}\\mathbf{A}^{-1} \\\\\n",
      "\\label{eqn:2}\n",
      "(\\mathbf{ABC\\ldots})^{-1} &= \\ldots\\mathbf{C}^{-1}\\mathbf{B}^{-1}\\mathbf{A}^{-1} \\\\\n",
      "\\label{eqn:3}\n",
      "(\\mathbf{A}^{T})^{-1} &= (\\mathbf{A}^{-1})^{T} \\\\\n",
      "\\label{eqn:4}\n",
      "(\\mathbf{A} + \\mathbf{B})^{T} &= \\mathbf{A}^{T} + \\mathbf{B}^{T} \\\\\n",
      "\\label{eqn:5}\n",
      "(\\mathbf{AB})^{T} &= \\mathbf{B}^{T}\\mathbf{A}^{T} \\\\\n",
      "\\label{eqn:6}\n",
      "(\\mathbf{ABC\\ldots})^{T} &= \\ldots\\mathbf{C}^{T}\\mathbf{B}^{T}\\mathbf{A}^{T} \\\\\n",
      "\\label{eqn:7}\n",
      "(\\mathbf{A}^{H})^{-1} &= (\\mathbf{A}^{-1})^{H} \\\\\n",
      "\\label{eqn:8}\n",
      "(\\mathbf{A} + \\mathbf{B})^{H} &= \\mathbf{B}^{H} + \\mathbf{A}^{H} \\\\\n",
      "\\label{eqn:9}\n",
      "(\\mathbf{AB})^{H} &= \\mathbf{B}^{H}\\mathbf{A}^{H} \\\\\n",
      "\\label{eqn:10}\n",
      "(\\mathbf{ABC\\ldots})^{H} &= \\ldots\\mathbf{C}^{H}\\mathbf{B}^{H}\\mathbf{A}^{H} \\\\\n",
      "\\end{align}"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# lets play with the above equations and equatities to see for ouselves\n",
      "# caution the above probably work only for square matrices\n",
      "width = 3\n",
      "height = 3\n",
      "A = np.random.random([width, height])\n",
      "B = np.random.random([width, height])\n",
      "C = np.random.random([width, height])\n",
      "\n",
      "\n",
      "# validate if results are equal of right and left hand side operations\n",
      "def validate(left, right, eqn):\n",
      "    print(\"results equation {}: {} \\n\".format(eqn, left.all() == right.all()))\n",
      "\n",
      "# equation 1:\n",
      "lft_hand = np.linalg.inv(A.dot(B))\n",
      "rgt_hand = np.dot(np.linalg.inv(B), np.linalg.inv(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "#check if results are identical\n",
      "validate(lft_hand, rgt_hand, 1)\n",
      "\n",
      "# equation 2:\n",
      "lft_hand = np.linalg.inv(A.dot(B).dot(C))\n",
      "rgt_hand = np.dot(np.linalg.inv(C), np.linalg.inv(B)).dot(np.linalg.inv(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 2)\n",
      "\n",
      "# equation 3:\n",
      "lft_hand = np.linalg.inv(A.T)\n",
      "rgt_hand = np.transpose(np.linalg.inv(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 3)\n",
      "\n",
      "# equation 4:\n",
      "lft_hand = (A+B).T\n",
      "rgt_hand = A.T+B.T\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 4)\n",
      "\n",
      "# equation 5:\n",
      "lft_hand = A.dot(B).T\n",
      "rgt_hand = B.T.dot(A.T)\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 5)\n",
      "\n",
      "# equation 7:\n",
      "lft_hand = np.linalg.inv(np.conjugate(A))\n",
      "rgt_hand = np.conjugate(np.linalg.inv(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 6)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "lft_hand = \n",
        " [[ 6.36883088 -1.60305506 -3.77668768]\n",
        " [-1.4344897  -3.47490041  3.37676019]\n",
        " [-2.5703541   5.13803128 -0.61004824]] \n",
        "\n",
        "rgt_hand = \n",
        " [[ 6.36883088 -1.60305506 -3.77668768]\n",
        " [-1.4344897  -3.47490041  3.37676019]\n",
        " [-2.5703541   5.13803128 -0.61004824]] \n",
        "\n",
        "results equation 1: True \n",
        "\n",
        "lft_hand = \n",
        " [[-14.1040691   -7.8483558   15.62389381]\n",
        " [  9.45606574  18.3975466  -17.77253408]\n",
        " [  4.52054108  -6.6687573    0.54822755]] \n",
        "\n",
        "rgt_hand = \n",
        " [[-14.1040691   -7.8483558   15.62389381]\n",
        " [  9.45606574  18.3975466  -17.77253408]\n",
        " [  4.52054108  -6.6687573    0.54822755]] \n",
        "\n",
        "results equation 2: True \n",
        "\n",
        "lft_hand = \n",
        " [[ 0.08000516  1.80811015 -0.35833643]\n",
        " [-1.16780641  0.88302211  1.37406956]\n",
        " [ 1.0612411  -1.45172247  0.16723357]] \n",
        "\n",
        "rgt_hand = \n",
        " [[ 0.08000516  1.80811015 -0.35833643]\n",
        " [-1.16780641  0.88302211  1.37406956]\n",
        " [ 1.0612411  -1.45172247  0.16723357]] \n",
        "\n",
        "results equation 3: True \n",
        "\n",
        "lft_hand = \n",
        " [[ 1.10746306  0.68734863  1.48630864]\n",
        " [ 1.37332026  0.5008557   0.91009057]\n",
        " [ 0.69108045  1.31366625  1.72722853]] \n",
        "\n",
        "rgt_hand = \n",
        " [[ 1.10746306  0.68734863  1.48630864]\n",
        " [ 1.37332026  0.5008557   0.91009057]\n",
        " [ 0.69108045  1.31366625  1.72722853]] \n",
        "\n",
        "results equation 4: True \n",
        "\n",
        "lft_hand = \n",
        " [[ 0.75723063  0.47504914  0.81053668]\n",
        " [ 1.01341738  0.67582365  1.42211955]\n",
        " [ 0.92163839  0.79990751  1.214679  ]] \n",
        "\n",
        "rgt_hand = \n",
        " [[ 0.75723063  0.47504914  0.81053668]\n",
        " [ 1.01341738  0.67582365  1.42211955]\n",
        " [ 0.92163839  0.79990751  1.214679  ]] \n",
        "\n",
        "results equation 5: True \n",
        "\n",
        "lft_hand = \n",
        " [[ 0.08000516 -1.16780641  1.0612411 ]\n",
        " [ 1.80811015  0.88302211 -1.45172247]\n",
        " [-0.35833643  1.37406956  0.16723357]] \n",
        "\n",
        "rgt_hand = \n",
        " [[ 0.08000516 -1.16780641  1.0612411 ]\n",
        " [ 1.80811015  0.88302211 -1.45172247]\n",
        " [-0.35833643  1.37406956  0.16723357]] \n",
        "\n",
        "results equation 6: True \n",
        "\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "According to equation $\\ref{eqn:1}$ you get this and according to equation $\\ref{eqn:2}$ you get this one"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### Trace\n",
      "<hr>\n",
      "\n",
      "\\begin{align}\n",
      "Tr(\\mathbf{A}) &= \\sum_{i} A_{ii} \\\\\n",
      "Tr(\\mathbf{A}) &= \\sum_{i}\\lambda_{i}, \\;\\;\\lambda_{i} = eig(\\mathbf{A}) \\\\\n",
      "Tr(\\mathbf{A}) &= Tr(\\mathbf{A}^{T}) \\\\\n",
      "Tr(\\mathbf{AB}) &= Tr(\\mathbf{BA}) \\\\\n",
      "Tr(\\mathbf{A}+\\mathbf{B}) &= Tr(\\mathbf{A}) + Tr(\\mathbf{B}) \\\\\n",
      "Tr(\\mathbf{ABC}) &= Tr(\\mathbf{BCA}) = Tr(\\mathbf{CAB}) \\\\\n",
      "\\vec{a}^{T}\\vec{a} &= Tr(\\vec{a}\\vec{a}^{T})\n",
      "\\end{align}"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# equation 11:\n",
      "lft_hand = np.trace(A)\n",
      "rgt_hand = np.sum(np.diag(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 11)\n",
      "\n",
      "# equation 12:\n",
      "lft_hand = np.trace(A)\n",
      "rgt_hand = np.sum(np.linalg.eigvals(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 12)\n",
      "\n",
      "# equation 13:\n",
      "lft_hand = np.trace(A)\n",
      "rgt_hand = np.trace(A.T)\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 13)\n",
      "\n",
      "# equation 14:\n",
      "lft_hand = np.trace(A.dot(B))\n",
      "rgt_hand = np.trace(B.dot(A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 14)\n",
      "\n",
      "# equation 15:\n",
      "lft_hand = np.trace(A+B)\n",
      "rgt_hand = np.trace(A) + np.trace(B)\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 15)\n",
      "\n",
      "# equation 16:\n",
      "lft_hand = np.trace(np.dot(A.dot(B), C))\n",
      "rgt_hand = np.trace(np.dot(B.dot(C), A))\n",
      "print(\"lft_hand = \\n {} \\n\".format(lft_hand))\n",
      "print(\"rgt_hand = \\n {} \\n\".format(rgt_hand))\n",
      "# check results\n",
      "validate(lft_hand, rgt_hand, 16)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}