{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "77c1109a-ca7c-4005-908e-e89b22de92ff",
   "metadata": {},
   "source": [
    "# Exam\n",
    "$$ \\frac{\\partial E}{\\partial w_{jk}}= -e_j\\cdot \\sigma\\left(\\sum_i w_{ij} o_i\\right) \\left(1-\\sigma\\left(\\sum_i w_{ij} o_i\\right) \\right) o_i $$\n",
    "\n",
    "$$w_{New} = w_{old}-\\alpha \\frac{\\partial E}{\\partial w} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9878b172-ce84-4d7d-822b-a2c0bda6b7cf",
   "metadata": {},
   "source": [
    "## Doing step by step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bafc3aff-e2eb-43cf-9d8e-6a7e252be593",
   "metadata": {},
   "outputs": [],
   "source": [
    "def af(x):\n",
    "    sigmoid = 1/(1+np.exp(-x))\n",
    "    return sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a83cf35-172a-436c-8e68-ed6ae9e2eeba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wih:  [[ 0.34666241  0.06116554 -0.0451246 ]\n",
      " [-0.14782509  0.08585138  0.03574974]\n",
      " [ 0.32745628 -0.2354578  -0.02162094]\n",
      " [-0.15221498 -0.36552168 -0.24002265]]\n",
      "Who:  [[-0.45236532 -0.1057067  -0.12838381  0.05673292]\n",
      " [ 0.39749455 -0.33265411 -0.09279358  0.15235334]\n",
      " [ 0.06774908  0.06651886  0.0243551   0.10758002]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "inN = 3\n",
    "hiN= 4\n",
    "outN=3\n",
    "lr= 0.4\n",
    "        \n",
    "#weight W11 W21 W31\n",
    "#   W12 W22 W32\n",
    "# .....\n",
    "np.random.seed(53)    \n",
    "wih=np.random.rand(hiN, inN)-0.5\n",
    "who=np.random.rand(outN, hiN)-0.5\n",
    "print(\"Wih: \", wih)\n",
    "print(\"Who: \", who)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c2a8d42-b5e2-40c4-9220-50e8cc42ed7e",
   "metadata": {},
   "source": [
    "## Feedforward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c3e66fc9-a311-4d30-b312-08e6519f4069",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Xh:  [[ 0.11932424]\n",
      " [-0.0176892 ]\n",
      " [ 0.03732063]\n",
      " [-0.19060372]]\n",
      "Oh: [[0.52979571]\n",
      " [0.49557782]\n",
      " [0.50932908]\n",
      " [0.45249281]]\n",
      "Xo:  [[-0.33176547]\n",
      " [ 0.06741123]\n",
      " [ 0.12994239]]\n",
      "Oo:  [[0.41781112]\n",
      " [0.51684643]\n",
      " [0.53243996]]\n"
     ]
    }
   ],
   "source": [
    "inputList = [0.32, 0.27, 0.18]\n",
    "inputs = np.array(inputList, ndmin=2).T\n",
    "Xh = np.dot(wih, inputs)\n",
    "print('Xh: ', Xh)\n",
    "\n",
    "Oh = af(Xh)\n",
    "print('Oh:', Oh)\n",
    "\n",
    "# computing output \n",
    "Xo = np.dot(who, Oh)\n",
    "print('Xo: ', Xo)\n",
    "Oo = af(Xo)\n",
    "print('Oo: ', Oo)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87b902d1-0046-425f-8ade-def73a605374",
   "metadata": {},
   "source": [
    "## Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9703ca9c-9a8d-4f4a-bc89-f3984620ef1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputList = [0.32, 0.27, 0.18]\n",
    "targetList = [0.82, 0.25, 0.44]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b335a7d5-33bf-4af0-930b-794c77bdc116",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = np.array(inputList, ndmin=2).T\n",
    "target = np.array(targetList, ndmin=2).T\n",
    "        \n",
    "#computting hidden layer\n",
    "Xh = np.dot(wih, inputs)\n",
    "Oh = af(Xh)\n",
    "    \n",
    "# computing output \n",
    "Xo = np.dot(who, Oh)\n",
    "Oo = af(Xo)\n",
    "        \n",
    "# Output error\n",
    "oe = target-Oo\n",
    "# E propagation\n",
    "hiddenE = np.dot(who.T, oe)\n",
    "        \n",
    "# updating weights\n",
    "#who+=lr*np.dot(oe*Oo*(1-Oo), Oh.T) \n",
    "#wih+=lr*np.dot(hiddenE*Oh*(1-Oh), inputs.T) \n",
    "#print('New wih: ', wih)\n",
    "#print('New who: ', who)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0164f74a-ee41-4cf4-92d3-72dfdc85d3bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.43163327, -0.08631366, -0.10845266,  0.07443995],\n",
       "       [ 0.38337319, -0.34586342, -0.10636942,  0.14029244],\n",
       "       [ 0.06287227,  0.06195702,  0.01966668,  0.10341479]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NewW=who-lr*np.dot(-oe*Oo*(1-Oo),Oh.T)\n",
    "NewW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "becd14fd-c1bf-400e-995d-c3ffee8167f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.43163327, -0.08631366, -0.10845266,  0.07443995],\n",
       "       [ 0.38337319, -0.34586342, -0.10636942,  0.14029244],\n",
       "       [ 0.06287227,  0.06195702,  0.01966668,  0.10341479]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newWho=who-lr*np.dot(-oe*Oo*(1-Oo), Oh.T)\n",
    "newWho"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03cba5b5-7d31-464a-b56f-04004efe67b2",
   "metadata": {},
   "source": [
    "## Using class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "de8798ec-ddf2-40d2-bdd3-ed9d82059829",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class NeuralNetwork:\n",
    "    # init method\n",
    "    def __init__(self, inputN,hiddenN, outputN, lr):\n",
    "        # creates a NN with three layers (input, hidden, output)\n",
    "        # inputN - Number of input nodes\n",
    "        # hiddenN - Number of hidden nodes\n",
    "        self.inN=inputN\n",
    "        self.hiN=hiddenN\n",
    "        self.outN=outputN\n",
    "        self.lr=lr\n",
    "        \n",
    "        #weight W11 W21 W31\n",
    "            #   W12 W22 W32\n",
    "            # .....\n",
    "        np.random.seed(53)    \n",
    "        self.wih=np.random.rand(self.hiN, self.inN)-0.5\n",
    "        self.who=np.random.rand(self.outN,self.hiN)-0.5\n",
    "        print(\"Wih: \", self.wih)\n",
    "        print(\"Who: \", self.who)\n",
    "        pass\n",
    "    \n",
    "    # NN computing method\n",
    "    def feedforward(self, inputList):\n",
    "        # computing hidden output\n",
    "        inputs = np.array(inputList, ndmin=2).T\n",
    "        self.Xh = np.dot(self.wih, inputs)\n",
    "        print('Xh: ', self.Xh)\n",
    "        self.af = lambda x:1/(1+np.exp(-x))\n",
    "        self.Oh = self.af(self.Xh)\n",
    "        print('Oh:', self.Oh)\n",
    "        \n",
    "        # computing output \n",
    "        self.Xo = np.dot(self.who, self.Oh)\n",
    "        print('Xo: ', self.Xo)\n",
    "        self.Oo = self.af(self.Xo)\n",
    "        print('Oo: ', self.Oo)\n",
    "        pass\n",
    "    \n",
    "    # NN trainning method    \n",
    "    def backpropagation(self, inputList, targetList):\n",
    "        # data\n",
    "        lr = self.lr \n",
    "        inputs = np.array(inputList, ndmin=2).T\n",
    "        target = np.array(targetList, ndmin=2).T\n",
    "        \n",
    "        #computting hidden layer\n",
    "        Xh = np.dot(self.wih, inputs)\n",
    "        af = lambda x:1/(1+np.exp(-x))\n",
    "        Oh = af(Xh)\n",
    "        \n",
    "        # computing output \n",
    "        Xo = np.dot(self.who, Oh)\n",
    "        Oo = af(Xo)\n",
    "        \n",
    "        # Output error\n",
    "        oe = target-Oo\n",
    "        # E propagation\n",
    "        hiddenE = np.dot(self.who.T, oe)\n",
    "        \n",
    "        # updating weights\n",
    "        self.who+=lr*np.dot(oe*Oo*(1-Oo), Oh.T) \n",
    "        self.wih+=lr*np.dot(hiddenE*Oh*(1-Oh), inputs.T) \n",
    "        return self.wih, self.who"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e38c98a0-2edc-4733-a2e0-d911f44236a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wih:  [[ 0.34666241  0.06116554 -0.0451246 ]\n",
      " [-0.14782509  0.08585138  0.03574974]\n",
      " [ 0.32745628 -0.2354578  -0.02162094]\n",
      " [-0.15221498 -0.36552168 -0.24002265]]\n",
      "Who:  [[-0.45236532 -0.1057067  -0.12838381  0.05673292]\n",
      " [ 0.39749455 -0.33265411 -0.09279358  0.15235334]\n",
      " [ 0.06774908  0.06651886  0.0243551   0.10758002]]\n"
     ]
    }
   ],
   "source": [
    "NN = NeuralNetwork(3,4,3,0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "87a9faaa-d8f8-4ea7-b098-d3e8cf06fea8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Xh:  [[ 0.11932424]\n",
      " [-0.0176892 ]\n",
      " [ 0.03732063]\n",
      " [-0.19060372]]\n",
      "Oh: [[0.52979571]\n",
      " [0.49557782]\n",
      " [0.50932908]\n",
      " [0.45249281]]\n",
      "Xo:  [[-0.33176547]\n",
      " [ 0.06741123]\n",
      " [ 0.12994239]]\n",
      "Oo:  [[0.41781112]\n",
      " [0.51684643]\n",
      " [0.53243996]]\n"
     ]
    }
   ],
   "source": [
    "NN.feedforward([0.32, 0.27, 0.18])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6ef426d3-3f6d-448d-9fe0-41c044a30fed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.33727924,  0.05324849, -0.05040263],\n",
       "        [-0.14654184,  0.08693412,  0.03647157],\n",
       "        [ 0.32652462, -0.23624388, -0.02214499],\n",
       "        [-0.15309598, -0.36626503, -0.24051822]]),\n",
       " array([[-0.43163327, -0.08631366, -0.10845266,  0.07443995],\n",
       "        [ 0.38337319, -0.34586342, -0.10636942,  0.14029244],\n",
       "        [ 0.06287227,  0.06195702,  0.01966668,  0.10341479]]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NN.backpropagation([0.32, 0.27, 0.18], [0.82, 0.25, 0.44])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab4fc050-df3d-470c-aede-629357e44df5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}