{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "#!/usr/bin/env python\n", "\n", "SAVE_PARAMS_EVERY = 5000\n", "\n", "import glob\n", "import random\n", "import numpy as np\n", "import os.path as op\n", "import pickle\n", "\n", "\n", "def load_saved_params():\n", " \"\"\"\n", " A helper function that loads previously saved parameters and resets\n", " iter_ation start.\n", " \"\"\"\n", " st = 0\n", " for f in glob.glob(\"saved_params_*.npy\"):\n", " iter_ = int(op.splitext(op.basename(f))[0].split(\"_\")[2])\n", " if (iter_ > st):\n", " st = iter_\n", "\n", " if st > 0:\n", " print(st)\n", " with open(\"saved_params_%d.npy\" % st, \"rb\") as f:\n", " params = pickle.load(f)\n", " state = pickle.load(f)\n", " return st, params, state\n", " else:\n", " return st, None, None\n", "\n", "\n", "def save_params(iter_, params):\n", " with open(\"saved_params_%d.npy\" % iter_, \"wb\") as f:\n", " pickle.dump(params, f)\n", " pickle.dump(random.getstate(), f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# SGD\n", "mini-batch已经写好了，学习率递减也写好了，直接算梯度乘以学习率即可" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Running sanity checks...\n", "iter_ 100: 0.004578\n", "iter_ 200: 0.004353\n", "iter_ 300: 0.004136\n", "iter_ 400: 0.003929\n", "iter_ 500: 0.003733\n", "iter_ 600: 0.003546\n", "iter_ 700: 0.003369\n", "iter_ 800: 0.003200\n", "iter_ 900: 0.003040\n", "iter_ 1000: 0.002888\n", "test 1 result: 8.414836786079764e-10\n", "iter_ 100: 0.000000\n", "iter_ 200: 0.000000\n", "iter_ 300: 0.000000\n", "iter_ 400: 0.000000\n", "iter_ 500: 0.000000\n", "iter_ 600: 0.000000\n", "iter_ 700: 0.000000\n", "iter_ 800: 0.000000\n", "iter_ 900: 0.000000\n", "iter_ 1000: 0.000000\n", "test 2 result: 0.0\n", "iter_ 100: 0.041205\n", "iter_ 200: 0.039181\n", "iter_ 300: 0.037222\n", "iter_ 400: 0.035361\n", "iter_ 500: 0.033593\n", "iter_ 600: 0.031913\n", "iter_ 700: 0.030318\n", "iter_ 800: 0.028802\n", "iter_ 900: 0.027362\n", "iter_ 1000: 0.025994\n", "test 3 result: -2.524451035823933e-09\n", "\n" ] } ], "source": [ "def sgd(f, x0, step, iterations, postprocessing=None, useSaved=False,\n", " PRINT_EVERY=10):\n", " \"\"\" Stochastic Gradient Descent\n", "\n", " Implement the stochastic gradient descent method in this function.\n", "\n", " Arguments:\n", " f -- the function to optimize, it should take a single\n", " argument and yield two outputs, a cost and the gradient\n", " with respect to the arguments\n", " x0 -- the initial point to start SGD from\n", " step -- the step size for SGD\n", " iter_ations -- total iter_ations to run SGD for\n", " postprocessing -- postprocessing function for the parameters\n", " if necessary. In the case of word2vec we will need to\n", " normalize the word vectors to have unit length.\n", " PRINT_EVERY -- specifies how many iter_ations to output loss\n", "\n", " Return:\n", " x -- the parameter value after SGD finishes\n", " \"\"\"\n", "\n", " # Anneal learning rate every several iter_ations\n", " ANNEAL_EVERY = 20000\n", "\n", " if useSaved:\n", " start_iter_, oldx, state = load_saved_params()\n", " if start_iter_ > 0:\n", " x0 = oldx\n", " step *= 0.5 ** (start_iter_ / ANNEAL_EVERY)\n", "\n", " if state:\n", " random.setstate(state)\n", " else:\n", " start_iter_ = 0\n", "\n", " x = x0\n", "\n", " if not postprocessing:\n", " postprocessing = lambda x: x\n", "\n", " expcost = None\n", "\n", " for iter_ in range(start_iter_ + 1, iterations + 1):\n", " # Don't forget to apply the postprocessing after every iteration!\n", " # You might want to print the progress every few iterations.\n", "\n", " cost = None\n", " \n", " ### YOUR CODE HERE\n", " # mini-batch已经写好了，学习率递减也写好了，直接算梯度乘以学习率即可\n", " cost, grad = f(x)\n", " x -= step * grad\n", " x = postprocessing(x)\n", " ### END YOUR CODE\n", "\n", " if iter_ % PRINT_EVERY == 0:\n", " if not expcost:\n", " expcost = cost\n", " else:\n", " expcost = .95 * expcost + .05 * cost\n", " \n", " print(\"iter_ %d: %f\" % (iter_, expcost))\n", " try:\n", " if iter_ % SAVE_PARAMS_EVERY == 0 and useSaved:\n", " save_params(iter_, x)\n", " print(\"saved!\")\n", "\n", " if iter_ % ANNEAL_EVERY == 0:\n", " step *= 0.5\n", " except Exception as e:\n", " print(str(e))\n", " return x\n", "\n", "\n", "def sanity_check():\n", " quad = lambda x: (np.sum(x ** 2), x * 2)\n", "\n", " print(\"Running sanity checks...\")\n", " t1 = sgd(quad, 0.5, 0.01, 1000, PRINT_EVERY=100)\n", " print(\"test 1 result:\", t1)\n", " assert abs(t1) <= 1e-6\n", "\n", " t2 = sgd(quad, 0.0, 0.01, 1000, PRINT_EVERY=100)\n", " print(\"test 2 result:\", t2)\n", " assert abs(t2) <= 1e-6\n", "\n", " t3 = sgd(quad, -1.5, 0.01, 1000, PRINT_EVERY=100)\n", " print(\"test 3 result:\", t3)\n", " assert abs(t3) <= 1e-6\n", "\n", " print(\"\")\n", "\n", "\n", "\n", "if __name__ == \"__main__\":\n", " sanity_check()\n", " # your_sanity_checks()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }