sample_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "33e06976",
   "metadata": {},
   "source": [
    "# Sample Notebook\n",
    "\n",
    "This notebook implements scikit-learn's [Clustering text documents using k-means](https://scikit-learn.org/stable/auto_examples/text/plot_document_clustering.html) tutorial in order to demonstrate program slicing with Python. The tutorial was written by Peter Prettenhofer and shared under a BSD 3 clause license. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44380d98",
   "metadata": {},
   "source": [
    "## Execution Log ##\n",
    "**Important** This code cell needs to be run before any of the others, else nothing will work! Thanks to Stack Overflow user Andrei Iatsuk [this solution](https://stackoverflow.com/a/60658965)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52b31177",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.magic import register_cell_magic\n",
    "\n",
    "@register_cell_magic\n",
    "def write_and_run(line, cell):\n",
    "    \"\"\"Write the contents of a cell to a file, but only if the execution succeeds without error. \n",
    "    \n",
    "    -a is an optional parameter, which opens the file in append mode.   \n",
    "    \n",
    "    e.g., %%write_and_run -a execution_log.py\n",
    "    \n",
    "    \"\"\"\n",
    "    argz = line.split()\n",
    "    file = argz[-1]\n",
    "    mode = 'w'\n",
    "    if len(argz) == 2 and argz[0] == '-a':\n",
    "        mode = 'a'\n",
    "        \n",
    "    result = get_ipython().run_cell(cell)\n",
    "    \n",
    "    if result.error_in_exec is None:\n",
    "        with open(file, mode) as f:\n",
    "            f.write(cell)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab68c281",
   "metadata": {},
   "source": [
    "Each time a cell is executed, the source code it contains is written to ```execution_log.py``` for analysis. Execute the first cell below to start over with an empty execution log. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72b5a260",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%write_and_run execution_log.py\n",
    "###########\n",
    "# File execution log\n",
    "###########"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7e4a656f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6b516cfb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 20 newsgroups dataset for categories:\n",
      "['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "# Load some categories from the training set\n",
    "categories = [\n",
    "    'alt.atheism',\n",
    "    'talk.religion.misc',\n",
    "    'comp.graphics',\n",
    "    'sci.space',\n",
    "]\n",
    "\n",
    "print(\"Loading 20 newsgroups dataset for categories:\")\n",
    "print(categories)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "359caedf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3387 documents\n",
      "4 categories\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "dataset = fetch_20newsgroups(subset='all', categories=categories,\n",
    "                             shuffle=True, random_state=42)\n",
    "\n",
    "print(\"%d documents\" % len(dataset.data))\n",
    "print(\"%d categories\" % len(dataset.target_names))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf4e6945",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features from the training dataset using a sparse vectorizer\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "labels = dataset.target\n",
    "true_k = np.unique(labels).shape[0]\n",
    "\n",
    "print(\"Extracting features from the training dataset \"\n",
    "      \"using a sparse vectorizer\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6cc06cb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_samples: 3387, n_features: 10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_df=0.5, max_features=10000,\n",
    "                             min_df=2, stop_words='english',\n",
    "                             use_idf=True)\n",
    "\n",
    "X = vectorizer.fit_transform(dataset.data)\n",
    "\n",
    "print(\"n_samples: %d, n_features: %d\" % X.shape)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "64d6a098",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing dimensionality reduction using LSA\n",
      "Explained variance of the SVD step: 24%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "print(\"Performing dimensionality reduction using LSA\")\n",
    "\n",
    "# Vectorizer results are normalized, which makes KMeans behave as\n",
    "# spherical k-means for better results. Since LSA/SVD results are\n",
    "# not normalized, we have to redo the normalization.\n",
    "svd = TruncatedSVD(100)\n",
    "normalizer = Normalizer(copy=False)\n",
    "lsa = make_pipeline(svd, normalizer)\n",
    "\n",
    "X = lsa.fit_transform(X)\n",
    "\n",
    "explained_variance = svd.explained_variance_ratio_.sum()\n",
    "print(\"Explained variance of the SVD step: {}%\".format(\n",
    "    int(explained_variance * 100)))\n",
    "\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37cfa9d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "# Cluster with MiniBatchKMeans\n",
    "km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,\n",
    "                     init_size=1000, batch_size=1000, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f4273874",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "# Cluster with regular KMeans\n",
    "km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fd994b03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "km.fit(X)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "dee5120f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Homogeneity: 0.644\n",
      "Completeness: 0.694\n",
      "V-measure: 0.668\n",
      "Adjusted Rand-Index: 0.666\n",
      "Silhouette Coefficient: 0.045\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(labels, km.labels_))\n",
    "print(\"Completeness: %0.3f\" % metrics.completeness_score(labels, km.labels_))\n",
    "print(\"V-measure: %0.3f\" % metrics.v_measure_score(labels, km.labels_))\n",
    "print(\"Adjusted Rand-Index: %.3f\"\n",
    "      % metrics.adjusted_rand_score(labels, km.labels_))\n",
    "print(\"Silhouette Coefficient: %0.3f\"\n",
    "      % metrics.silhouette_score(X, km.labels_, sample_size=1000))\n",
    "\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "82be8bad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "# Inverse transform from SVD\n",
    "original_space_centroids = svd.inverse_transform(km.cluster_centers_)\n",
    "order_centroids = original_space_centroids.argsort()[:, ::-1]\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cc2cedac",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "# No SVD\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "29e26fe2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top terms per cluster:\n",
      "Cluster 0: graphics thanks university image com files 3d help program posting\n",
      "Cluster 1: space nasa access henry com digex moon gov pat shuttle\n",
      "Cluster 2: god com people jesus don religion article sandvik bible just\n",
      "Cluster 3: keith sgi livesey caltech objective morality wpd solntze jon com\n"
     ]
    }
   ],
   "source": [
    "%%write_and_run -a execution_log.py\n",
    "\n",
    "print(\"Top terms per cluster:\")\n",
    "\n",
    "terms = vectorizer.get_feature_names()\n",
    "for i in range(true_k):\n",
    "    print(\"Cluster %d:\" % i, end='')\n",
    "    for ind in order_centroids[i, :10]:\n",
    "        print(' %s' % terms[ind], end='')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4e578c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}