openbus_12_gtfstk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put these at the top of every notebook, to get automatic reloading and inline plotting\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import partridge as ptg\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import altair as alt\n",
    "import datetime\n",
    "import gtfstk as gt\n",
    "\n",
    "from gtfs_utils import *\n",
    "\n",
    "alt.renderers.enable('notebook')\n",
    "alt.data_transformers.enable('json')\n",
    "\n",
    "sns.set_style(\"white\")\n",
    "sns.set_context(\"talk\")\n",
    "sns.set_palette('Set2', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2016-05-06.zip',\n",
       " '2017-01-01.zip',\n",
       " '2018-02-01.zip',\n",
       " '2018-02-02.zip',\n",
       " '2018-02-03.zip',\n",
       " '2018-02-04.zip',\n",
       " '2018-02-05.zip',\n",
       " '2018-02-06.zip',\n",
       " '2018-02-07.zip',\n",
       " '2018-02-08.zip',\n",
       " '2018-02-09.zip',\n",
       " '2018-02-10.zip',\n",
       " '2018-02-11.zip',\n",
       " '2018-02-12.zip',\n",
       " '2018-02-13.zip',\n",
       " '2018-02-14.zip',\n",
       " '2018-02-15.zip',\n",
       " '2018-02-16.zip',\n",
       " '2018-02-17.zip',\n",
       " '2018-02-18.zip',\n",
       " '2018-02-19.zip',\n",
       " '2018-02-21.zip',\n",
       " '2018-02-22.zip',\n",
       " '2018-02-23.zip',\n",
       " '2018-02-24.zip',\n",
       " '2018-02-25.zip',\n",
       " '2018-02-26.zip',\n",
       " '2018-02-27.zip',\n",
       " '2018-02-28.zip',\n",
       " '2018-04-25.zip',\n",
       " '2018-04-26.zip',\n",
       " '2018-05-01.zip',\n",
       " '2018-05-02.zip',\n",
       " '2018-05-15.zip']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gtfs_repo = 'data/gtfs_feeds/'\n",
    "output_folder = 'data/gtfstk_stats/'\n",
    "import os\n",
    "os.listdir(gtfs_repo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_list = [ '2018-02-01', ]\n",
    "ts = []\n",
    "rs = []\n",
    "for d in date_list:\n",
    "    feed = gt.read_gtfs(gtfs_repo+d+'.zip', dist_units='m')\n",
    "    trip_stats = gt.compute_trip_stats(feed)\n",
    "    ts.append(trip_stats)\n",
    "    route_stats = feed.compute_route_stats(trip_stats, [d.replace('-', '')], split_directions=True )\n",
    "    rs.append(route_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>route_id</th>\n",
       "      <th>direction_id</th>\n",
       "      <th>route_short_name</th>\n",
       "      <th>route_type</th>\n",
       "      <th>num_trips</th>\n",
       "      <th>num_trip_starts</th>\n",
       "      <th>num_trip_ends</th>\n",
       "      <th>is_loop</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>...</th>\n",
       "      <th>peak_num_trips</th>\n",
       "      <th>peak_start_time</th>\n",
       "      <th>peak_end_time</th>\n",
       "      <th>service_distance</th>\n",
       "      <th>service_duration</th>\n",
       "      <th>is_bidirectional</th>\n",
       "      <th>service_speed</th>\n",
       "      <th>mean_trip_distance</th>\n",
       "      <th>mean_trip_duration</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>05:10:00</td>\n",
       "      <td>21:54:07</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>05:10:00</td>\n",
       "      <td>05:29:07</td>\n",
       "      <td>183170.0</td>\n",
       "      <td>8.283889</td>\n",
       "      <td>0</td>\n",
       "      <td>22111.595466</td>\n",
       "      <td>7045.0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>20180201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>06:05:00</td>\n",
       "      <td>22:51:36</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15:55:00</td>\n",
       "      <td>16:06:36</td>\n",
       "      <td>283920.0</td>\n",
       "      <td>12.600000</td>\n",
       "      <td>0</td>\n",
       "      <td>22533.333333</td>\n",
       "      <td>8112.0</td>\n",
       "      <td>0.360000</td>\n",
       "      <td>20180201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10000</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>08:17:31</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>08:17:31</td>\n",
       "      <td>6326.0</td>\n",
       "      <td>0.291944</td>\n",
       "      <td>0</td>\n",
       "      <td>21668.506185</td>\n",
       "      <td>6326.0</td>\n",
       "      <td>0.291944</td>\n",
       "      <td>20180201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10001</td>\n",
       "      <td>1</td>\n",
       "      <td>91</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13:00:00</td>\n",
       "      <td>16:51:42</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13:00:00</td>\n",
       "      <td>13:21:42</td>\n",
       "      <td>41880.0</td>\n",
       "      <td>1.808333</td>\n",
       "      <td>0</td>\n",
       "      <td>23159.447005</td>\n",
       "      <td>8376.0</td>\n",
       "      <td>0.361667</td>\n",
       "      <td>20180201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10002</td>\n",
       "      <td>0</td>\n",
       "      <td>92</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>07:30:00</td>\n",
       "      <td>07:53:03</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>07:30:00</td>\n",
       "      <td>07:53:03</td>\n",
       "      <td>7175.0</td>\n",
       "      <td>0.384167</td>\n",
       "      <td>0</td>\n",
       "      <td>18676.789588</td>\n",
       "      <td>7175.0</td>\n",
       "      <td>0.384167</td>\n",
       "      <td>20180201</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  route_id  direction_id route_short_name  route_type  num_trips  \\\n",
       "0        1             0                1         3.0       26.0   \n",
       "1       10             1                4         3.0       35.0   \n",
       "2    10000             0               91         3.0        1.0   \n",
       "3    10001             1               91         3.0        5.0   \n",
       "4    10002             0               92         3.0        1.0   \n",
       "\n",
       "   num_trip_starts  num_trip_ends  is_loop start_time  end_time    ...     \\\n",
       "0             26.0           26.0      0.0   05:10:00  21:54:07    ...      \n",
       "1             35.0           35.0      0.0   06:05:00  22:51:36    ...      \n",
       "2              1.0            1.0      0.0   08:00:00  08:17:31    ...      \n",
       "3              5.0            5.0      0.0   13:00:00  16:51:42    ...      \n",
       "4              1.0            1.0      0.0   07:30:00  07:53:03    ...      \n",
       "\n",
       "   peak_num_trips  peak_start_time  peak_end_time  service_distance  \\\n",
       "0             1.0         05:10:00       05:29:07          183170.0   \n",
       "1             2.0         15:55:00       16:06:36          283920.0   \n",
       "2             1.0         08:00:00       08:17:31            6326.0   \n",
       "3             2.0         13:00:00       13:21:42           41880.0   \n",
       "4             1.0         07:30:00       07:53:03            7175.0   \n",
       "\n",
       "  service_duration is_bidirectional  service_speed  mean_trip_distance  \\\n",
       "0         8.283889                0   22111.595466              7045.0   \n",
       "1        12.600000                0   22533.333333              8112.0   \n",
       "2         0.291944                0   21668.506185              6326.0   \n",
       "3         1.808333                0   23159.447005              8376.0   \n",
       "4         0.384167                0   18676.789588              7175.0   \n",
       "\n",
       "   mean_trip_duration      date  \n",
       "0            0.318611  20180201  \n",
       "1            0.360000  20180201  \n",
       "2            0.291944  20180201  \n",
       "3            0.361667  20180201  \n",
       "4            0.384167  20180201  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trip_id</th>\n",
       "      <th>route_id</th>\n",
       "      <th>route_short_name</th>\n",
       "      <th>route_type</th>\n",
       "      <th>direction_id</th>\n",
       "      <th>shape_id</th>\n",
       "      <th>num_stops</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>start_stop_id</th>\n",
       "      <th>end_stop_id</th>\n",
       "      <th>is_loop</th>\n",
       "      <th>duration</th>\n",
       "      <th>distance</th>\n",
       "      <th>speed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>161982</th>\n",
       "      <td>29789140_300118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>71053</td>\n",
       "      <td>18</td>\n",
       "      <td>05:05:00</td>\n",
       "      <td>05:24:07</td>\n",
       "      <td>38725</td>\n",
       "      <td>15745</td>\n",
       "      <td>0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>7045</td>\n",
       "      <td>22111.595466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162004</th>\n",
       "      <td>29789162_300118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>71053</td>\n",
       "      <td>18</td>\n",
       "      <td>05:10:00</td>\n",
       "      <td>05:29:07</td>\n",
       "      <td>38725</td>\n",
       "      <td>15745</td>\n",
       "      <td>0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>7045</td>\n",
       "      <td>22111.595466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161983</th>\n",
       "      <td>29789141_300118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>71053</td>\n",
       "      <td>18</td>\n",
       "      <td>05:30:00</td>\n",
       "      <td>05:49:07</td>\n",
       "      <td>38725</td>\n",
       "      <td>15745</td>\n",
       "      <td>0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>7045</td>\n",
       "      <td>22111.595466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162005</th>\n",
       "      <td>29789163_300118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>71053</td>\n",
       "      <td>18</td>\n",
       "      <td>05:40:00</td>\n",
       "      <td>05:59:07</td>\n",
       "      <td>38725</td>\n",
       "      <td>15745</td>\n",
       "      <td>0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>7045</td>\n",
       "      <td>22111.595466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161984</th>\n",
       "      <td>29789142_300118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>71053</td>\n",
       "      <td>18</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>06:19:07</td>\n",
       "      <td>38725</td>\n",
       "      <td>15745</td>\n",
       "      <td>0</td>\n",
       "      <td>0.318611</td>\n",
       "      <td>7045</td>\n",
       "      <td>22111.595466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                trip_id route_id route_short_name  route_type  direction_id  \\\n",
       "161982  29789140_300118        1                1           3             0   \n",
       "162004  29789162_300118        1                1           3             0   \n",
       "161983  29789141_300118        1                1           3             0   \n",
       "162005  29789163_300118        1                1           3             0   \n",
       "161984  29789142_300118        1                1           3             0   \n",
       "\n",
       "       shape_id  num_stops start_time  end_time start_stop_id end_stop_id  \\\n",
       "161982    71053         18   05:05:00  05:24:07         38725       15745   \n",
       "162004    71053         18   05:10:00  05:29:07         38725       15745   \n",
       "161983    71053         18   05:30:00  05:49:07         38725       15745   \n",
       "162005    71053         18   05:40:00  05:59:07         38725       15745   \n",
       "161984    71053         18   06:00:00  06:19:07         38725       15745   \n",
       "\n",
       "        is_loop  duration  distance         speed  \n",
       "161982        0  0.318611      7045  22111.595466  \n",
       "162004        0  0.318611      7045  22111.595466  \n",
       "161983        0  0.318611      7045  22111.595466  \n",
       "162005        0  0.318611      7045  22111.595466  \n",
       "161984        0  0.318611      7045  22111.595466  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir(output_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d, t, r in zip(date_list, ts, rs):\n",
    "    t.to_csv(output_folder+d+'_trip_stats.csv')\n",
    "    r.to_csv(output_folder+d+'_route_stats.csv')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import gzip\n",
    "for d, t, r in zip(date_list, ts, rs):\n",
    "    with gzip.open('2014/seed11-csv 27.5.2018.merged.csv.gz', 'wb') as f:\n",
    "        f.write(t.to_csv(None).encode('utf-8'))\n",
    "        \n",
    "    t.to_csv(output_folder+d+'_trip_stats.csv')\n",
    "    r.to_csv(output_folder+d+'_route_stats.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}