correct_yoruba_diacritics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yorùbá diacritic restoration &rarr; Quickstart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 0  &rarr; Confam the python environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "print(sys.executable)\n",
    "print(sys.version)\n",
    "print(sys.version_info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1  &rarr; Confam the pre-built Yorùbá ADR model (67 MB) in `./models` sub-directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "! ls -alh ./models/yo_adr_bahdanau_lstm_128_2_2_step_90000_release.pt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2 &rarr; View the file we will now correct\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "awon okunrin nse ise agbara bi ise ode"
     ]
    }
   ],
   "source": [
    "! cat ./data/test/one_phrase.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3 &rarr; Generate correct diacritics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/iroro/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:46: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.3 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n",
      "[2019-04-28 10:34:54,043 INFO] Translating shard 0.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PRED AVG SCORE: -0.1102, PRED PPL: 1.1166\n",
      "\n",
      "awon okunrin nse ise agbara bi ise ode\n",
      "àwọn ọkùnrin nṣe iṣẹ́ agbára bí iṣẹ́ ọdẹ\n"
     ]
    }
   ],
   "source": [
    "%run ./src/translate.py \\\n",
    "        -model models/yo_adr_bahdanau_lstm_128_2_2_step_90000_release.pt \\\n",
    "        -src data/test/one_phrase.txt \\\n",
    "        -output data/test/pred.txt \\\n",
    "        -replace_unk\n",
    "\n",
    "# View input and output\n",
    "! echo\n",
    "! cat ./data/test/one_phrase.txt\n",
    "! echo \n",
    "! cat ./data/test/pred.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (Alternate) Step 3 &rarr; Generate correct diacritics with shell script &rarr; `02_run_prediction.sh`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] running inference on test sources\n",
      "/Users/iroro/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:46: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.3 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n",
      "[2019-04-28 10:34:55,325 INFO] Translating shard 0.\n",
      "\n",
      "SENT 1: ['awon', 'okunrin', 'nse', 'ise', 'agbara', 'bi', 'ise', 'ode']\n",
      "PRED 1: àwọn ọkùnrin nṣe iṣẹ́ agbára bí iṣẹ́ ọdẹ\n",
      "PRED SCORE: -0.8820\n",
      "PRED AVG SCORE: -0.1102, PRED PPL: 1.1166\n",
      "\n",
      "awon okunrin nse ise agbara bi ise ode\n",
      "àwọn ọkùnrin nṣe iṣẹ́ agbára bí iṣẹ́ ọdẹ\n"
     ]
    }
   ],
   "source": [
    "! bash '02_run_prediction.sh'\n",
    "\n",
    "! echo \n",
    "! cat ./data/test/one_phrase.txt\n",
    "! echo \n",
    "! cat ./data/test/pred.txt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}