Merge branch 'dev' of https://github.com/afunTW/Python-Crawling-Tutorial into dev

Author: afunTW

00_GET_POST/01_POST_thsrc_time_table.ipynb

@@ -16,9 +16,7 @@
  {
  "cell_type": "code",
  "execution_count": 1,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
  "outputs": [],
  "source": [
  "import os\n",
@@ -34,15 +32,13 @@
  {
  "cell_type": "code",
  "execution_count": 2,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
  "outputs": [
  {
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "The date after one week - 2018/02/09\n"
+ "The date after one week - 2018/02/28\n"
  ]
  }
  ],
@@ -65,9 +61,7 @@
  {
  "cell_type": "code",
  "execution_count": 3,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
  "outputs": [],
  "source": [
  "resp = requests.post(url, data=form_data)\n",
@@ -78,14 +72,25 @@
  {
  "cell_type": "code",
  "execution_count": 4,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
  "outputs": [
  {
  "data": {
  "text/html": [
  "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
  "<table border=\"1\" class=\"dataframe\">\n",
  " <thead>\n",
  " <tr style=\"text-align: right;\">\n",
@@ -104,157 +109,77 @@
  " <td>14:11</td>\n",
  " <td>16:11</td>\n",
  " <td>02:00</td>\n",
- " <td>65折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>1</th>\n",
- " <td>0833</td>\n",
- " <td>14:11</td>\n",
- " <td>16:11</td>\n",
- " <td>02:00</td>\n",
- " <td>65折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>2</th>\n",
- " <td>1649</td>\n",
- " <td>14:21</td>\n",
- " <td>16:06</td>\n",
- " <td>01:45</td>\n",
  " <td>8折起</td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>3</th>\n",
- " <td>1649</td>\n",
- " <td>14:21</td>\n",
- " <td>16:06</td>\n",
- " <td>01:45</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>4</th>\n",
- " <td>0651</td>\n",
- " <td>14:46</td>\n",
- " <td>16:32</td>\n",
- " <td>01:46</td>\n",
- " <td></td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>5</th>\n",
+ " <th>1</th>\n",
  " <td>0651</td>\n",
  " <td>14:46</td>\n",
  " <td>16:32</td>\n",
  " <td>01:46</td>\n",
  " <td></td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>6</th>\n",
- " <td>0837</td>\n",
- " <td>15:11</td>\n",
- " <td>17:11</td>\n",
- " <td>02:00</td>\n",
- " <td>65折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>7</th>\n",
+ " <th>2</th>\n",
  " <td>0837</td>\n",
  " <td>15:11</td>\n",
  " <td>17:11</td>\n",
  " <td>02:00</td>\n",
- " <td>65折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>8</th>\n",
- " <td>1655</td>\n",
- " <td>15:21</td>\n",
- " <td>17:06</td>\n",
- " <td>01:45</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>9</th>\n",
- " <td>1655</td>\n",
- " <td>15:21</td>\n",
- " <td>17:06</td>\n",
- " <td>01:45</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>10</th>\n",
- " <td>0657</td>\n",
- " <td>15:46</td>\n",
- " <td>17:32</td>\n",
- " <td>01:46</td>\n",
  " <td>8折起</td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>11</th>\n",
+ " <th>3</th>\n",
  " <td>0657</td>\n",
  " <td>15:46</td>\n",
  " <td>17:32</td>\n",
  " <td>01:46</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>12</th>\n",
- " <td>1237</td>\n",
- " <td>15:51</td>\n",
- " <td>17:17</td>\n",
- " <td>01:26</td>\n",
- " <td></td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>13</th>\n",
- " <td>1237</td>\n",
- " <td>15:51</td>\n",
- " <td>17:17</td>\n",
- " <td>01:26</td>\n",
  " <td></td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>14</th>\n",
- " <td>0841</td>\n",
- " <td>16:11</td>\n",
- " <td>18:11</td>\n",
- " <td>02:00</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>15</th>\n",
+ " <th>4</th>\n",
  " <td>0841</td>\n",
  " <td>16:11</td>\n",
  " <td>18:11</td>\n",
  " <td>02:00</td>\n",
- " <td>8折起</td>\n",
- " </tr>\n",
- " <tr>\n",
- " <th>16</th>\n",
- " <td>0661</td>\n",
- " <td>16:21</td>\n",
- " <td>18:06</td>\n",
- " <td>01:45</td>\n",
- " <td></td>\n",
+ " <td>65折起</td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>17</th>\n",
+ " <th>5</th>\n",
  " <td>0661</td>\n",
  " <td>16:21</td>\n",
  " <td>18:06</td>\n",
  " <td>01:45</td>\n",
- " <td></td>\n",
+ " <td>8折起</td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>18</th>\n",
+ " <th>6</th>\n",
  " <td>0663</td>\n",
  " <td>16:46</td>\n",
  " <td>18:32</td>\n",
  " <td>01:46</td>\n",
  " <td></td>\n",
  " </tr>\n",
  " <tr>\n",
- " <th>19</th>\n",
- " <td>0663</td>\n",
- " <td>16:46</td>\n",
- " <td>18:32</td>\n",
+ " <th>7</th>\n",
+ " <td>0845</td>\n",
+ " <td>17:11</td>\n",
+ " <td>19:11</td>\n",
+ " <td>02:00</td>\n",
+ " <td>65折起</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>8</th>\n",
+ " <td>0667</td>\n",
+ " <td>17:21</td>\n",
+ " <td>19:06</td>\n",
+ " <td>01:45</td>\n",
+ " <td>8折起</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>9</th>\n",
+ " <td>0669</td>\n",
+ " <td>17:46</td>\n",
+ " <td>19:32</td>\n",
  " <td>01:46</td>\n",
  " <td></td>\n",
  " </tr>\n",
@@ -263,27 +188,17 @@
  "</div>"
  ],
  "text/plain": [
- " 車次 出發時間 抵達時間 行車時間 早鳥\n",
- "0 0833 14:11 16:11 02:00 65折起\n",
- "1 0833 14:11 16:11 02:00 65折起\n",
- "2 1649 14:21 16:06 01:45 8折起\n",
- "3 1649 14:21 16:06 01:45 8折起\n",
- "4 0651 14:46 16:32 01:46 \n",
- "5 0651 14:46 16:32 01:46 \n",
- "6 0837 15:11 17:11 02:00 65折起\n",
- "7 0837 15:11 17:11 02:00 65折起\n",
- "8 1655 15:21 17:06 01:45 8折起\n",
- "9 1655 15:21 17:06 01:45 8折起\n",
- "10 0657 15:46 17:32 01:46 8折起\n",
- "11 0657 15:46 17:32 01:46 8折起\n",
- "12 1237 15:51 17:17 01:26 \n",
- "13 1237 15:51 17:17 01:26 \n",
- "14 0841 16:11 18:11 02:00 8折起\n",
- "15 0841 16:11 18:11 02:00 8折起\n",
- "16 0661 16:21 18:06 01:45 \n",
- "17 0661 16:21 18:06 01:45 \n",
- "18 0663 16:46 18:32 01:46 \n",
- "19 0663 16:46 18:32 01:46 "
+ " 車次 出發時間 抵達時間 行車時間 早鳥\n",
+ "0 0833 14:11 16:11 02:00 8折起\n",
+ "1 0651 14:46 16:32 01:46 \n",
+ "2 0837 15:11 17:11 02:00 8折起\n",
+ "3 0657 15:46 17:32 01:46 \n",
+ "4 0841 16:11 18:11 02:00 65折起\n",
+ "5 0661 16:21 18:06 01:45 8折起\n",
+ "6 0663 16:46 18:32 01:46 \n",
+ "7 0845 17:11 19:11 02:00 65折起\n",
+ "8 0667 17:21 19:06 01:45 8折起\n",
+ "9 0669 17:46 19:32 01:46 "
  ]
  },
  "execution_count": 4,
@@ -292,7 +207,7 @@
  }
  ],
  "source": [
- "rows = soup.table.find_all('tr')\n",
+ "rows = soup.table.find_all('tr', recursive=False)\n",
  "\n",
  "colname, rows = rows[1], rows[2:]\n",
  "colname = list(colname.stripped_strings)\n",
@@ -312,23 +227,21 @@
  " early_ticket = early_ticket[0] if early_ticket else ''\n",
  " \n",
  " rows[i] = [trips, t_departure, t_arrive, duration, early_ticket]\n",
- " \n",
+ "\n",
  "df = pd.DataFrame(rows, columns=colname)\n",
  "df"
  ]
  },
  {
  "cell_type": "code",
  "execution_count": 5,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
  "outputs": [
  {
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "Save csv to /home/afun/github/Python-Crawling-Tutorial/results/thsrc_20180209.csv\n"
+ "Save csv to /home/afun/github/Python-Crawling-Tutorial/results/thsrc_20180228.csv\n"
  ]
  }
  ],