modified to crawl today's articles

Author: afunTW

appendix_ptt/02_yesterday_articles.ipynb renamed to appendix_ptt/02_today_articles.ipynb

@@ -4,7 +4,7 @@
  "cell_type": "markdown",
  "metadata": {},
  "source": [
- "# 爬取昨天的所有文章\n",
+ "# 爬取今天到目前為止的所有文章\n",
  "\n",
  "https://www.ptt.cc/bbs/Gossiping/index.html"
  ]
@@ -20,7 +20,7 @@
  "import json\n",
  "\n",
  "from bs4 import BeautifulSoup, NavigableString\n",
- "from datetime import datetime, timedelta\n",
+ "from datetime import datetime\n",
  "from pprint import pprint\n",
  "from urllib.parse import urljoin"
  ]
@@ -34,15 +34,15 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "09/25\n"
+ "09/27\n"
  ]
  }
  ],
  "source": [
  "base_url = 'https://www.ptt.cc/bbs/Gossiping/index.html'\n",
- "ptt_yesterday = datetime.now() - timedelta(days=1)\n",
- "ptt_yesterday_str = ptt_yesterday.strftime('%m/%d')\n",
- "print(ptt_yesterday_str)"
+ "ptt_today = datetime.now()\n",
+ "ptt_today_str = ptt_today.strftime('%m/%d')\n",
+ "print(ptt_today_str)"
  ]
  },
  {
@@ -74,7 +74,7 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "total page = 39219\n"
+ "total page = 39228\n"
  ]
  }
  ],
@@ -210,11 +210,62 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "https://www.ptt.cc/bbs/Gossiping/index39219.html - date 9/26 result 1\n"
+ "https://www.ptt.cc/bbs/Gossiping/index39228.html - date 9/27 result 0\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978608.A.325.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978662.A.45A.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978695.A.9A7.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978699.A.194.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978724.A.356.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978750.A.39A.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978768.A.08B.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978815.A.5B2.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978820.A.119.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978934.A.F8E.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978941.A.754.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978960.A.779.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978973.A.B90.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978993.A.F88.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537979013.A.67C.html\n",
+ "https://www.ptt.cc/bbs/Gossiping/index39227.html - date 9/27 result 0\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977913.A.4EE.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977930.A.01B.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977933.A.013.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977952.A.904.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977959.A.A7B.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977966.A.77C.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978043.A.03E.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978060.A.9DF.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978098.A.D36.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978140.A.C44.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978152.A.31C.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978156.A.B1A.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978179.A.844.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978195.A.D33.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978272.A.533.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978295.A.B6A.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978350.A.D02.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978378.A.746.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978494.A.B6B.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537978521.A.06B.html\n",
+ "https://www.ptt.cc/bbs/Gossiping/index39226.html - date 9/26 result -1\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977639.A.3F8.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977693.A.A67.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977700.A.FD6.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977711.A.493.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977729.A.BE4.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977740.A.534.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977827.A.B50.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977851.A.17A.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977857.A.B1D.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977877.A.292.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977878.A.13E.html\n",
+ "Start to Crawling https://www.ptt.cc/bbs/Gossiping/M.1537977910.A.566.html\n",
+ "Save - today_articles.json\n"
  ]
  }
  ],
  "source": [
+ "data = []\n",
  "for page in range(total_page, 1, -1):\n",
  " current_url = 'https://www.ptt.cc/bbs/Gossiping/index{}.html'.format(page)\n",
  " resp_page = requests.get(current_url, cookies={'over18': '1'})\n",
@@ -228,17 +279,34 @@
  " container_tag = soup_page.find('div', class_='r-list-container')\n",
  " first_article = container_tag.find('div', class_='r-ent')\n",
  " first_article_date = first_article.find('div', class_='date').text.strip()\n",
- " compare_datetime = compare_timestamp_md(ptt_yesterday_str, first_article_date)\n",
+ " compare_datetime = compare_timestamp_md(ptt_today_str, first_article_date)\n",
  " print('{} - date {} result {}'.format(current_url, first_article_date, compare_datetime))\n",
- " break"
+ " \n",
+ " if compare_datetime == 1:\n",
+ " continue\n",
+ " else:\n",
+ " # only crawling today's article before r-list-sep line\n",
+ " for article_row_tag in container_tag.findChildren('div', recursive=False):\n",
+ " if 'r-list-sep' in article_row_tag['class']:\n",
+ " break\n",
+ " if 'r-ent' in article_row_tag['class']:\n",
+ " article_date = article_row_tag.find('div', class_='date').text.strip()\n",
+ " article_date_compare = compare_timestamp_md(ptt_today_str, article_date)\n",
+ " if article_date_compare != 0:\n",
+ " continue\n",
+ " article_tag = article_row_tag.find('a', href=True)\n",
+ " article_url = urljoin(base_url, article_tag['href'])\n",
+ " article_data = crawl_article(article_url)\n",
+ " data.append(article_data)\n",
+ "\n",
+ " # if the first article date is earlier than current date, should break the iteration\n",
+ " if compare_datetime == -1:\n",
+ " break\n",
+ "\n",
+ "with open('today_articles.json', 'w+', encoding='utf-8') as f:\n",
+ " json.dump(data, f, indent=2, ensure_ascii=False)\n",
+ " print('Save - today_articles.json')"
  ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
  }
  ],
  "metadata": {