notebook/5-1-selenium.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Selenium tutorial\n",
    "\n",
    "**[Selenium](http://selenium-python.readthedocs.io) is a project to test website then be used to scraping because some website need to run javascript. If using what we learned before, we cannot get what we need when the actural content run by javascript.**\n",
    "\n",
    "**Install [Selenium](http://selenium-python.readthedocs.io/installation.html) and it will open a actural browser to do all the jobs.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.makedirs('./img/', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"cmn-Hans en\"><head>\n  <meta charset=\"UTF-8\" />\n  <meta name=\"\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "\n",
    "driver = webdriver.Chrome()\n",
    "driver.get(\"https://mofanpy.com/\")\n",
    "driver.find_element_by_xpath(u\"//img[@alt='强化学习 (Reinforcement Learning)']\").click()\n",
    "driver.find_element_by_link_text(\"About\").click()\n",
    "driver.find_element_by_link_text(u\"赞助\").click()\n",
    "driver.find_element_by_link_text(u\"教程 ▾\").click()\n",
    "driver.find_element_by_link_text(u\"数据处理 ▾\").click()\n",
    "driver.find_element_by_link_text(u\"网页爬虫\").click()\n",
    "\n",
    "html = driver.page_source       # get html\n",
    "driver.get_screenshot_as_file(\"./img/sreenshot1.png\")\n",
    "driver.close()\n",
    "print(html[:200])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![img](/img/sreenshot1.png)\n",
    "**If don't want to the broswer to show up, you can define the option first.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\"><html xmlns=\"http://www.w3.org/1999/xhtml\" lang=\"cmn-Hans en\"><head>\n  <meta charset=\"UTF-8\" />\n  <meta name=\"\n"
     ]
    }
   ],
   "source": [
    "from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--headless\")       # define headless\n",
    "\n",
    "# add the option when creating driver\n",
    "driver = webdriver.Chrome(chrome_options=chrome_options)    \n",
    "driver.get(\"https://mofanpy.com/\")\n",
    "driver.find_element_by_xpath(u\"//img[@alt='强化学习 (Reinforcement Learning)']\").click()\n",
    "driver.find_element_by_link_text(\"About\").click()\n",
    "driver.find_element_by_link_text(u\"赞助\").click()\n",
    "driver.find_element_by_link_text(u\"教程 ▾\").click()\n",
    "driver.find_element_by_link_text(u\"数据处理 ▾\").click()\n",
    "driver.find_element_by_link_text(u\"网页爬虫\").click()\n",
    "\n",
    "html = driver.page_source           # get html\n",
    "driver.get_screenshot_as_file(\"./img/sreenshot2.png\")\n",
    "driver.close()\n",
    "print(html[:200])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![img](/img/sreenshot2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}