Python微医挂号网医生数据抓取

import asyncio
from pyppeteer import launch
async def main():
  browser = await launch() # 运行一个无头的浏览器
  page = await browser.newPage() # 打开一个选项卡
  await page.goto('http://www.baidu.com') # 加载一个页面
  await page.screenshot({'path': 'baidu.png'}) # 把网页生成截图
  await browser.close()
asyncio.get_event_loop().run_until_complete(main()) # 异步

browser = await launch(headless=False) # 可以打开浏览器
await page.click('#login_user') # 点击一个按钮
await page.type('#login_user', 'admin') # 输入内容
await page.click('#password') 
await page.type('#password', '123456')
await page.click('#login-submit')
await page.waitForNavigation() 
# 设置浏览器窗口大小
await page.setViewport({
  'width': 1350,
  'height': 850
})
content = await page.content() # 获取网页内容
cookies = await page.cookies() # 获取网页cookies

import asyncio
from pyppeteer import launch
class DoctorSpider(object):
  async def main(self, num):
    try:
      browser = await launch(headless=True)
      page = await browser.newPage()
      print(f"正在爬取第 {num} 页面")
      await page.goto("https://www.guahao.com/expert/all/全国/all/不限/p{}".format(num))
      content = await page.content()
      print(content)
    except Exception as e:
      print(e.args)
    finally:
      num += 1
      await browser.close()
      await self.main(num)
  def run(self):
    loop = asyncio.get_event_loop()
    asyncio.get_event_loop().run_until_complete(self.main(1))
if __name__ == '__main__':
  doctor = DoctorSpider()
  doctor.run()

import asyncio
from pyppeteer import launch
from pyquery import PyQuery as pq
import pandas as pd # 保存csv文件
class DoctorSpider(object):
  def __init__(self):
    self._data = list()
  async def main(self,num):
    try:
      browser = await launch(headless=True)
      page = await browser.newPage()
      print(f"正在爬取第 {num} 页面")
      await page.goto("https://www.guahao.com/expert/all/全国/all/不限/p{}".format(num))
      content = await page.content()
      self.parse_html(content)
      print("正在存储数据....")
      data = pd.DataFrame(self._data)
      data.to_csv("微医数据.csv", encoding='utf_8_sig')
    except Exception as e:
      print(e.args)
    finally:
      num+=1
      await browser.close()
      await self.main(num)
  def parse_html(self,content):
    doc = pq(content)
    items = doc(".g-doctor-item").items()
    for item in items:
      #doctor_name = item.find(".seo-anchor-text").text()
      name_level = item.find(".g-doc-baseinfo>dl>dt").text() # 姓名和级别
      department = item.find(".g-doc-baseinfo>dl>dd>p:eq(0)").text() # 科室
      address = item.find(".g-doc-baseinfo>dl>dd>p:eq(1)").text() # 医院地址
      star = item.find(".star-count em").text() # 评分
      inquisition = item.find(".star-count i").text() # 问诊量
      expert_team = item.find(".expert-team").text() # 专家团队
      service_price_img = item.find(".service-name:eq(0)>.fee").text()
      service_price_video = item.find(".service-name:eq(1)>.fee").text()
      one_data = {
        "name": name_level.split(" ")[0],
        "level": name_level.split(" ")[1],
        "department": department,
        "address": address,
        "star": star,
        "inquisition": inquisition,
        "expert_team": expert_team,
        "service_price_img": service_price_img,
        "service_price_video": service_price_video
      }
      self._data.append(one_data)
  def run(self):
    loop = asyncio.get_event_loop()
    asyncio.get_event_loop().run_until_complete(self.main(1))
if __name__ == '__main__':
  doctor = DoctorSpider()
  doctor.run()