python 中xpath爬虫实例详解

import requests 倒入requests库
from lxml import etree 倒入lxml 库（没有这个库，pip install lxml安装）
url = "https://www.xxxx.com/gc/" 请求地址
response = requests.get(url= url) 返回结果
wb_data = response.text 文本展示返回结果
html = etree.HTML(wb_data) 将页面转换成文档树
b = html.xpath('//ul[@class = "clearfix"]//@href') 这一步的意思是class“clearfix”下所有属性为“href”赋值给“b”，因为我们的目标内容都展示在class“clearfix”下，且在href属性后面
print(b) 打印b，这里的b是一个数组
print(b[0]) 打印b的第一项数据

url = "https://www.xxxx.com/gc/toutiao/87098.html"
response = requests.get(url= url)
wb_data = response.text # 将页面转换成文档树
html = etree.HTML(wb_data)
b = html.xpath('//div[@class = "pictext"]//@href')
c=b[1]  #需要注意的地方，class = "pictext"下有两个href，我们只需要第一个href的值，所以返回值再赋值给c且取第二项数据
print(c)

url = "http://xxx.la/xam9I6"
response = requests.get(url= url)
wb_data = response.text
# 将页面转换成文档树
html = etree.HTML(wb_data)
b = html.xpath('//tr/td/text()')
c=b[6]#提取码
d = html.xpath('//tr//@href')#百度地址
print(c)
print(d)

# -*-coding:utf8-*-
# encoding:utf-8

import requests
from lxml import etree

firstlink = "https://www.xxx.com/gc/qt/83720.html"
AA=["https://www.xxx.com/gc/",
 "https://www.xxx.com/gc/index_2.html",
 "https://www.xxx.com/gc/index_3.html",
 "https://www.xxx.com/gq/",
 "https://www.xxx.com/gq/index_2.html",
 "https://www.xxx.com/gq/index_3.html",
 "https://www.xxx.com/gq/index_4.html"]

#第1步，获取第一页面所有的地址
def stepa (AA):
 lit=[]
 for url in AA:
  response = requests.get(url=url)
  wb_data = response.text
  # 将页面转换成文档树
  html = etree.HTML(wb_data)
  a = html.xpath('//ul[@class = "clearfix"]//@href')
  lit.append(a)
 return(lit) 
alllink = stepa(AA)

#第2步，获取的地址，循环读取打开，从而获取百度网盘信息
def stepb(alllink,firstlink):
 for list in alllink:
  for url in list:
   if url in firstlink:
    continue
   elif "www" in url:
    url2 = url
   else:
    url2 ="https://www.xxx.com" +url
   response = requests.get(url=url2)
   wb_data = response.text # 将页面转换成文档树
   html = etree.HTML(wb_data)
   b = html.xpath('//div[@class = "pictext"]//@href')
   c = b[1]
   #print(c)
   #获取到广告页地址
   url3 = c
   response = requests.get(url=url3)
   wb_data = response.text
   # 将页面转换成文档树
   html = etree.HTML(wb_data)
   d = html.xpath('//tr/td/text()')
   #print(d)
   e=d[6]#获取提取码
   f = html.xpath('//tr//@href')#获取地址
   test = e[-5:]#提取码值只保留提取码（4位）
   test2 = f[-1]#链接只保留链接内容，去掉前后['']
   test3=test2+test#把链接和提取码拼接成一条数据
   print(test3)
   with open('C:/Users/Beckham/Desktop/python/1.txt', 'a',encoding='utf-8') as w:
    w.write('\n'+test3)
    w.close()
stepb(alllink,firstlink)

#第3步：提示爬取完成
def over():
 print("ok")
over()

# -*-coding:utf8-*-
# encoding:utf-8

import requests
from lxml import etree

firstlink = "https://book.douban.com/subject/30172069/comments/hot"

def stepa (firstlink):
 response = requests.get(url=firstlink)
 wb_data = response.text
 html = etree.HTML(wb_data)
 a = html.xpath('//*[@id="comments"]//div[2]/p/span')
 print(a)
stepa (firstlink)

# -*-coding:utf8-*-
# encoding:utf-8

import requests
from lxml import etree

firstlink = "https://book.douban.com/subject/30172069/comments/hot"

def stepa (firstlink):
 response = requests.get(url=firstlink)
 wb_data = response.text
 html = etree.HTML(wb_data)
 a = html.xpath('//*[@id="comments"]//div[2]/p/span/text()')
 print(a)
stepa (firstlink)