python爬虫获取淘宝天猫商品详细参数

import re 
from collections import OrderedDict 
from bs4 import BeautifulSoup 
from pyquery import PyQuery as pq #获取整个网页的源代码 
from config import * #可引用congif的所有变量 
 
import pymysql 
import urllib 
import json 
import bs4 
import requests 
from selenium import webdriver 
from selenium.webdriver.support.ui import WebDriverWait 
from pyquery import PyQuery as pq #获取整个网页的源代码 
import pandas as pd 
 
# 测试 淘宝+天猫，可完整输出及保存 
 
browser = webdriver.Firefox() 
wait = WebDriverWait(browser,10) 
 
####### 天猫上半部分详情 ############# 
def get_tianmao_header(url): 
 browser.get(url) 
 # wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item'))) #加载所有宝贝 
 html=browser.page_source 
 doc = pq(html) 
 # print(doc) 
 info = OrderedDict() # 存放该商品所具有的全部信息 
 items = doc('#page') 
 
 # info['店铺名'] = items.find('.slogo').find('.slogo-shopname').text() 
 # info['ID'] = items.find('#LineZing').attr['itemid'] 
 info['宝贝'] = items.find('.tb-detail-hd').find('h1').text() 
 info['促销价'] = items.find('#J_PromoPrice').find('.tm-promo-price').find('.tm-price').text() 
 info['原价'] = items.find('#J_StrPriceModBox').find('.tm-price').text() 
 # '月销量' :items.find('.tm-ind-panel').find('.tm-ind-item tm-ind-sellCount').find('.tm-indcon').find('.tm-count').text(), 
 info['月销量'] = items.find('.tm-ind-panel').find('.tm-indcon').find('.tm-count').text().split(' ',2)[0] 
 info['累计评价'] = items.find('#J_ItemRates').find('.tm-indcon').find('.tm-count').text() 
 # print(info) 
 return info 
 
######## 淘宝上半部分详情 ############### 
def get_taobao_header(url): 
 browser.get(url) 
 # wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item'))) #加载所有宝贝 
 html=browser.page_source 
 doc = pq(html) 
 # print(doc) 
 info = OrderedDict() # 存放该商品所具有的全部信息 
 items = doc('#page') 
 
 # info['店铺名'] = items.find('.tb-shop-seller').find('.tb-seller-name').text() 
 # info['ID'] = items.find('#J_Pine').attr['data-itemid'] 
 info['宝贝'] = items.find('#J_Title').find('h3').text() 
 info['原价'] = items.find('#J_StrPrice').find('.tb-rmb-num').text() 
 info['促销价'] = items.find('#J_PromoPriceNum').text() 
 # '月销量' :items.find('.tm-ind-panel').find('.tm-ind-item tm-ind-sellCount').find('.tm-indcon').find('.tm-count').text(), 
 info['月销量'] = items.find('#J_SellCounter').text() 
 info['累计评价'] = items.find('#J_RateCounter').text() 
 # print(info) 
 return info 
 
####################### 详情 ############################ 
# 抓取所有商品详情 
def get_Details(attrs,info): 
 # res = requests.get(url) 
 # soup = BeautifulSoup(res.text, "html.parser") 
 # 
 # attrs = soup.select('.attributes-list li') 
 
 # attrs= [<li title=" 薄">厚薄: 薄</li>, <li title=" 其他100%">材质成分: 其他100%</li>,<li ...</li>] 
 attrs_name = [] 
 attrs_value = [] 
 ''''' 
 [\s] 匹配空格，[\s]*,后面有 *，则可以为空 
 * : 匹配前面的子表达式任意次 
 ''' 
 
 for attr in attrs: 
  attrs_name.append(re.search(r'(.*"html.parser") 
    attrs = soup.select('.attributes-list li') # 淘宝 class 
   else: 
    info = get_tianmao_header(url) 
 
    res = requests.get(url) 
    soup = BeautifulSoup(res.text, "html.parser") 
    attrs = soup.select('#J_AttrUL li') # 天猫 id 
    # print('attrs=',attrs) 
 
   d = get_Details(attrs,info) 
   print(d) 
   # for j in f[shop_id]: 
   #  d['店铺ID'] = j 
   # for s in f['shop']: 
   #  d['店铺名'] = s 
   #'Shop':d['店铺名'],'Data_id':d['ID'], 
   writer.writerow({'Link':url,'Brand':d['品牌'],'Title':d['宝贝'], 'Price':d['原价'], 'Sale price':d['促销价'], 'Sales':d['月销量'], 'Evaluations':d['累计评价'], 
        'Component':d['材质成分'], 'Fashion elements':d['流行元素'], 'Sleeve':d['袖长'], 'Seasons':d['年份季节'], 'Sales channels':d['销售渠道类型'], 
        'Number':d['货号'],'Clothes_Style':d['服装版型'],'Long':d['衣长'],'Collar type':d['领型'], 'Sleeve type':d['袖型'], 
        'Skirt type':d['裙型'], 'Skirt length':d['裙长'], 'Waist':d['腰型'], 'Combining form':d['组合形式'], 'Outseam':d['裤长'], 
        'Design':d['图案'], 'Fashion pattern detail':d['服装款式细节'], 'Applicable age':d['适用年龄'], 
        'Style':d['风格'], 'Commuter':d['通勤'], 'color':d['主要颜色'], 'Size':d['尺码']}) 
 
if __name__=='__main__': 
 main()