python - seleium 爬网页数据，只能怕当前页，如果我输入两页的话，会出现初始页数据下载两次的情况

问题描述

import requestsfrom lxml import html,etreefrom selenium import webdriverimport time, json#how many page do you want to scanpage_numnotint = input('how many page do you want to scan')page_num = int(page_numnotint)file_name = ’jd_goods_data.json’url = ’https://list.jd.com/list.html?cat=1713,3264,3414&page=1&delivery=1&sort=sort_totalsales15_desc&trans=1&JL=4_10_0#J_main ’driver = webdriver.Chrome()driver.get(url)base_html = driver.page_sourceselctor = etree.HTML(base_html)date_info = []name_data, price_data = [], []jd_goods_data = {}for q in range(page_num): i = int(1) while True:name_string = ’//*[@id='plist']/ul/li[%d]/p/p[3]/a/em/text()’ %(i)price_string = ’//*[@id='plist']/ul/li[%d]/p/p[2]/strong[1]/i/text()’ %(i)if i == 60: breakelse: i += 1name = selctor.xpath(name_string)[0]name_data.append(name)price = selctor.xpath(price_string)[0]price_data.append(price)jd_goods_data[name] = priceprint(name_data)with open(file_name, ’w’) as f: json.dump(jd_goods_data, f) time.sleep(2) driver.find_element_by_xpath(’//*[@id='J_bottomPage']/span[1]/a[10]’).click() time.sleep(2)# for k, v in jd_goods_data.items(): # print(k,v) # with open(file_name, ’w’) as f: # json.dump(jd_goods_data, f)

问题解答

import requestsfrom lxml import html,etreefrom selenium import webdriverimport time, json#how many page do you want to scanpage_numnotint = input('how many page do you want to scan')page_num = int(page_numnotint)file_name = ’jd_goods_data.json’driver = webdriver.Chrome()date_info = []name_data, price_data = [], []jd_goods_data = {}for q in range(page_num): url = ’https://list.jd.com/list.html?cat=1713,3264,3414&page={page}&delivery=1&sort=sort_totalsales15_desc&trans=1&JL=4_10_0#J_main’.format(page=q) driver.get(url) base_html = driver.page_source selctor = etree.HTML(base_html) i = 1 while True:name_string = ’//*[@id='plist']/ul/li[%d]/p/p[3]/a/em/text()’ %(i)price_string = ’//*[@id='plist']/ul/li[%d]/p/p[2]/strong[1]/i/text()’ %(i)if i == 60: breakelse: i += 1name = selctor.xpath(name_string)[0]name_data.append(name)price = selctor.xpath(price_string)[0]price_data.append(price)jd_goods_data[name] = priceprint(name_data)with open(file_name, ’w’) as f: json.dump(jd_goods_data, f)driver.quit()