文章详情页

python基于scrapy爬取京东笔记本电脑数据并进行简单处理和分析

更多 QQ空间微信 QQ好友腾讯朋友复制链接

【字号：大中小】日期：2022-06-22 14:32:32浏览：13作者：猪猪

一、环境准备 python3.8.3 pycharm 项目所需第三方包

pip install scrapy fake-useragent requests selenium virtualenv -i https://pypi.douban.com/simple

1.1 创建虚拟环境

切换到指定目录创建

virtualenv .venv

创建完记得激活虚拟环境

1.2 创建项目

scrapy startproject 项目名称

1.3 使用pycharm打开项目，将创建的虚拟环境配置到项目中来1.4 创建京东spider

scrapy genspider 爬虫名称 url

1.5 修改允许访问的域名，删除https：

二、问题分析

爬取数据的思路是先获取首页的基本信息，在获取详情页商品详细信息；爬取京东数据时，只返回40条数据，这里，作者使用selenium，在scrapy框架中编写下载器中间件，返回页面所有数据。爬取的字段分别是：

商品价格

商品评数

商品店家

商品SKU（京东可直接搜索到对应的产品）

商品标题

商品详细信息

三、spider

import reimport scrapyfrom lianjia.items import jd_detailItemclass JiComputerDetailSpider(scrapy.Spider): name = ’ji_computer_detail’ allowed_domains = [’search.jd.com’, ’item.jd.com’] start_urls = [’https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=1.def.0.base&wq=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&page=1&s=1&click=0’] def parse(self, response):lls = response.xpath(’//ul[@class='gl-warp clearfix']/li’)for ll in lls: item = jd_detailItem() computer_price = ll.xpath(’.//div[@class='p-price']/strong/i/text()’).extract_first() computer_commit = ll.xpath(’.//div[@class='p-commit']/strong/a/text()’).extract_first() computer_p_shop = ll.xpath(’.//div[@class='p-shop']/span/a/text()’).extract_first() item[’computer_price’] = computer_price item[’computer_commit’] = computer_commit item[’computer_p_shop’] = computer_p_shop meta = {’item’: item } shop_detail_url = ll.xpath(’.//div[@class='p-img']/a/@href’).extract_first() shop_detail_url = ’https:’ + shop_detail_url yield scrapy.Request(url=shop_detail_url, callback=self.detail_parse, meta=meta)for i in range(2, 200, 2): next_page_url = f’https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=1.def.0.base&wq=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&page={i}&s=116&click=0’ yield scrapy.Request(url=next_page_url, callback=self.parse) def detail_parse(self, response):item = response.meta.get(’item’)computer_sku = response.xpath(’//a[@class='notice J-notify-sale']/@data-sku’).extract_first()item[’computer_sku’] = computer_skucomputer_title = response.xpath(’//div[@class='sku-name']/text()’).extract_first().strip()computer_title = ’’.join(re.findall(’S’, computer_title))item[’computer_title’] = computer_titlecomputer_detail = response.xpath(’string(//ul[@class='parameter2 p-parameter-list'])’).extract_first().strip()computer_detail = ’’.join(re.findall(’S’, computer_detail))item[’computer_detail’] = computer_detailyield item四、item

class jd_detailItem(scrapy.Item): # define the fields for your item here like: computer_sku = scrapy.Field() computer_price = scrapy.Field() computer_title = scrapy.Field() computer_commit = scrapy.Field() computer_p_shop = scrapy.Field() computer_detail = scrapy.Field()五、setting

import randomfrom fake_useragent import UserAgentua = UserAgent()USER_AGENT = ua.randomROBOTSTXT_OBEY = FalseDOWNLOAD_DELAY = random.uniform(0.5, 1)DOWNLOADER_MIDDLEWARES = { ’lianjia.middlewares.jdDownloaderMiddleware’: 543}ITEM_PIPELINES = { ’lianjia.pipelines.jd_csv_Pipeline’: 300}六、pipelines

class jd_csv_Pipeline: # def process_item(self, item, spider): # return item def open_spider(self, spider):self.fp = open(’./jd_computer_message.xlsx’, mode=’w+’, encoding=’utf-8’)self.fp.write(’computer_skutcomputer_titletcomputer_p_shoptcomputer_pricetcomputer_committcomputer_detailn’) def process_item(self, item, spider):# 写入文件try: line = ’t’.join(list(item.values())) + ’n’ self.fp.write(line) return itemexcept: pass def close_spider(self, spider):# 关闭文件self.fp.close()七、middlewares

class jdDownloaderMiddleware: def process_request(self, request, spider):# 判断是否是ji_computer_detail的爬虫# 判断是否是首页if spider.name == ’ji_computer_detail’ and re.findall(f’.*(item.jd.com).*’, request.url) == []: options = ChromeOptions() options.add_argument('--headless') driver = webdriver.Chrome(options=options) driver.get(request.url) for i in range(0, 15000, 5000):driver.execute_script(f’window.scrollTo(0, {i})’)time.sleep(0.5) body = driver.page_source.encode() time.sleep(1) return HtmlResponse(url=request.url, body=body, request=request)return None八、使用jupyter进行简单的处理和分析

其他文件：百度停用词库、简体字文件下载第三方包

!pip install seaborn jieba wordcloud PIL -i https://pypi.douban.com/simple

8.1导入第三方包

import reimport osimport jiebaimport wordcloudimport pandas as pdimport numpy as npfrom PIL import Imageimport seaborn as snsfrom docx import Documentfrom docx.shared import Inchesimport matplotlib.pyplot as pltfrom pandas import DataFrame,Series

8.2设置可视化的默认字体和seaborn的样式

sns.set_style(’darkgrid’)plt.rcParams[’font.sans-serif’] = [’SimHei’]plt.rcParams[’axes.unicode_minus’] = False

8.3读取数据

df_jp = pd.read_excel(’./jd_shop.xlsx’)

8.4筛选Inteli5、i7、i9处理器数据

def convert_one(s): if re.findall(f’.*?(i5).*’, str(s)) != []:return re.findall(f’.*?(i5).*’, str(s))[0] elif re.findall(f’.*?(i7).*’, str(s)) != []:return re.findall(f’.*?(i7).*’, str(s))[0] elif re.findall(f’.*?(i9).*’, str(s)) != []:return re.findall(f’.*?(i9).*’, str(s))[0]df_jp[’computer_intel’] = df_jp[’computer_detail’].map(convert_one)

8.5筛选笔记本电脑的屏幕尺寸范围

def convert_two(s): if re.findall(f’.*?(d+.d+英寸-d+.d+英寸).*’, str(s)) != []:return re.findall(f’.*?(d+.d+英寸-d+.d+英寸).*’, str(s))[0]df_jp[’computer_in’] = df_jp[’computer_detail’].map(convert_two)

8.6将评论数转化为整形

def convert_three(s): if re.findall(f’(d+)万+’, str(s)) != []:number = int(re.findall(f’(d+)万+’, str(s))[0]) * 10000return number elif re.findall(f’(d+)+’, str(s)) != []:number = re.findall(f’(d+)+’, str(s))[0]return numberdf_jp[’computer_commit’] = df_jp[’computer_commit’].map(convert_three)

8.7筛选出需要分析的品牌

def find_computer(name, s): sr = re.findall(f’.*({name}).*’, str(s))[0] return srdef convert(s): if re.findall(f’.*(联想).*’, str(s)) != []:return find_computer(’联想’, s) elif re.findall(f’.*(惠普).*’, str(s)) != []:return find_computer(’惠普’, s) elif re.findall(f’.*(华为).*’, str(s)) != []:return find_computer(’华为’, s) elif re.findall(f’.*(戴尔).*’, str(s)) != []:return find_computer(’戴尔’, s) elif re.findall(f’.*(华硕).*’, str(s)) != []:return find_computer(’华硕’, s) elif re.findall(f’.*(小米).*’, str(s)) != []:return find_computer(’小米’, s) elif re.findall(f’.*(荣耀).*’, str(s)) != []:return find_computer(’荣耀’, s) elif re.findall(f’.*(神舟).*’, str(s)) != []:return find_computer(’神舟’, s) elif re.findall(f’.*(外星人).*’, str(s)) != []:return find_computer(’外星人’, s)df_jp[’computer_p_shop’] = df_jp[’computer_p_shop’].map(convert)

8.8删除指定字段为空值的数据

for n in [’computer_price’, ’computer_commit’, ’computer_p_shop’, ’computer_sku’, ’computer_detail’, ’computer_intel’, ’computer_in’]: index_ls = df_jp[df_jp[[n]].isnull().any(axis=1)==True].index df_jp.drop(index=index_ls, inplace=True)

8.9查看各品牌的平均价格

plt.figure(figsize=(10, 8), dpi=100)ax = sns.barplot(x=’computer_p_shop’, y=’computer_price’, data=df_jp.groupby(by=’computer_p_shop’)[[’computer_price’]].mean().reset_index())for index,row in df_jp.groupby(by=’computer_p_shop’)[[’computer_price’]].mean().reset_index().iterrows(): ax.text(row.name,row[’computer_price’] + 2,round(row[’computer_price’],2),color='black',ha='center')ax.set_xlabel(’品牌’)ax.set_ylabel(’平均价格’)ax.set_title(’各品牌平均价格’)boxplot_fig = ax.get_figure()boxplot_fig.savefig(’各品牌平均价格.png’, dpi=400)