python爬虫基础实战案例

文章目录1 实战一爬取豆瓣Top250图书信息目标分析完整代码2 实战二批量下载图片关键技巧多线程加速下载3 实战三爬取表格数据并存入Excel安装依赖爬取天气历史数据4 完整项目知乎热榜数据采集下面通过几个经典案例把前面的知识串联起来。包含三类实战项目静态页面爬取、图片批量下载、表格数据提取与存储。每个案例都有完整代码可以直接运行。1 实战一爬取豆瓣Top250图书信息豆瓣图书是学习爬虫的经典案例页面结构清晰适合练习CSS选择器和数据存储。目标分析目标爬取豆瓣Top250图书的书名、作者、评分、出版信息、评价人数保存为CSV文件。打开F12分析页面结构每本书在div.item容器中书名在span.title中评分在span.rating_num中作者和出版信息在p标签中翻页链接在a标签URL参数为?start0,25,50...完整代码importrequestsfrombs4importBeautifulSoupimportcsvimporttimeimportrandomdefget_headers():返回伪装请求头return{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36,Accept-Language:zh-CN,zh;q0.9}defparse_book_list(html):解析单页图书列表soupBeautifulSoup(html,lxml)books[]foriteminsoup.select(div.item):# 书名有时会有副标题title_tagsitem.select(span.title)titletitle_tags[0].text.strip()iftitle_tagselse未知# 评分rating_tagitem.select_one(span.rating_num)ratingrating_tag.text.strip()ifrating_tagelse0# 评价人数inq_tagitem.select_one(span.inq)inqinq_tag.text.strip()ifinq_tagelse# 作者/出版信息info_tagitem.select_one(div.bd p)infoinfo_tag.text.strip().replace(\n, )ifinfo_tagelse# 排名rank_tagitem.select_one(em)rankrank_tag.textifrank_tagelsebooks.append({rank:rank,title:title,rating:rating,inq:inq,info:info})returnbooksdefcrawl_douban_books():爬取豆瓣图书Top250all_books[]base_urlhttps://book.douban.com/top250forstartinrange(0,250,25):urlf{base_url}?start{start}print(f爬取第{start//251}页:{url})try:responserequests.get(url,headersget_headers(),timeout10)response.raise_for_status()booksparse_book_list(response.text)all_books.extend(books)print(f本页提取{len(books)}本书)exceptExceptionase:print(f爬取失败:{e})# 随机延时 1-3 秒避免被封time.sleep(random.uniform(1,3))returnall_booksdefsave_to_csv(books,filenamedouban_books.csv):保存为CSV文件ifnotbooks:print(没有数据可保存)returnwithopen(filename,w,encodingutf-8-sig,newline)asf:fieldnames[rank,title,rating,inq,info]writercsv.DictWriter(f,fieldnamesfieldnames)writer.writeheader()writer.writerows(books)print(f已保存{len(books)}条数据至{filename})if__name____main__:bookscrawl_douban_books()save_to_csv(books)# 打印前5条预览forbookinbooks[:5]:print(f[{book[rank]}]{book[title]}-{book[rating]}分)2 实战二批量下载图片批量下载图片是爬虫的常见需求。这个案例演示如何下载图片并保存到本地。关键技巧图片是二进制数据需要用response.content获取而不是response.text。importrequestsimportosimporttimeimportrandomfrompathlibimportPathfromurllib.parseimporturlparsedefdownload_image(url,save_path,headersNone):下载单张图片try:responserequests.get(url,headersheaders,timeout15,streamTrue)response.raise_for_status()# 检查是否是图片类型content_typeresponse.headers.get(Content-Type,)ifimagenotincontent_type:print(f不是图片类型:{content_type})returnFalse# 从URL中获取文件名parsed_urlurlparse(url)filenameos.path.basename(parsed_url.path)ifnotfilenameor.notinfilename:filenamefimage_{int(time.time())}.jpg# 保存图片full_pathos.path.join(save_path,filename)withopen(full_path,wb)asf:forchunkinresponse.iter_content(chunk_size8192):f.write(chunk)file_sizeos.path.getsize(full_path)print(f下载成功:{filename}({file_size/1024:.1f}KB))returnTrueexceptExceptionase:print(f下载失败{url}:{e})returnFalsedefbatch_download_images(image_urls,save_dirimages):批量下载图片# 创建保存目录Path(save_dir).mkdir(parentsTrue,exist_okTrue)headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64),Referer:https://example.com# 防盗链处理}success_count0fail_count0fori,urlinenumerate(image_urls,1):print(f[{i}/{len(image_urls)}] 下载:{url[:60]}...)ifdownload_image(url,save_dir,headers):success_count1else:fail_count1# 下载间隔time.sleep(random.uniform(0.5,1.5))print(f\n下载完成: 成功{success_count}张失败{fail_count}张)# 使用示例先爬取图片URL列表再批量下载defget_image_urls_from_page(page_url):从页面中提取图片URLheaders{User-Agent:Mozilla/5.0 ...}responserequests.get(page_url,headersheaders)soupBeautifulSoup(response.text,lxml)image_urls[]forimginsoup.select(div.gallery img):srcimg.get(src)orimg.get(data-src,)ifsrcandsrc.startswith(http):image_urls.append(src)returnimage_urls# 主流程# page_url https://example.com/gallery# urls get_image_urls_from_page(page_url)# batch_download_images(urls, save_dirdownloaded_images)多线程加速下载当图片数量多时单线程速度太慢。可以用多线程并发下载。fromconcurrent.futuresimportThreadPoolExecutor,as_completeddefbatch_download_with_threads(image_urls,save_dirimages,max_workers5):多线程批量下载Path(save_dir).mkdir(parentsTrue,exist_okTrue)headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64)}defdownload_task(args):idx,urlargsreturndownload_image(url,save_dir,headers)taskslist(enumerate(image_urls,1))success0withThreadPoolExecutor(max_workersmax_workers)asexecutor:future_to_url{executor.submit(download_task,task):taskfortaskintasks}forfutureinas_completed(future_to_url):idx,urlfuture_to_url[future]try:iffuture.result():success1exceptExceptionase:print(f任务异常:{e})print(f多线程下载完成:{success}/{len(image_urls)})3 实战三爬取表格数据并存入Excel很多网站有表格形式的数据比如排行榜、统计数据等。这个案例演示如何完整爬取这类数据。安装依赖pipinstallopenpyxl pandas爬取天气历史数据importrequestsfrombs4importBeautifulSoupimportpandasaspdfromdatetimeimportdatetimedefcrawl_table_data(url,table_selectortable): 通用表格数据爬取函数 headers{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36}responserequests.get(url,headersheaders,timeout10)response.encodingutf-8soupBeautifulSoup(response.text,lxml)tablesoup.select_one(table_selector)ifnottable:print(f未找到表格:{table_selector})returnNone# 提取表头headers_rowtable.select(tr:first-child th)ifnotheaders_row:headers_rowtable.select(thead tr th)columns[th.text.strip()forthinheaders_row]# 提取数据行rows[]fortrintable.select(tbody tr):cellstr.select(td)ifcells:row_data[cell.text.strip()forcellincells]rows.append(row_data)ifnotcolumnsornotrows:print(表格数据为空)returnNone# 转换为DataFramedfpd.DataFrame(rows,columnscolumns)returndfdefsave_to_excel(df,filenametable_data.xlsx,sheet_name数据):保存到Excel支持格式美化withpd.ExcelWriter(filename,engineopenpyxl)aswriter:df.to_excel(writer,sheet_namesheet_name,indexFalse)# 获取worksheet对象进行格式调整wswriter.sheets[sheet_name]# 自动调整列宽forcolumninws.columns:max_length0col_lettercolumn[0].column_letterforcellincolumn:ifcell.value:max_lengthmax(max_length,len(str(cell.value)))ws.column_dimensions[col_letter].widthmin(max_length4,30)print(f已保存到{filename})# 多页表格数据爬取defcrawl_multi_page_table(base_url,total_pages,page_parampage):多页表格数据爬取all_data[]forpageinrange(1,total_pages1):urlf{base_url}?{page_param}{page}print(f爬取第{page}页...)dfcrawl_table_data(url)ifdfisnotNone:df[来源页码]page all_data.append(df)importtime time.sleep(1)ifall_data:resultpd.concat(all_data,ignore_indexTrue)print(f共爬取{len(result)}条数据)returnresultreturnNone4 完整项目知乎热榜数据采集importrequestsimportjsonimportcsvfromdatetimeimportdatetimedefcrawl_zhihu_hot(): 爬取知乎热榜接口版本比解析HTML更稳定 # 知乎热榜接口api_urlhttps://www.zhihu.com/api/v3/feed/topstory/hot-lists/totalheaders{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36,Referer:https://www.zhihu.com/hot,Accept:application/json}params{limit:50,desktop:true}responserequests.get(api_url,headersheaders,paramsparams,timeout10)ifresponse.status_code!200:print(f请求失败:{response.status_code})return[]dataresponse.json()hot_listdata.get(data,[])results[]forrank,iteminenumerate(hot_list,1):targetitem.get(target,{})results.append({排名:rank,标题:target.get(title,),热度:item.get(detail_text,),回答数:target.get(answer_count,0),关注数:target.get(follower_count,0),链接:fhttps://www.zhihu.com/question/{target.get(id,)},采集时间:datetime.now().strftime(%Y-%m-%d %H:%M)})returnresultsdefsave_hot_list(data,filenameNone):ifnotfilename:filenamefzhihu_hot_{datetime.now().strftime(%Y%m%d_%H%M)}.csvwithopen(filename,w,encodingutf-8-sig,newline)asf:writercsv.DictWriter(f,fieldnamesdata[0].keys())writer.writeheader()writer.writerows(data)print(f已保存{len(data)}条热榜数据至{filename})if__name____main__:hot_datacrawl_zhihu_hot()ifhot_data:save_hot_list(hot_data)print(f\n当前热榜前10:)foriteminhot_data[:10]:print(f[{item[排名]}]{item[标题]}-{item[热度]})以上三个案例覆盖了静态爬虫的核心场景。注意在实际使用时需要添加适当的请求延时遵守网站robots.txt规则不要爬取隐私数据。下一章进入爬虫的进阶领域动态网页爬取。