python协程异步爬取王者荣耀所有英雄皮肤

模块导入

# 导入模块
import os
import aiohttp
import asyncio
import aiofile

初始化参数

# 初始化参数
    def __init__(self):
    	# 英雄列表获取
        self.hero_url = 'https://pvp.qq.com/web201605/js/herolist.json'
        # 英雄详情页获取
        self.img_url = 'https://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/{}/{}-bigskin-{}.jpg'
        # 定义请求头
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
        }

启动函数

    async def main(self):
    	# 创建任务列表
        tasks = list()
        # 创建连接
        async with aiohttp.ClientSession() as session:
            async with session.get(self.hero_url, headers=self.headers) as response:
            	# 获得英雄列表json，内含英雄姓名和ID
                result = await response.json(content_type=None)

                for i in result:
                    hero_name = i['cname']
                    hero_id = i['ename']
                    # 创建获取英雄图片任务
                    coro_obj = self.get_hero_img(session, hero_name, hero_id)
                    # 添加至任务列表
                    tasks.append(asyncio.create_task(coro_obj))
            # 执行任务
            await asyncio.wait(tasks)

异步获取英雄图片

    async def get_hero_img(self, session, hero_name, hero_id):
    	# 创建英雄名称的文件夹
        if not os.path.exists('王者荣耀图片/{}'.format(hero_name)):
            os.mkdir('王者荣耀图片/{}'.format(hero_name))
        for sink_id in range(1, 10):
            async with session.get(self.img_url.format(hero_id, hero_id, sink_id), headers=self.headers) as response:
                if response.status == 200:
                    img_data = await response.read()
                    img_name = '{}_{}.jpg'.format(hero_name, sink_id)
                    async with aiofile.async_open('王者荣耀图片/{}/{}'.format(hero_name, img_name), 'wb') as f:
                        await f.write(img_data)
                        print('正在保存:{},{}'.format(hero_name, sink_id))
                else:
                    break

主进程

if __name__ == '__main__':
    if not os.path.exists('王者荣耀图片'):
        os.mkdir('王者荣耀图片')
    spider = WzrySpider()
    # 创建事件循环
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    # 启动任务
    loop.run_until_complete(spider.main())