如何编写爬取豌豆荚游戏排行榜的源码爬虫？

本文共计428个文字，预计阅读时间需要2分钟。

pythonimport requestsfrom bs4 import BeautifulSoup

def crawl_bilibili_games(): headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' } url='https://www.bilibili.com/game/gamecenter/' response=requests.get(url, headers=headers) soup=BeautifulSoup(response.text, '.parser')

games_list=[] for game in soup.select('.game-item'): name=game.select_one('.name').text.strip() download_link=game.select_one('.download-link')['href'] img_url=game.select_one('.cover img')['data-src'] games_list.append({ 'name': name, 'download_link': download_link, 'img_url': img_url })

return games_list

获取B站游戏列表games=crawl_bilibili_games()for game in games: print(f'游戏名称: {game[name]}') print(f'下载链接: {game[download_link]}') print(f'图片链接: {game[img_url]}') print('-' * 20)

看到有人问豌豆荚游戏榜单的爬虫如何写，这里特别贴一下完整的代码。
这里只爬了豌豆荚应用的名称，下载量和图标地址，有其他需求的朋友可以自己写xpath进行提取。
运行方法：复制代码到神箭手云爬虫平台上运行即可。

1.[代码][JavaScript]代码

var configs = { domains: ["apps.wandoujia.com"], scanUrls: ["apps.wandoujia.com/api/v1/apps?type=weeklytopgame&max=12&start=0"], contentUrlRegexes: ["www\\.wandoujia\\.com/apps/.*"], helperUrlRegexes: ["apps\\.wandoujia\\.com/api/v1/apps\\?type=weeklytopgame&max=12&start=\\d+"],//可留空 fields: [ { // 第一个抽取项 name: "title", selector: "//span[contains(@class,'title')]", required: true //是否不能为空 }, { // 第二个抽取项 name: "download", selector: "//i[@itemprop='interactionCount']", required: false //是否不能为空 }, { //第三个抽取项 name:"thumb", selector:"//div[contains(@class,'app-icon')]/img[@itemprop='image']/@src", } ] }; configs.onProcessHelperUrl = function(url, content, site) { var jarr = JSON.parse(content); //发现内容页 for (var i = 0, n = jarr.length; i < n; i++) { var new_url = "www.wandoujia.com/apps/"+jarr[i].packageName; site.addUrl(new_url); } var currentStart = parseInt(url.substring(url.indexOf("&start=") + 7)); var start = currentStart+12; if(start < 100){ site.addUrl("apps.wandoujia.com/api/v1/apps?type=weeklytopgame&max=12&start="+start); } return true; } var crawler = new Crawler(configs); crawler.start();