使用scrapy+scrapy-splash爬取网易云音乐歌单

1
class Music163Spider(scrapy.Spider):
2

3
    name = "music163"
4
    allowed_domains = ["163.com"]
5
    start_urls = [
6
        "https://music.163.com/discover/playlist/?order=new",
7
        "https://music.163.com/#/discover/playlist/"
8

9
    ]
10
    def parse(self, response):
11
        for url in self.start_urls:
12
            yield SplashRequest(url=url, callback=self.parseRequest,endpoint='render.json', args={'wait': 1,'html':1,'iframes': 1,'script':1})
13

14
    def parseRequest(self,response):
15
        iframe_html = response.data['childFrames'][0]['html']
16
        sel = parsel.Selector(iframe_html)
17
        # for se in sel.xpath('//div[@class="n-rcmd"]/div[@class="v-hd2"]/div[@class="tab"]/a'):
18
        for se in sel.xpath("//div[@id='cateListBox']/div[@class='bd']//a")[1:]:
19
            item = Scrapymusic163Item()
20
            item['title'] = se.xpath('text()')[0].extract()
21
            item['link'] = se.xpath('@href')[0].extract()
22
            url = response.urljoin(item['link'])
23
            yield SplashRequest(url,self.parse_gedan,endpoint='render.json', args={'wait': 2,'html':1,'iframes': 1,'script':1})
24
    def parse_gedan(self, response):
25
        iframe_html = response.data['childFrames'][0]['html']
26
        sel = parsel.Selector(iframe_html)
27
        for se in sel.xpath('//ul[@id="m-pl-container"]/li'):
28
            item = Scrapymusic163Item()
29
            item['title'] = se.xpath('p[@class="dec"]/a/text()')[0].extract()
30
            item['link'] = se.xpath('p[@class="dec"]/a/@href')[0].extract()
31
            url=response.urljoin(item['link'])
32
            item['realLink']=url
33
            item['listenCount']=se.xpath('div/div/span[@class="nb"]/text()')[0].extract()
34
            if(item['listenCount'] and "万" in item['listenCount']):
35
                item['listenCount']=int(item['listenCount'].replace("万","0000"))
36
            item['listenCount']=int(item['listenCount'])
37
            item['imgUrl']=se.xpath('div/img[@class="j-flag"]/@src')[0].extract()
38
            yield item
39
        next_pages = sel.xpath('//a[@class="zbtn znxt"]')
40

41
        if next_pages:
42
            url = response.urljoin(next_pages.xpath('@href')[0].extract())
43
            yield SplashRequest(url,self.parse_gedan,endpoint='render.json', args={'wait': 2,'html':1,'iframes': 1,'script':1})
44
if __name__ == '__main__':
45
    pass

1
/* 1 */
2
{
3
    "_id" : ObjectId("5aea7a036d5b76bf8d42659d"),
4
    "title" : "华语速爆新歌",
5
    "link" : "/playlist?id=924680166",
6
    "realLink" : "https://music.163.com/playlist?id=924680166",
7
    "listenCount" : 152900000,
8
    "imgUrl" : "http://p1.music.126.net/Zcv4NgK5T0bd1yMzAFp8Iw==/109951163276364917.jpg?param=140y140"
9
}
10

11
/* 2 */
12
{
13
    "_id" : ObjectId("5aea78a56d5b76bf8d422179"),
14
    "title" : "【节奏控】那些超带感的音乐（典藏版）",
15
    "link" : "/playlist?id=306397077",
16
    "realLink" : "https://music.163.com/playlist?id=306397077",
17
    "listenCount" : 140930000,
18
    "imgUrl" : "http://p1.music.126.net/RnOZHM0BNxXuy-RwQQI5BA==/3313928048221849.jpg?param=140y140"
19
}
20

21
/* 3 */
22
{
23
    "_id" : ObjectId("5aea7fd16d5b76bf8d43b0db"),
24
    "title" : "【旋律控】超级好听的外文歌",
25
    "link" : "/playlist?id=310970433",
26
    "realLink" : "https://music.163.com/playlist?id=310970433",
27
    "listenCount" : 101740000,
28
    "imgUrl" : "http://p1.music.126.net/2MsstS-M9w5-li0aRy3sUQ==/1380986606815861.jpg?param=140y140"
29
}
30

31
/* 4 */
32
{
33
    "_id" : ObjectId("5aea95f76d5b76bf8d46289a"),
34
    "title" : "那些只听前奏就中毒的英文歌",
35
    "link" : "/playlist?id=37432514",
36
    "realLink" : "https://music.163.com/playlist?id=37432514",
37
    "listenCount" : 95420000,
38
    "imgUrl" : "http://p1.music.126.net/mQy3lRj6YJ0TW3fM9v85YA==/6643249256145165.jpg?param=140y140"
39
}
40

41
/* 5 */
42
{
43
    "_id" : ObjectId("5aea789a6d5b76bf8d42194c"),
44
    "title" : "【欧美男团】秒杀耳朵系列",
45
    "link" : "/playlist?id=117377955",
46
    "realLink" : "https://music.163.com/playlist?id=117377955",
47
    "listenCount" : 71150000,
48
    "imgUrl" : "http://p1.music.126.net/FJAxNkFoq3dGiS9tz_bGgQ==/3405187512421439.jpg?param=140y140"
49
}