1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

import numpy as np
import pandas as pd

from bs4 import BeautifulSoup as bs
import urllib.request as ur
import urllib.parse as up
import urllib.error as ue
import http.cookiejar as hc

import re
import gzip
import json

import time
import os
import socket

os.chdir('...')
socket.setdefaulttimeout(30)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

headers={
    'Host': 'bangumi.tv',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
    'Sec-Fetch-Dest': 'document',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-User': '?1',
    'Accept-Encoding': 'gzip',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cookie': '...(此处略去)...__utmb=1.7.10.1593136256'
}

cj=hc.CookieJar()

hl=ur.HTTPCookieProcessor(cj)

opener=ur.build_opener(hl)


pattern=re.compile(r'li id="item_(\d+)"')

subjectlist=[]


def update_cookie(cookie=str()):
    return cookie[:cookie.rfind('utmb=')]+cookie[cookie.rfind('utmb='):].split('.')[0]+'.'+str(int(cookie[cookie.rfind('utmb='):].split('.')[1])+1)+'.'+cookie[cookie.rfind('utmb='):].split('.')[2]+'.'+cookie[cookie.rfind('utmb='):].split('.')[3]


def bangumidownload(start,end,subjectlist):
    for i in range(start,end+1):
        url='https://bangumi.tv/anime/browser?sort=rank&page=%d'%(i)
        print('downloading page',i,end='\r')
        
        try_time=0
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=headers)
                response=opener.open(r) 
                break
            except Exception as e:
                try_time+=1
                if try_time>1:
                    headers['Cookie']=update_cookie(headers['Cookie'])
                print('retrying with cookie=',headers['Cookie'][headers['Cookie'].rfind('utmb='):],try_time)
        else:
            raise Exception('Download Failed!!')
            
        content=str(gzip.decompress(response.read()),'utf-8')
        response.close()

        thispage=re.findall(pattern,content)
        
        subjectlist.extend(thispage)

bangumidownload(1,243,subjectlist)


date=str(time.localtime().tm_mon)+'_'+str(time.localtime().tm_mday)
with open(r'bangumi\subjectlist_'+date+'.json','w') as fp:
    json.dump(subjectlist,fp)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

bgmdb=[] 

picpattern=re.compile(r'href="//(.*?)"') 

def bangumifulldownload(bgmsubjects):
    for i in bgmsubjects:
        url='https://bangumi.tv/subject/'+i
        print('downloading subject '+i)
        
        try_time=0
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=headers)
                response=opener.open(r,timeout=30) 
                break
            except Exception as e:
                try_time+=1
                if try_time>2:
                    headers['Cookie']=update_cookie(headers['Cookie'])
                print('retrying with cookie=',headers['Cookie'][headers['Cookie'].rfind('utmb='):],try_time)
        else:
            raise Exception('Download Failed!!')
            
        content=str(gzip.decompress(response.read()),'utf-8')
        response.close()

        soup=bs(content)
        mainWrapper=soup.find('div',class_='mainWrapper')
        name=soup.find('h1',class_='nameSingle')
        if mainWrapper==None or name==None:
            continue
        
        infobox=mainWrapper.find('ul',id='infobox')
        if infobox==None:
            continue
        infodict=dict()
        infodict.update({'subject':i,'原名':name.find('a').text if name.find('a')!=None else '',
                         '类型':name.find('small').text if name.find('small')!=None else ''})
        
        summary=mainWrapper.find('div',id='subject_summary')
        if summary is not None:
            infodict.update({'简介':summary.text})

        pic=mainWrapper.find('a',class_='thickbox cover')
        if pic is not None:
            pic=re.findall(picpattern,str(pic))
            if len(pic):
                infodict.update({'封面':'https://'+pic[0]})
        
        info=infobox.find_all('li')
        for each_info in info:
            kv=each_info.text.split(':',maxsplit=1)
            infodict.update({kv[0].strip():kv[1].strip()})
        
        tagWrapper=mainWrapper.find('div',class_='inner')
        if tagWrapper==None:
            continue
        tagtext=tagWrapper.select('.l span,a small')
        tags=[]
        for everytag in tagtext:
            tags.append(everytag.text)
        tags=' '.join(tags)
        infodict.update({'tags':tags})
        
        chartWrapper=mainWrapper.find('div',id='ChartWarpper')
        infodict.update({'votes':chartWrapper.find('span',property='v:votes').text})
        
        rating_list=[]
        for each_rater in chartWrapper.find_all('span',{'class':'count'}):
            rating_list.append(each_rater.text[1:-1])
        infodict.update({'ratings':rating_list})
        
        
        overall_score=0
        overall_vote=0
        for score in range(10,0,-1):
            overall_vote+=int(infodict['ratings'][10-score])
            overall_score+=score*int(infodict['ratings'][10-score])
        overall_score=overall_score/overall_vote
        infodict.update({'rating':str('%.3f'%(overall_score))})
        
        print(infodict)
        bgmdb.append(infodict)

bangumifulldownload(subjectlist)

date=str(time.localtime().tm_mon)+'_'+str(time.localtime().tm_mday)
with open(r'bangumi\bgmdb_'+date+'.json','w') as fp:
    json.dump(bgmdb,fp)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

bgmfulldb=pd.read_json(r'bangumi\bgmdb_'+date+'.json')

indexs=bgmfulldb[~bgmfulldb.isna()].count().sort_values(ascending=False)[:60].index

bgmdb2=[]
for i in bgmdb:
    thisanime=[]
    
    for each_key in indexs:
        if each_key in i.keys():
            thisanime.append(i[each_key])
        else:
            thisanime.append('')
    
    bgmdb2.append(thisanime)

bgmdb2=pd.DataFrame(bgmdb2,columns=indexs)
bgmdb2.to_csv(r'bangumi\bgmdb_'+date+'.csv',index=False)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

import numpy as np
import pandas as pd

from bs4 import BeautifulSoup as bs
import urllib.request as ur
import urllib.parse as up
import urllib.error as ue
import http.cookiejar as hc

import re
import gzip
import json

import time
import os
import socket

os.chdir(r'...')
socket.setdefaulttimeout(30)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

apiheaders={
    'Host': 'api.bilibili.com',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'Accept':' application/json, text/plain, */*',
    'Sec-Fetch-Dest': 'empty',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
    'Origin': 'https://www.bilibili.com',
    'Sec-Fetch-Site': 'same-site',
    'Sec-Fetch-User': '?1',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Dest': 'document',
    'Referer': 'https://www.bilibili.com/anime/index/',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cookie':cookie
}

wwwheaders={
    'Host': 'www.bilibili.com',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
    'Sec-Fetch-Dest': 'document',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-User': '?1',
    'Referer': 'https://www.bilibili.com/anime/index/',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cookie':cookie
}

imageheaders={
    'Host': 'i0.hdslb.com',
    'Connection': 'keep-alive',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
    'Sec-Fetch-Dest': 'image',
    'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
    'Sec-Fetch-Site': 'cross-site',
    'Sec-Fetch-Mode': 'no-cors',
    'Referer': 'https://www.bilibili.com/bangumi/media/md1178/?from=search&seid=17806546061422186816',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9'
}

cj=hc.CookieJar()

hl=ur.HTTPCookieProcessor(cj)

opener=ur.build_opener(hl)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

sslist=list()

pattern=re.compile(r'https://www.bilibili.com/bangumi/play/ss\d+')
pattern2=re.compile(r'"title":"(.*?)"')

def ssdownload():
    for i in range(1,1000):
        url='https://api.bilibili.com/pgc/season/index/result?season_version=-1&area=-1&is_finish=-1&copyright=-1&season_status=-1&season_month=-1&year=-1&style_id=-1&order=5&st=1&sort=0&page='+str(i)+'&season_type=1&pagesize=20&type=1'
        print('downloading page='+str(i))
        
        try_time=0
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=headers)
                response=opener.open(r) 
                break
            except ue.HTTPError as e:
                print('Page Not found...skipped')
                break
            except Exception as e:
                try_time+=1
                print('retrying',try_time)
        else:
            raise Exception('Download Failed!!')
        
        
        try:
            content=str(gzip.decompress(response.read()),'utf-8')
        except Exception as e:
            break
            
        response.close()
        
        titles=re.findall(pattern2,content)
        ssurl=re.findall(pattern,content)
        
        
        for i in range(len(ssurl)):
            sslist.append({'title':titles[i],'ssurl':ssurl[i]}) 

def ssdownload2():
    for i in range(1,1000):
        url='https://api.bilibili.com/pgc/season/index/result?season_version=-1&is_finish=-1&copyright=-1&season_status=-1&year=-1&style_id=-1&order=5&st=4&sort=0&page='+str(i)+'&season_type=4&pagesize=20&type=1'
        print('downloading page='+str(i))
        
        try_time=0
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=headers)
                response=opener.open(r) 
                break
            except ue.HTTPError as e:
                print('Page Not found...skipped')
                break
            except Exception as e:
                try_time+=1
                print('retrying',try_time)
        else:
            raise Exception('Download Failed!!')
        
        
        try:
            content=str(gzip.decompress(response.read()),'utf-8')
        except Exception as e:
            break
            
        response.close()
        
        titles=re.findall(pattern2,content)
        ssurl=re.findall(pattern,content)
        
        
        for i in range(len(ssurl)):
            sslist.append({'title':titles[i],'ssurl':ssurl[i]}) 

ssdownload()
ssdownload2()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

mdpattern=re.compile('\d+')
mdlist=[]
summary_dict={}
def mddownload():
    for each_ssurl in sslist:
        print('downloading item='+str(sslist.index(each_ssurl)))
        
        try_time=0
        httperror=False
        while try_time<=5:
            try:
                r=ur.Request(url=each_ssurl['ssurl'],headers=wwwheaders)
                response=opener.open(r) 
                break
            except ue.HTTPError as e:
                httperror=True
                print('Page Not found...skipped')
                break
            except Exception as e:
                try_time+=1
                print('retrying',try_time)
        else:
            raise Exception('Download Failed!!')
        
        if httperror:
            continue
        
        content=str(gzip.decompress(response.read()),'utf-8')
        response.close()
        
        soup=bs(content)
        
        mdlist.append(re.search(mdpattern,str(soup.find('div', id='media_module').find('a'))).group())
        summary_dict.update({mdlist[-1]:soup.find('span',class_='absolute').text})

mddownload()
mdlist=sorted(mdlist)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

bilidb=list()
for i in mdlist:
    url='https://api.bilibili.com/pgc/review/user?media_id='+str(i)
    r=ur.Request(url=url,headers=apiheaders)
    response=opener.open(r)
    content=str(response.read(),'utf-8')
    mdjson=json.loads(content)
    
    if 'result' in mdjson.keys():
        if 'media' in mdjson['result'].keys():
            url='https://api.bilibili.com/pgc/web/season/stat?season_id='+str(mdjson['result']['media']['season_id'])
            r=ur.Request(url=url,headers=headers)
            response=opener.open(r)
            content=str(response.read(),'utf-8')
            mdjson2=json.loads(content)
            mdjson['result'].update(mdjson2['result'])
    
    bilidb.append(mdjson['result'] if 'result' in mdjson.keys() else dict())
    print(mdlist.index(i),end='\r')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

tp_finished=re.compile(r'(\d+)年(\d+)月(\d+)日开播.*(\d+)')
tp_not_finished=re.compile(r'(\d+)年(\d+)月(\d+)日开播')
tp_movie=re.compile(r'(\d+)年(\d+)月(\d+)日上映')
def download():
    for i in range(len(bilidb)):
        print(i,end='\r')
        
        if 'media' not in bilidb[i].keys():
            continue
        url=bilidb[i]['media']['share_url']
        
        try_time=0
        httperror=False
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=wwwheaders)
                response=opener.open(r) 
                break
            except ue.HTTPError as e:
                httperror=True
                print('Page Not found...skipped')
                break
            except Exception as e:
                try_time+=1
                print('retrying',try_time)
        else:
            raise Exception('Download Failed!!')
        
        if httperror:
            continue
        
        content=str(gzip.decompress(response.read()),'utf-8')
        response.close()
        
        soup=bs(content)
        tags=soup.find('div',class_='media-info-r').find_all('span',{'class':'media-tag'})
        tags=[each_tag.text for each_tag in tags]
        tags=' '.join(tags) if len(tags) else ''
        time=soup.find('div',class_='media-info-r').find_all('div',{'class':'media-info-time'})[0]
        
        res=re.search(tp_finished,time.text)
        finished=False
        if res is not None: 
            finished=True
            res=res.groups()
            date='-'.join(res[0:3])
            episodes=res[3]
        else:
            res=re.search(tp_not_finished,time.text)
            if res is not None: 
                res=res.groups()
                date='-'.join(res[0:3])
                episodes=0
            else:
                res=re.search(tp_movie,time.text)
                if res is not None: 
                    res=res.groups()
                    date='-'.join(res[0:3])
                    episodes=1
                else:
                    date=''
                    episodes=-1
        
        bilidb[i].update({'tags':tags,'date':date,'episodes':episodes})
        print(i,{'tags':tags,'date':date,'episodes':episodes})
download()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

bilidb2=[]
for i in range(len(bilidb)):
    a=dict()
    if not len(bilidb[i]):
        continue
    a.update({
            'title':bilidb[i]['media']['title'],
            'type_name':bilidb[i]['media']['type_name'],
            'season_id':bilidb[i]['media']['season_id'],
            'area':bilidb[i]['media']['areas'][0]['name'] if len(bilidb[i]['media']['areas'])>0 else '',
            'media_id':bilidb[i]['media']['media_id'],
            'rating':bilidb[i]['media']['rating']['score'] if 'rating' in bilidb[i]['media'].keys() else 0,
            'raters':bilidb[i]['media']['rating']['count'] if 'rating' in bilidb[i]['media'].keys() else 0,
            'cover':bilidb[i]['media']['cover'],
        
            'follow':bilidb[i]['follow'] if 'follow' in bilidb[i].keys() else 0,
            'series_follow':bilidb[i]['series_follow'] if 'series_follow' in bilidb[i].keys() else 0,
            'views':bilidb[i]['views'] if 'views' in bilidb[i].keys() else 0,
            'coins':bilidb[i]['coins'] if 'coins' in bilidb[i].keys() else 0,
            'danmakus':bilidb[i]['danmakus'] if 'danmakus' in bilidb[i].keys() else 0,
            'tags':bilidb[i]['tags'] if 'tags' in bilidb[i].keys() else '',
            'date':bilidb[i]['date'] if 'date' in bilidb[i].keys() else '',
            'episodes':bilidb[i]['episodes'] if 'episodes' in bilidb[i].keys() else 0
             })
    bilidb2.append(a)

for each_item in bilidb2:
    summary=summary_dict.get(str(each_item['media_id']))
    each_item.update({'简介':summary if summary is not None else ''})

date=str(time.localtime().tm_mon)+'_'+str(time.localtime().tm_mday)
with open(r'bilibili\sslist_'+date+'.json','w') as fp:
    json.dump(sslist,fp)
with open(r'bilibili\mdlist_'+date+'.json','w') as fp:
    json.dump(mdlist,fp)
with open(r'bilibili\bilidb_'+date+'.json','w') as fp:
    json.dump(bilidb2,fp)

bilidb3=pd.DataFrame(bilidb2)
bilidb3.to_csv(r'bilibili\bilidb_'+date+'.csv',index=False)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

def imgdl():
    for i in range(len(bilidb)):
        print(i,end='\r')
        url=bilidb.cover.values[i]
        md=bilidb.media_id.values[i]
        
        try_time=0
        httperror=False
        while try_time<=5:
            try:
                r=ur.Request(url=url,headers=headers)
                response=opener.open(r) 
                break
            except ue.HTTPError as e:
                httperror=True
                print('Page Not found...skipped')
                break
            except Exception as e:
                try_time+=1
                print('retrying',try_time)
        else:
            raise Exception('Download Failed!!')


        content=response.read()
        response.close()
        
        with open('bilibili\\covers\\' + str(md) + '.jpg','wb') as fp:
            fp.write(content)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

import numpy as np
import pandas as pd
from scipy import stats
import matplotlib.pyplot as plt
import time
import os
from bs4 import BeautifulSoup as bs
import urllib.request as ur
import urllib.parse as up
import urllib.error as ue
import http.cookiejar as hc

import re
import gzip

import json
%matplotlib inline
plt.rcParams['font.sans-serif']=['SimHei'] 
plt.rcParams['axes.unicode_minus']=False 
os.chdir(r'...')

1

bilidb.date

1
2
3
4
5
6
7
8
9
10
11
12

0        2013-1-3
1        2014-7-8
2        2014-7-2
3       2012-10-7
4        2013-7-7
          ...    
3703          NaN
3704          NaN
3705          NaN
3706          NaN
3707          NaN
Name: date, Length: 3708, dtype: object

1
2

bilidb.date=pd.to_datetime(bilidb.date,format='%Y-%m-%d',errors='coerce').copy()
bilidb.date

1
2
3
4
5
6
7
8
9
10
11
12

0      2013-01-03
1      2014-07-08
2      2014-07-02
3      2012-10-07
4      2013-07-07
          ...    
3703          NaT
3704          NaT
3705          NaT
3706          NaT
3707          NaT
Name: date, Length: 3708, dtype: datetime64[ns]

1
2
3

bilidb['year']=bilidb.date.apply(lambda x: x.year)
bilidb.year=bilidb.year.apply(lambda x: 0 if not pd.notna(x) else int(x))
bilidb.year

1
2
3
4
5
6
7
8
9
10
11
12

0       2013
1       2014
2       2014
3       2012
4       2013
        ... 
3703       0
3704       0
3705       0
3706       0
3707       0
Name: date, Length: 3708, dtype: int64

1
2
3
4
5
6
7
8
9
10
11
12
13
14

bgmdb['日期']=pd.to_datetime(bgmdb['放送开始'],format='%Y年%m月%d日',errors='coerce')
bgmdb['日期2']=pd.to_datetime(bgmdb['上映年度'],format='%Y年%m月%d日',errors='coerce')

for i in range(len(bgmdb['日期'])):
    if bgmdb['日期'][i] is pd.NaT:
        bgmdb.loc[i,'日期']=bgmdb.loc[i,'日期2']
bgmdb.drop('日期2',axis=1,inplace=True)

bgmdb['年度']=0
for i in range(len(bgmdb)):
    bgmdb.loc[i,'年度']=bgmdb.loc[i,'日期'].year if bgmdb.loc[i,'日期'] is not pd.NaT else 0
    
bgmdb[['放送开始','上映年度','日期','年度']].head(10)

1
2

bilidb.rating=bilidb.rating.apply(lambda x: np.nan if x==0 else x)
bilidb.raters=bilidb.raters.apply(lambda x: np.nan if x==0 else x)

1
2
3

(len(bilidb.rating),
 bilidb.rating.dropna().describe(),
 bilidb.rating[bilidb.rating==bilidb.rating.mode()[0]].count())

1
2
3
4
5
6
7
8
9
10
11

(3708,
 count    2581.00000
 mean        9.12592
 std         1.01527
 min         2.20000
 25%         9.10000
 50%         9.50000
 75%         9.70000
 max         9.90000
 Name: rating, dtype: float64,
 431)

1
2
3
4
5
6
7
8
9
10

fig=plt.figure(num=100,figsize=(6,4),dpi=200)
ax=fig.gca()

nx=np.arange(2,11,0.1)
ny=normfun(nx,bilidb.rating.dropna().mean(),bilidb.rating.dropna().std())
ax.plot(nx,ny)

ax.hist(x=bilidb.rating.dropna(),bins=20,color='yellow',edgecolor='black',density=True)
ax.set_title('bilibili评分频数分布直方图与正态分布曲线')
ax.text(2.5,0.8,'average=%.4f\nstd=%.4f'%(bilidb.rating.dropna().mean(),bilidb.rating.dropna().std()),fontsize=15)

1
2

(bilidb.raters.dropna().sum(),
 (bilidb.rating.dropna()*bilidb.raters.dropna()).sum()/bilidb.raters.sum())

1

(14881442.0, 8.987174381353636)

1

bgmdb.rating.describe()

1
2
3
4
5
6
7
8
9

count    5830.000000
mean        6.633856
std         0.880986
min         1.068000
25%         6.127250
50%         6.691500
75%         7.236750
max         9.143000
Name: rating, dtype: float64

1
2
3
4
5
6
7
8
9
10

fig=plt.figure(num=101,figsize=(6,4),dpi=200)
ax=fig.gca()

nx=np.arange(1,10,0.1)
ny=normfun(nx,bgmdb.rating.mean(),bgmdb.rating.std())
ax.plot(nx,ny)

ax.hist(x=bgmdb.rating,bins=32,color='yellow',edgecolor='black',density=True)
ax.set_title('bangumi评分频数分布直方图与正态分布曲线')
ax.text(2.1,0.4,'average=%.4f\nstd=%.4f'%(bgmdb.rating.mean(),bgmdb.rating.std()),fontsize=15)

1

bgmdb.votes.sum(),(bgmdb.rating*bgmdb.votes).sum()/bgmdb.votes.sum()

1

(4525600, 7.185151881518473)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

yr_prod=pd.crosstab(index=bgmdb['年度'],columns=bgmdb['动画制作'])

studios=bgmdb['动画制作'].dropna().unique()

studio_dict={}
for each_studio in studios:
    studio_dict.update({str(each_studio):
                        bgmdb['动画制作'].dropna()[
                            bgmdb['动画制作'].dropna().str.contains(each_studio)
                        ].count()})

studio_dict=pd.Series(studio_dict)

yr_prod.loc[:,studio_dict.sort_values(ascending=False).index.tolist()[:20]][15:]

1
2
3
4
5
6
7
8
9
10

fig=plt.figure(num=104,figsize=(6,4),dpi=200,facecolor='white')
ax=fig.gca()

y=studio_dict.sort_values(ascending=False).index.tolist()[:17]
x=studio_dict.sort_values(ascending=False).values.tolist()[:17]
ax.pie(x,labels=y,autopct='%.1f%%',pctdistance=0.5,labeldistance=1.1, \
        startangle=120,radius=1.2,counterclock=False,wedgeprops={'linewidth':1.5,'edgecolor':'green'}, \
       textprops={'fontsize':10,'color':'black'})
ax.set_title('动画制作分布(前%.0f%%作品)'%(
    100*studio_dict.sort_values(ascending=False)[:17].sum()/studio_dict.sum()),pad=30)

1
2
3
4
5
6
7
8

studios_ratings=pd.pivot_table(bgmdb,values='rating',index='动画制作',aggfunc=np.mean,margins=False,dropna=True)

studios_counts=pd.pivot_table(bgmdb,values='subject',index='动画制作',aggfunc=len,margins=False,dropna=True)

studios_counts_ratings=pd.merge(studios_ratings,studios_counts,on='动画制作').sort_values(
    by=['subject','rating'],ascending=False).head(30)
studios_counts_ratings

1

studios_counts_ratings.sort_values(by=['rating'],ascending=False).head(10)

1
2
3
4
5
6

a=bgmdb[bgmdb.['年度']==2000].sort_values(by='votes',ascending=False).loc[:,'中文名'][:5].values
a.resize(5,1)
for i in range(2001,2020):
    b=bgmdb[bgmdb.['年度']==i].sort_values(by='votes',ascending=False).loc[:,'中文名'][:5].values
    b.resize(5,1)
    a=np.concatenate([a,b],axis=1)

1
2
3
4
5
6
7

a=bgmdb[bgmdb['年度']==2000].sort_values(by='rating',ascending=False).loc[:,'中文名'][:5].values
a.resize(5,1)
for i in range(2001,2020):
    b=bgmdb[bgmdb['年度']==i].sort_values(by='rating',ascending=False).loc[:,'中文名'][:5].values
    b.resize(5,1)
    a=np.concatenate([a,b],axis=1)
pd.DataFrame(data=a,index=['评分第%d名'%i for i in range(1,6)],columns=range(2000,2020))

1
2
3
4
5
6
7
8
9
10
11
12
13
14

bgmdb_by_year=bgmdb.groupby(by='年度')
def count_rank(x):
    count=0
    for i in x:
        if i<100:
            count+=1
    return count
TOP100_by_year=bgmdb_by_year.bgmrank.apply(count_rank)[-21:-1]
print(TOP100_by_year)

fig = plt.figure(num=112,figsize=(6,4),dpi=200,facecolor='white')
ax = plt.gca() 
ax.plot(TOP100_by_year.index,TOP100_by_year.values)
ax.set(title='各年度TOP100进榜数量',xticks=range(2000,2019,2))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

年度
2000    0
2001    3
2002    3
2003    3
2004    4
2005    3
2006    7
2007    4
2008    4
2009    2
2010    3
2011    4
2012    2
2013    2
2014    7
2015    5
2016    1
2017    2
2018    1
2019    1

1
2
3
4
5
6
7

TOP10=pd.DataFrame(data=[
    bgmdb[bgmdb['年度']==i].sort_values(
    by='rating',ascending=False).loc[:,'rating'][:10].mean()
    for i in range(2000,2020)
    ],columns=['TOP10均分'],index=range(2000,2020))

TOP10

1
2
3
4

fig = plt.figure(num=111,figsize=(6,4),dpi=200,facecolor='white')
ax = plt.gca() 
ax.plot(TOP10.index,TOP10.values)
ax.set(title='TOP10均分走势',xticks=range(2000,2019,2))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

url='https://search.bilibili.com/all?'
pattern=re.compile(r'www.bilibili.com/bangumi/media/md(.*?)/')
interface=dict()

for i in range(0,len(bgmdb)):
    res=[]
    print('item',i,'searching',bgmdb['原名'][i])
    querystring={'keyword':bgmdb['原名'][i]}
    querystring=up.urlencode(querystring)
    final_url=url+querystring

    try_succeed=False
    while not try_succeed:
        try:
            try_succeed=True
            r=ur.Request(url=final_url,headers=headers)
            response=opener.open(r) 
        except Exception as e:
            try_succeed=False
            time.sleep(1)
            print('URL timeout! retrying...')

    content=str(gzip.decompress(response.read()),'utf-8')
    res.extend(re.findall(pattern,content))
    response.close()
    
    print('item',i,'searching',bgmdb['中文名'][i])
    querystring={'keyword':bgmdb['中文名'][i]}
    querystring=up.urlencode(querystring)
    final_url=url+querystring

    try_succeed=False
    while not try_succeed:
        try:
            try_succeed=True
            r=ur.Request(url=final_url,headers=headers)
            response=opener.open(r) 
        except Exception as e:
            try_succeed=False
            time.sleep(1)
            print('URL timeout! retrying...')
    
    content=str(gzip.decompress(response.read()),'utf-8')
    res.extend(re.findall(pattern,content))
    response.close()
    
    res=list(set(res))
    interface.update({bgmdb['subject'][i]:res})
    print({bgmdb['subject'][i]:res})
    
with open('bangumi\\interfaces\\origin-subject2md.json','w') as fp:
    json.dump(interface,fp)

1
2
3
4
5
6
7
8

{253: ['8023271', '3008', '5383'],
 326: ['1714', '1705', '1565'],
 324: ['1564', '1712', '1705'],
 876: ['1178', '1180'],
 265: ['10352', '10372', '10332', '1635'],
 237: ['1714', '1568', '1566', '28228268'],
 6049: ['10272'],
...}

1
2
3
4
5

interface2={}
for k,v in interface.items():
    if len(v)>0:
        v=[int(i) for i in v if len(i)>0]
        interface2.update({int(k):v})

1
2
3
4
5
6
7
8
9
10

interface3=list()
for k,v in interface2.items():
    bgm=bgmdb.loc[bgmdb.subject==k,['年度','subject','原名','中文名','别名','日期']].values[0]
    
    bili=[bilidb.loc[bilidb.media_id==each_bilimd,['year','media_id','title','date']].values for each_bilimd in v]
    
    bili=[each_bilimd[0] for each_bilimd in bili if len(each_bilimd)]
    interface3.append({'bgm':bgm,'bili':bili})

interface3

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

[{'bgm': array([1998, 253, 'カウボーイビバップ', '星际牛仔', '恶男杰特',
         Timestamp('1998-10-23 00:00:00')], dtype=object),
  'bili': [array([2001, 3008, '星际牛仔 天国之扉', Timestamp('2001-09-01 00:00:00')],
         dtype=object),
   array([1998, 5383, '星际牛仔 SP', Timestamp('1998-10-23 00:00:00')],
         dtype=object)]},
 {'bgm': array([2004, 326, '攻殻機動隊 S.A.C. 2nd GIG', '攻壳机动队 S.A.C. 2nd GIG', nan,
         Timestamp('2004-01-01 00:00:00')], dtype=object),
  'bili': [array([2006, 1714, '攻壳机动队 个别的十一人', Timestamp('2006-01-27 00:00:00')],
         dtype=object),
   array([2006, 1705, '攻壳机动队 S.A.C. Solid State Society',
          Timestamp('2006-09-01 00:00:00')], dtype=object),
   array([2004, 1565, '攻壳机动队 S.A.C. 2nd GIG',
          Timestamp('2004-01-01 00:00:00')], dtype=object)]},
...]

1
2
3
4
5
6
7
8
9
10
11

matched=[]
for searchres in interface3:
    bgm=searchres['bgm']
    bili=searchres['bili']
    for bilimd in bili:
        for bgmname in bgm[2:5]:
            if bilimd[2]==bgmname and bilimd[0]==bgm[0]:
                matched.append({'subject':bgm[1],'md':bilimd[1]})

import copy
fullymatched=copy.deepcopy(matched)

1
2
3
4
5
6
7
8

matched=[]
for searchres in interface3:
    bgm=searchres['bgm']
    bili=searchres['bili']
    for bilimd in bili:
        if bilimd[2] not in bgm and (bilimd[3] is not pd.NaT and bilimd[3]==bgm[5]):
            matched.append({'bgm':bgm[:-1],'bili':bilimd[:-1]})
...

1
2
3
4
5
6

fullymatched=pd.DataFrame(fullymatched)
manuallymatched=pd.DataFrame(manuallymatched)
finalmatch=pd.concat([fullymatched,manuallymatched],axis=0)
finalmatch.drop_duplicates(subset='subject',keep='first',inplace=True)

len(finalmatch),finalmatch.head(5)

1
2
3
4
5
6
7

(1837,
   subject    md
 0     326  1565
 1     324  1564
 2     876  1178
 3    1428  1089
 4  211567  6445)

1
2
3
4
5
6

bilidb=pd.merge(left=finalmatch,right=bilidb,left_on='md',right_on='media_id',how='inner')
db=pd.merge(left=bilidb,right=bgmdb,on='subject',how='inner')
db.drop(columns=['md'],inplace=True)
db.rename(columns={'rating_x':'bilirating','rating_y':'bgmrating'},inplace=True)

db.head(5)

1
2

res=stats.linregress(x=db.bgmrating,y=db.bilirating)
res,res.rvalue**2

1
2

(LinregressResult(slope=0.8046754433744006, intercept=3.709341364364734, rvalue=0.736131603271121, pvalue=7.026688311545783e-200, stderr=0.02166711159881075),
 0.5418897373345112)

1
2
3
4
5
6
7
8

fig=plt.figure(num=105,figsize=(8,4),dpi=300,facecolor='white')
ax=fig.gca()
ax.scatter(x=db.bgmrating,y=db.bilirating,color='red',marker='.',s=0.1)
linx=np.arange(2,9,0.1)
liny=res.slope*linx+res.intercept
ax.plot(linx,liny,ls='--',lw=0.5)
ax.set(title='两站评分关系图',xlabel='bangumi',ylabel='bilibili')
ax.text(2,9,'bili=%.4fbgm+%.4f\nR=%.2f'%(res.slope,res.intercept,res.rvalue),fontsize=10)

1
2
3
4
5

db['bgmrating_2digit']=db['bgmrating'].copy()
db['bgmrating_2digit']=db['bgmrating_2digit'].apply(lambda x : ((x*10)//1)/10)
bb=pd.crosstab(index=db['bgmrating_2digit'],columns=db['bilirating'])

bb.iloc[20:,40:]