python爬虫实操 |
您所在的位置:网站首页 › nba球星得分分布图高清 › python爬虫实操 |
本次我们将爬取Ajax动态加载数据并进行简单数据分析,其主要方式是找到数据的json包,将其保存到本地目录,进行数据分析 文章目录 一、业务分析二、步骤1.找到数据2.抓取数据3.分析数据 总结 一、业务分析目标网站:NBA中国官方网站https://china.nba.com/statistics/ 爬取字段: 使用到的库:requests, json,csv,pandas numpy ,matplotlib 其中requests,json进行数据抓取 cxv保存到本地 pandas,numpy进行数据分析 matplotlib可视化 二、步骤 1.找到数据首先,我们要找到储存数据的json包,F12进入开发者模式 点击网络,选择XHR,进行刷新,就可以看到json包了 这里我们可以得到请求头信息以及json包 这就是一会儿要抓取的数据 2.抓取数据这里我选择了抓取本赛季前50球员的数据,在json包中寻找,可以看到 这里存放的是本赛季的数据 导入库 import requests import json import csv import pandas as pd import numpy as np import matplotlib.pyplot as plt先定义抓取json包方法 def getJson(url): headers={ 'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.53' } response = requests.get(url,headers=headers) json_data = json.loads(response.text) return json_data定义抓取数据的方法 def getData(json_data): playerList=[] for item in json_data['payload']['players']: player_dataDict={} #球员名字 name=item['playerProfile']['code'] #出场次数 games=item['statAverage']['games'] #先发 gamesStarted=item['statAverage']['gamesStarted'] #分钟 mins=item['statAverage']['minsPg'] #三分命中 tpm=item['statAverage']['tppct'] #罚球命中 ftm=item['statAverage']['ftpct'] #进攻 offRebs=item['statAverage']['offRebsPg'] #防守 defRebs=item['statAverage']['defRebsPg'] #篮板 rebs=item['statAverage']['rebsPg'] #助攻 assists=item['statAverage']['assistsPg'] #抢断 steals=item['statAverage']['stealsPg'] #盖帽 blocks=item['statAverage']['blocksPg'] #失误 turnovers=item['statAverage']['turnoversPg'] #犯规 fouls=item['statAverage']['foulsPg'] #得分 points=item['statAverage']['pointsPg'] player_dataDict['球员']=name player_dataDict['场次']=games player_dataDict['先发']=gamesStarted player_dataDict['出场时间']=mins player_dataDict['三分命中率']=tpm player_dataDict['罚球命中率']=ftm player_dataDict['进攻效率']=offRebs player_dataDict['防守效率']=defRebs player_dataDict['篮板']=rebs player_dataDict['助攻']= assists player_dataDict['抢断']=steals player_dataDict['盖帽']=blocks player_dataDict['失误']=turnovers player_dataDict['犯规']=fouls player_dataDict['得分']=points print(player_dataDict) playerList.append(player_dataDict) return playerList接下来进行存储 def writeData(playerList): #写入数据 with open('player_data.csv','w',encoding='utf-8',newline='')as f: write=csv.DictWriter(f, fieldnames=['球员','场次','先发','出场时间','三分命中率','罚球命中率', '进攻效率','防守效率','篮板','助攻', '抢断','盖帽','失误','犯规','得分']) write.writeheader() for each in playerList: write.writerow(each)完整代码 import requests import json import csv import pandas as pd import numpy as np import matplotlib.pyplot as plt url='https://china.nba.com/static/data/league/playerstats_All_All_All_0_All_false_2021_2_All_Team_points_All_perGame.json' def getJson(url): headers={ 'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.53' } response = requests.get(url,headers=headers) json_data = json.loads(response.text) return json_data def getData(json_data): playerList=[] for item in json_data['payload']['players']: player_dataDict={} #球员名字 name=item['playerProfile']['code'] #出场次数 games=item['statAverage']['games'] #先发 gamesStarted=item['statAverage']['gamesStarted'] #分钟 mins=item['statAverage']['minsPg'] #三分命中 tpm=item['statAverage']['tppct'] #罚球命中 ftm=item['statAverage']['ftpct'] #进攻 offRebs=item['statAverage']['offRebsPg'] #防守 defRebs=item['statAverage']['defRebsPg'] #篮板 rebs=item['statAverage']['rebsPg'] #助攻 assists=item['statAverage']['assistsPg'] #抢断 steals=item['statAverage']['stealsPg'] #盖帽 blocks=item['statAverage']['blocksPg'] #失误 turnovers=item['statAverage']['turnoversPg'] #犯规 fouls=item['statAverage']['foulsPg'] #得分 points=item['statAverage']['pointsPg'] player_dataDict['球员']=name player_dataDict['场次']=games player_dataDict['先发']=gamesStarted player_dataDict['出场时间']=mins player_dataDict['三分命中率']=tpm player_dataDict['罚球命中率']=ftm player_dataDict['进攻效率']=offRebs player_dataDict['防守效率']=defRebs player_dataDict['篮板']=rebs player_dataDict['助攻']= assists player_dataDict['抢断']=steals player_dataDict['盖帽']=blocks player_dataDict['失误']=turnovers player_dataDict['犯规']=fouls player_dataDict['得分']=points print(player_dataDict) playerList.append(player_dataDict) return playerList def writeData(playerList): #写入数据 with open('player_data.csv','w',encoding='utf-8',newline='')as f: write=csv.DictWriter(f, fieldnames=['球员','场次','先发','出场时间','三分命中率','罚球命中率', '进攻效率','防守效率','篮板','助攻', '抢断','盖帽','失误','犯规','得分']) write.writeheader() for each in playerList: write.writerow(each) if __name__ == "__main__": json_data = getJson(url) playerList=[] playerList += getData(json_data) writeData(playerList) 3.分析数据数据都存放到本地了,我们当然可以为所欲为 这里我们选取了几个字段,生成了每个球员的雷达图,方便进行比较 import pandas as pd import numpy as np import matplotlib.pyplot as plt df=pd.read_csv('player_data.csv') for i in range(50): x=df.loc[i] name=x.loc[['球员']] y=x.loc[['犯规','篮板','助攻','抢断','盖帽','失误']] labels=np.array(['犯规','篮板','助攻','抢断','盖帽','失误']) data=np.array(y) plt.rcParams['font.sans-serif']=['SimHei'] angles=np.linspace(0, 2*np.pi,len(labels),endpoint=False) labels=np.concatenate((labels,[labels[0]])) data=np.concatenate((data,[data[0]])) angles=np.concatenate((angles,[angles[0]])) plt.polar(angles, data,'bo-',linewidth=1) plt.thetagrids(angles*180/np.pi,labels) plt.fill(angles, data,facecolor='b',alpha=0.25) plt.title(str(name)) plt.show()
就不一个个上图了 总结Ajax动态数据还是非常容易爬取的,同时pandas和numpy库也非常值得学习 最后一句 “湖人总冠军” |
今日新闻 |
点击排行 |
|
推荐新闻 |
图片新闻 |
|
专题文章 |
CopyRight 2018-2019 实验室设备网 版权所有 win10的实时保护怎么永久关闭 |