高考期间爬取大学和专业信息的一些爬虫

1.专业热度的爬虫

 

import json
import requests
import time
import numpy as np
import pandas as pd
#每页获取数据
#https://api.eol.cn/gkcx/api/?access_token=&keyword=&level1=1&page=1&request_type=1&signsafe=&size=20&sort=view_total&uri=apidata/api/gk/special/lists
def get_gd_zyhot_one(page_num):
url="https://api.eol.cn/gkcx/api/"
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
'Origin':'https://gkcx.eol.cn',
'Referer':'https://gkcx.eol.cn/hotschool?province=%E5%B9%BF%E4%B8%9C',
}
data={
'access_token':"",
'keyword':"",
'level1':"1",
'page':page_num,
'request_type':1,
'signsafe':"",
'size':20,
'sort':"view_total",
'uri':"apidata/api/gk/special/lists",
}
try:
response=requests.post(url=url,data=data,headers=headers)
except Exception as gk:
print(gk)
time.sleep(3)
response=requests.post(url=url,data=data,headers=headers)
major_hot=json.loads(response.text)['data']['item']
#专业昵称
name=[i.get('name') for i in major_hot]
# 专业类别
marjor_lb = [i.get('level3_name') for i in major_hot]
# 学科类别
xk_type = [i.get('level2_name') for i in major_hot]
#人气值
rank_hot=[i.get('view_total') for i in major_hot]
#专业修读年数limit_year
limit_year=[i.get('limit_year') for i in major_hot]

df=pd.DataFrame({
'专业名称':name,
'学科类别':xk_type,
'专业类别':marjor_lb,
'最少修读年数':limit_year,
'人气值':rank_hot

})
return df
def get_all_page(all_page_num):
df_all=pd.DataFrame()
for i in range(all_page_num):
print(f'正在读取第{i+1}页的数据')
df_one=get_gd_zyhot_one(page_num=i+1)
df_all=df_all.append(df_one,ignore_index=True)
time.sleep(np.random.uniform(2))
return df_all

showtable_sj=get_all_page(all_page_num=34)
showtable_sj.to_excel('./data/专业热度.xlsx',index=False)
2.广东省高考大学热度爬虫
import json
import requests
import time
import numpy as np
import pandas as pd
#每页获取数据
#https://api.eol.cn/gkcx/api/?access_token=&keyword=&page=1&province_id=44&school_type=&signsafe=&size=20&sort=view_total&sorttype=desc&type=&uri=apidata/api/gk/school/lists
def get_gd_zyhot_one(page_num):
url="https://api.eol.cn/gkcx/api/"
headers={
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
'Origin':'https://gkcx.eol.cn',
'Referer':'https://gkcx.eol.cn/hotschool?province=%E5%B9%BF%E4%B8%9C',
}
data={
'access_token':"",
'keyword':"",
'page':page_num,
'province_id':44,
'school_type':"",
'signsafe':"",
'size':20,
'sort':"view_total",
'sorttype':"desc",
'type':"",
'uri':"apidata/api/gk/school/lists",
}
try:
response=requests.post(url=url,data=data,headers=headers)
except Exception as gk:
print(gk)
time.sleep(3)
response=requests.post(url=url,data=data,headers=headers)
major_hot=json.loads(response.text)['data']['item']
#高校昵称
name=[i.get('name') for i in major_hot]
print(name)
# 热度排名
rank = [i.get('rank') for i in major_hot]
# 高校类别
type = [i.get('type_name') for i in major_hot]
#高校热度
rank_hot=[i.get('view_total') for i in major_hot]

df=pd.DataFrame({
'广东高校名称':name,
'热度总排名':rank,
'高校类别':type,
'高校热度量':rank_hot

})
return df
def get_all_page(all_page_num):
df_all=pd.DataFrame()
for i in range(all_page_num):
print(f'正在读取第{i+1}页的数据')
df_one=get_gd_zyhot_one(page_num=i+1)
df_all=df_all.append(df_one,ignore_index=True)
time.sleep(np.random.uniform(2))
return df_all

showtable_sj=get_all_page(all_page_num=9)
showtable_sj.to_excel('./data/广东高考大学热度.xlsx',index=False)

 

点赞

发表评论

昵称和uid可以选填一个,填邮箱必填(留言回复后将会发邮件给你)
tips:输入uid可以快速获得你的昵称和头像