财经数据（5）

2024-04-01 18:35| 来源: 网络整理| 查看: 265

本次主要是抓取开盘啦股票概念数据

采用多进程、requests完成数据的爬取

采用Pandas库完成数据比对，实现mysql数据存储

具体代码如下：

复制代码

-- coding: utf-8 --

import pandas as pd import tushare as ts import time import requests import json from sqlalchemy import create_engine from multiprocessing import Pool from requests.packages.urllib3.exceptions import InsecureRequestWarning

Tushare股票code获取================================================================================================

def getCode(): print("-------------------------------------------") print(“开始从Tushare接口获取股票行情列表数据”)

# 初始化tushare.pro接口 pro = ts.pro_api('ac16b470869c5d82db5033ae9288f77b282d2b5519507d6d2c72fdd7') # L 表示正常上市，P 表示暂停上市 l_list = pro.stock_basic(list_status='L', fields='ts_code,symbol,name,area,exchange,list_status,list_date') p_list = pro.stock_basic(list_status='P', fields='ts_code,symbol,name,area,exchange,list_status,list_date') # 合并正常上市、暂停上市数据 stock_list = pd.concat([l_list, p_list], axis=0, ignore_index=True) # 创建空列表 code_list = [] for index, row in stock_list.iterrows(): symbol = row['symbol'] code_list.append(symbol) return code_list 爬取PC端开盘啦板块数据================================================================================================

def Kplspider(data_list): print("-------------------------------------------") # 构造空html列表 html_list = []

# 构造URL请求、user-agent头文件 url = 'https://pchq.kaipanla.com/w1/api/index.php' headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3314.0 Safari/537.36 SE 2.X MetaSr 1.0'} session = requests.Session() # 禁用安全请求警告 requests.packages.urllib3.disable_warnings(InsecureRequestWarning) for data in data_list: try: html = session.post(url=url, data=data, headers=headers, verify=False).text html_list.append(html) except Exception as spider_error: print("html抓取过程报错，错误信息为：%s" % spider_error) # 分别创建用于存储tag、concept的空Dataframe stock_tag = pd.DataFrame(); stock_concept = pd.DataFrame() print("-------------------------------------------") print("股票标签、所属概念数据开始解析") for html in html_list: # 解析开盘啦股票标签 code = json.loads(html)['trend']['code'] day = json.loads(html)['trend']['day'] tag = json.loads(html)["pankou"]["tag"] stock_tag = stock_tag.append({'symbol': code, 'tag': tag, 'in_date':day}, ignore_index=True) cept_list = json.loads(html)["stockplate"] try: for cept in cept_list: stock_concept = stock_concept.append({'symbol':code, 'concept': cept[0], 'in_date': day}, ignore_index=True) except Exception as parser_error: print("html抓取过程报错，错误信息为：%s" % parser_error) print("%s概念数据请求为空，请知悉" % code) # 创建Pandas读写数据库引擎 engine = create_engine('mysql://root:[email protected]/quant?charset=utf8') # 开始存储标签数据 old_tag = pd.read_sql('select * from is_belong_zyj', engine) stock_tag = stock_tag[['symbol','tag','in_date']] stock_tag = stock_tag.append(old_tag,ignore_index=True,sort=False) stock_tag.drop_duplicates(subset=['symbol', 'tag'], keep=False,inplace=True) stock_tag.to_sql('is_belong_zyj', engine, if_exists='append', index=False) print(stock_tag) print("本次存储开盘啦标签数据%s条" % stock_tag.shape[0]) # 开始存储所属概念数据 old_concept = pd.read_sql('select * from belong_concept',engine) stock_concept = stock_tag[['symbol','concept','in_date']] stock_concept = stock_tag.append(old_tag,ignore_index=True,sort=False) stock_concept.drop_duplicates(subset=['symbol', 'concept'], keep=False,inplace=True) stock_concept.to_sql('belong_concept', engine, if_exists='append', index=False) print(stock_concept) print("本次存储开盘啦标签数据%s条" % stock_concept.shape[0]) 主函数================================================================================================================

if name == ‘main’: print(“开盘啦股票标签及概念爬虫程序开始执行”) print("-------------------------------------") start = time.time()

# 调用getCode code_list = getCode() # 获取当前日期 cur_date = time.strftime("%Y%m%d", time.localtime()) # 创建多进程 pool = Pool(processes=4) # 构造post请求表单 data_list = [] for code in code_list: data = {'c': 'PCArrangeData','a': 'GetHQPlate','StockID': code,'Day': cur_date,'SelType': '1, 2, 3, 8, 9, 5, 6, 7','UserID': 399083,'Token': '71aef0e806e61ad3169ddc9473e37886'} data_list.append(data) # 开启多进程爬取开盘啦数据 try: pool.map(Kplspider, (data_list,)) except Exception as error: print("进程执行过程报错，错误信息为：%s" % error) end = time.time() print('开盘啦股票标签及概念爬虫程序共执行%0.2f秒.' % ((end - start))) print("开盘啦股票标签及概念爬虫程序执行完成")

【本文地址】

公司简介

联系我们