stock-tracker

This commit is contained in:
C菌
2026-07-04 00:17:11 +08:00
commit 6087341a48
6463 changed files with 1929869 additions and 0 deletions
@@ -0,0 +1,6 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Date: 2020/10/18 12:54
Desc:
"""
@@ -0,0 +1,462 @@
# -*- coding:utf-8 -*-
# !/usr/bin/env python
"""
Date: 2026/4/12 17:00
Desc: 百度股市通-经济数据
https://finance.baidu.com/calendar
"""
import math
import re
import pandas as pd
from curl_cffi import requests
def _get_baidu_cookie(headers: dict) -> str:
"""
安全获取百度股市通所需的 Cookie
:param headers: 基础请求头
:return: 格式化的 Cookie字符串
:raises ValueError: 当无法获取必要Cookie时
:raises ConnectionError: 网络请求失败时
"""
try:
# 使用Session保持Cookie上下文
with requests.Session() as session:
session.headers.update(headers)
# 第一步:获取基础Cookie (BAIDUID系列)
resp1 = session.get(
"https://finance.baidu.com/calendar",
impersonate="chrome110",
timeout=10,
)
resp1.raise_for_status()
# 验证必要Cookie
baiduid = resp1.cookies.get("BAIDUID")
baiduid_bfess = resp1.cookies.get("BAIDUID_BFESS")
if not all([baiduid, baiduid_bfess]):
raise ValueError("Missing BAIDUID cookies in first response")
# 第二步:提取并请求hm.js
hm_pattern = r"https://hm\.baidu\.com/hm\.js\?\w+"
hm_match = re.search(hm_pattern, resp1.text)
if not hm_match:
# 尝试备用正则模式
hm_match = re.search(r"//hm\.baidu\.com/hm\.js\?\w+", resp1.text)
if not hm_match:
raise ValueError("Failed to extract hm.js URL from response")
hm_url = (
"https:" + hm_match.group()
if hm_match.group().startswith("//")
else hm_match.group()
)
# 第二步请求 (自动携带第一步的Cookie)
resp2 = session.get(hm_url, impersonate="chrome110", timeout=10)
resp2.raise_for_status()
# 验证必要Cookie
hmac_count = resp2.cookies.get("HMACCOUNT")
hmac_count_bfess = resp2.cookies.get("HMACCOUNT_BFESS")
if not all([hmac_count, hmac_count_bfess]):
raise ValueError("Missing HMACCOUNT cookies in second response")
# 安全拼接Cookie
return (
f"BAIDUID={baiduid}; "
f"BAIDUID_BFESS={baiduid_bfess}; "
f"HMACCOUNT={hmac_count}; "
f"HMACCOUNT_BFESS={hmac_count_bfess}"
)
except requests.exceptions.RequestException as e:
raise ConnectionError(f"Network request failed: {str(e)}") from e
except re.error as e:
raise ValueError(f"Regex pattern error: {str(e)}") from e
def _baidu_finance_calendar(
date: str, cate: str, process_func, cookie: str = None
) -> pd.DataFrame:
"""
百度股市通日历数据基础函数(支持分页)
:param date: 查询日期 (格式: YYYYMMDD)
:param cate: 数据类别 ("economic_data""notify_suspend")
:param process_func: 数据处理函数
:param cookie: cookie
:return: 处理后的DataFrame
"""
# 日期格式转换
formatted_date = "-".join([date[:4], date[4:6], date[6:]])
# 构建请求参数
base_params = {
"start_date": formatted_date,
"end_date": formatted_date,
"pn": "0",
"rn": "100", # 每页100条
"cate": cate,
"finClientType": "pc",
}
# 构建请求头
headers = {
"accept": "application/vnd.finance-web.v1+json",
"accept-encoding": "gzip, deflate, br, zstd",
"accept-language": "en,zh-CN;q=0.9,zh;q=0.8",
"cache-control": "no-cache",
"origin": "https://finance.baidu.com",
"pragma": "no-cache",
"priority": "u=1, i",
"referer": "https://finance.baidu.com/",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/142.0.0.0 Safari/537.36",
}
# 在_baidu_finance_calendar函数中替换原代码块
if cookie is None:
try:
cookie = _get_baidu_cookie(headers.copy()) # 保护原始headers
except Exception as e:
# 可降级处理或保留原始行为
raise RuntimeError(f"Failed to obtain Baidu cookies: {str(e)}") from e
headers["cookie"] = cookie
url = "https://finance.pae.baidu.com/sapi/v1/financecalendar"
big_df = pd.DataFrame()
# 获取指定日期的总记录数
target_date = formatted_date
total_records = 0
# 第一次请求
params = base_params.copy()
response = requests.get(
url=url, params=params, headers=headers, impersonate="chrome110"
)
response.raise_for_status()
data_json = response.json()
# 从JSON中提取指定日期的总记录数
if "Result" in data_json and "calendarInfo" in data_json["Result"]:
calendar_info = data_json["Result"]["calendarInfo"]
# 查找目标日期的记录
for item in calendar_info:
if item.get("date") == target_date:
total_records = item.get("total", 0)
break
# 计算总页数 (每页100条)
total_pages = math.ceil(total_records / 100) if total_records > 0 else 1
# 处理所有页码
for page in range(total_pages):
if page > 0: # 第一页已在前面获取
params = base_params.copy()
params["pn"] = str(page)
response = requests.get(url=url, params=params, headers=headers)
response.raise_for_status()
data_json = response.json()
# 提取并处理指定日期的数据
if "Result" in data_json and "calendarInfo" in data_json["Result"]:
for item in data_json["Result"]["calendarInfo"]:
if item.get("date") == target_date and item.get("list"):
processed_df = process_func(item["list"])
big_df = pd.concat([big_df, processed_df], ignore_index=True)
return big_df
def _process_economic_data(data_list: list) -> pd.DataFrame:
"""处理经济数据"""
if not data_list:
return pd.DataFrame()
temp_df = pd.DataFrame(data_list)
rename_dict = {
"date": "日期",
"time": "时间",
"title": "事件",
"star": "重要性",
"formerVal": "前值",
"pubVal": "公布",
"region": "地区",
"indicateVal": "预期",
"country": "国家",
"timePeriod": "统计周期",
}
temp_df.rename(columns=rename_dict, inplace=True)
required_cols = ["公布", "预期", "前值", "重要性"]
for col in required_cols:
if col not in temp_df.columns:
temp_df[col] = None
available_cols = []
for col in [
"日期",
"时间",
"国家",
"地区",
"事件",
"统计周期",
"公布",
"预期",
"前值",
"重要性",
]:
if col in temp_df.columns:
available_cols.append(col)
if available_cols:
temp_df = temp_df[available_cols]
for col in ["公布", "预期", "前值", "重要性"]:
if col in temp_df.columns:
temp_df[col] = pd.to_numeric(temp_df[col], errors="coerce")
if "日期" in temp_df.columns:
temp_df["日期"] = pd.to_datetime(temp_df["日期"], errors="coerce").dt.date
return temp_df
def _process_suspend_data(data_list: list) -> pd.DataFrame:
"""处理停复牌数据 - 根据实际JSON结构精确修正"""
if not data_list:
return pd.DataFrame()
temp_df = pd.DataFrame(data_list)
rename_dict = {
"code": "股票代码",
"name": "股票简称",
"exchange": "交易所代码",
"start": "停牌时间",
"reason": "停牌事项说明",
"marketValue": "市值",
"date": "公告日期",
"time": "公告时间",
"type": "证券类型",
"market": "市场类型",
"isSkip": "是否跳过",
"end": "复牌时间",
}
temp_df.rename(columns=rename_dict, inplace=True)
if "复牌时间" not in temp_df.columns:
temp_df["复牌时间"] = "-"
temp_df = temp_df[
[
"股票代码",
"股票简称",
"交易所代码",
"停牌时间",
"复牌时间",
"停牌事项说明",
"市值",
"公告日期",
"公告时间",
"证券类型",
"市场类型",
"是否跳过",
]
]
return temp_df
def news_economic_baidu(date: str = "20251126", cookie: str = None) -> pd.DataFrame:
"""
百度股市通-经济数据
https://finance.baidu.com/calendar
:param date: 查询日期 (格式: YYYYMMDD)
:param cookie: cookie
:return: 经济数据 pd.DataFrame
"""
return _baidu_finance_calendar(
date=date,
cate="economic_data",
process_func=_process_economic_data,
cookie=cookie,
)
def news_trade_notify_suspend_baidu(
date: str = "20251126", cookie: str = None
) -> pd.DataFrame:
"""
百度股市通-交易提醒-停复牌
https://finance.baidu.com/calendar
:param date: 查询日期 (格式: YYYYMMDD)
:param cookie: cookie
:return: 停复牌数据DataFrame
"""
return _baidu_finance_calendar(
date=date,
cate="notify_suspend",
process_func=_process_suspend_data,
cookie=cookie,
)
def _process_dividend_data(data_list: list) -> pd.DataFrame:
"""处理分红派息数据"""
if not data_list:
return pd.DataFrame()
temp_df = pd.DataFrame(data_list)
# 字段映射
rename_dict = {
"code": "股票代码",
"market": "-", # 这个字段在最终结果中会被删除
"exchange": "交易所",
"name": "股票简称",
"diviDate": "除权日",
"date": "报告期",
"diviCash": "分红",
"shareDivide": "送股",
"transfer": "转增",
"physical": "实物",
}
temp_df.rename(columns=rename_dict, inplace=True)
# 确保必要列存在
if "分红" not in temp_df.columns:
temp_df["分红"] = "-"
if "实物" not in temp_df.columns:
temp_df["实物"] = "-"
if "送股" not in temp_df.columns:
temp_df["送股"] = "-"
if "转增" not in temp_df.columns:
temp_df["转增"] = "-"
# 选择需要的列
temp_df = temp_df[
[
"股票代码",
"除权日",
"分红",
"送股",
"转增",
"实物",
"交易所",
"股票简称",
"报告期",
]
]
# 日期格式转换
if "除权日" in temp_df.columns:
temp_df["除权日"] = pd.to_datetime(temp_df["除权日"], errors="coerce").dt.date
if "报告期" in temp_df.columns:
temp_df["报告期"] = pd.to_datetime(temp_df["报告期"], errors="coerce").dt.date
return temp_df
def news_trade_notify_dividend_baidu(
date: str = "20251126", cookie: str = None
) -> pd.DataFrame:
"""
百度股市通-交易提醒-分红派息
https://finance.baidu.com/calendar
:param date: 查询日期 (格式: YYYYMMDD)
:param cookie: cookie
:return: 交易提醒-分红派息DataFrame
"""
return _baidu_finance_calendar(
date=date,
cate="notify_divide",
process_func=_process_dividend_data,
cookie=cookie,
)
def _process_report_data(data_list: list) -> pd.DataFrame:
"""处理财报发行数据 - 根据实际JSON结构精确修正"""
if not data_list:
return pd.DataFrame()
# 创建DataFrame
temp_df = pd.DataFrame(data_list)
# 精确字段映射 (根据提供的JSON结构)
rename_dict = {
"code": "股票代码",
"name": "股票简称",
"exchange": "交易所",
"reportType": "财报类型",
"time": "发布时间",
"marketValue": "市值",
"capitalization": "总市值",
"date": "发布日期",
}
temp_df.rename(columns=rename_dict, inplace=True)
# 确保必要列存在
if "财报类型" not in temp_df.columns:
temp_df["财报类型"] = "-"
if "发布时间" not in temp_df.columns:
temp_df["发布时间"] = "-"
if "市值" not in temp_df.columns and "总市值" in temp_df.columns:
temp_df["市值"] = temp_df["总市值"]
# 选择并排序列
available_cols = []
for col in [
"股票代码",
"股票简称",
"交易所",
"财报类型",
"发布时间",
"市值",
"发布日期",
]:
if col in temp_df.columns:
available_cols.append(col)
if available_cols:
temp_df = temp_df[available_cols]
else:
# 如果没有匹配的列,返回空DataFrame
return pd.DataFrame()
# 类型转换
if "市值" in temp_df.columns:
temp_df["市值"] = pd.to_numeric(temp_df["市值"], errors="coerce")
if "发布日期" in temp_df.columns:
temp_df["发布日期"] = pd.to_datetime(
temp_df["发布日期"], errors="coerce"
).dt.date
return temp_df
def news_report_time_baidu(date: str = "20251126", cookie: str = None) -> pd.DataFrame:
"""
百度股市通-财报发行
https://finance.baidu.com/calendar
:param date: 查询日期 (格式: YYYYMMDD)
:param cookie: cookie
:return: 财报发行DataFrame
"""
return _baidu_finance_calendar(
date=date, cate="report_time", process_func=_process_report_data, cookie=cookie
)
if __name__ == "__main__":
news_economic_baidu_df = news_economic_baidu(date="20251126")
print(news_economic_baidu_df)
news_trade_notify_suspend_baidu_df = news_trade_notify_suspend_baidu(
date="20251126"
)
print(news_trade_notify_suspend_baidu_df)
news_trade_notify_dividend_baidu_df = news_trade_notify_dividend_baidu(
date="20251126"
)
print(news_trade_notify_dividend_baidu_df)
news_report_time_baidu_df = news_report_time_baidu(date="20251126")
print(news_report_time_baidu_df)
@@ -0,0 +1,181 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Date: 2024/4/25 17:00
Desc: 新闻联播文字稿
https://tv.cctv.com/lm/xwlb
"""
import re
import pandas as pd
import requests
from bs4 import BeautifulSoup
from tqdm import tqdm
def news_cctv(date: str = "20240424") -> pd.DataFrame:
"""
新闻联播文字稿
https://tv.cctv.com/lm/xwlb
:param date: 需要获取数据的日期; 目前 20160203 年后
:type date: str
:return: 新闻联播文字稿
:rtype: pandas.DataFrame
"""
if int(date) <= int("20130708"):
url = f"https://cctv.cntv.cn/lm/xinwenlianbo/{date}.shtml"
r = requests.get(url)
r.encoding = "gbk"
raw_list = re.findall(r"title_array_01\((.*)", r.text)
page_url = [
re.findall("(http.*)", item)[0].split("'")[0] for item in raw_list[1:]
]
title_list = []
content_list = []
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
"image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
"Host": "tv.cctv.com",
"Pragma": "no-cache",
"Proxy-Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/92.0.4515.159 Safari/537.36",
}
for page in tqdm(page_url, leave=False):
try:
r = requests.get(page, headers=headers)
r.encoding = "utf-8"
soup = BeautifulSoup(r.text, "lxml")
title = soup.find("h3").text
content = soup.find("div", attrs={"class": "cnt_bd"}).text
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
content_list.append(
content.strip()
.strip("央视网消息(新闻联播)")
.strip("央视网消息(新闻联播):")
.strip("(新闻联播)")
.strip()
.replace("\n", " ")
)
except: # noqa: E722
continue
temp_df = pd.DataFrame(
data=[[date] * len(title_list), title_list, content_list],
index=["date", "title", "content"],
).T
return temp_df
elif int(date) < int("20160203"):
url = f"https://cctv.cntv.cn/lm/xinwenlianbo/{date}.shtml"
r = requests.get(url)
r.encoding = "utf-8"
soup = BeautifulSoup(r.text, "lxml")
page_url = [
item.find("a")["href"]
for item in soup.find(
name="div", attrs={"id": "contentELMT1368521805488378"}
).find_all("li")[1:]
]
title_list = []
content_list = []
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
"Host": "tv.cctv.com",
"Pragma": "no-cache",
"Proxy-Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/92.0.4515.159 Safari/537.36",
}
for page in tqdm(page_url, leave=False):
try:
r = requests.get(page, headers=headers)
r.encoding = "utf-8"
soup = BeautifulSoup(r.text, features="lxml")
title = soup.find("h3").text
content = soup.find(name="div", attrs={"class": "cnt_bd"}).text
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
content_list.append(
content.strip()
.strip("央视网消息(新闻联播)")
.strip("央视网消息(新闻联播):")
.strip("(新闻联播)")
.strip()
.replace("\n", " ")
)
except: # noqa: E722
continue
temp_df = pd.DataFrame(
data=[[date] * len(title_list), title_list, content_list],
index=["date", "title", "content"],
).T
return temp_df
elif int(date) > int("20160203"):
url = f"https://tv.cctv.com/lm/xwlb/day/{date}.shtml"
r = requests.get(url)
r.encoding = "utf-8"
soup = BeautifulSoup(r.text, "lxml")
page_url = [item.find("a")["href"] for item in soup.find_all("li")[1:]]
title_list = []
content_list = []
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Cache-Control": "no-cache",
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
"Host": "tv.cctv.com",
"Pragma": "no-cache",
"Proxy-Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/92.0.4515.159 Safari/537.36",
}
for page in tqdm(page_url, leave=False):
try:
r = requests.get(page, headers=headers)
r.encoding = "utf-8"
soup = BeautifulSoup(r.text, features="lxml")
if soup.find("h3"):
title = soup.find("h3").text
else:
title = soup.find(name="div", attrs={"class": "tit"}).text
if soup.find(name="div", attrs={"class": "cnt_bd"}):
content = soup.find(name="div", attrs={"class": "cnt_bd"}).text
else:
content = soup.find(
name="div", attrs={"class": "content_area"}
).text
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
content_list.append(
content.strip()
.strip("央视网消息(新闻联播)")
.strip("央视网消息(新闻联播):")
.strip("(新闻联播)")
.strip()
.replace("\n", " ")
)
except: # noqa: E722
continue
temp_df = pd.DataFrame(
data=[[date] * len(title_list), title_list, content_list],
index=["date", "title", "content"],
).T
return temp_df
if __name__ == "__main__":
news_cctv_df = news_cctv(date="20240424")
print(news_cctv_df)
@@ -0,0 +1,123 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
Date: 2025/12/23 17:15
Desc: 个股新闻数据
https://so.eastmoney.com/news/s?keyword=603777
"""
import json
import pandas as pd
from curl_cffi import requests
def stock_news_em(symbol: str = "603777") -> pd.DataFrame:
"""
东方财富-个股新闻-最近 100 条新闻
https://so.eastmoney.com/news/s?keyword=603777
:param symbol: 股票代码
:type symbol: str
:return: 个股新闻
:rtype: pandas.DataFrame
"""
url = "https://search-api-web.eastmoney.com/search/jsonp"
inner_param = {
"uid": "",
"keyword": symbol,
"type": ["cmsArticleWebOld"],
"client": "web",
"clientType": "web",
"clientVersion": "curr",
"param": {
"cmsArticleWebOld": {
"searchScope": "default",
"sort": "default",
"pageIndex": 1,
"pageSize": 10,
"preTag": "<em>",
"postTag": "</em>",
}
},
}
params = {
"cb": "jQuery35101792940631092459_1764599530165",
"param": json.dumps(inner_param, ensure_ascii=False), # 保留中文,
"_": "1764599530176",
}
headers = {
"accept": "*/*",
"accept-encoding": "gzip, deflate, br, zstd",
"accept-language": "en,zh-CN;q=0.9,zh;q=0.8",
"cache-control": "no-cache",
"connection": "keep-alive",
"cookie": "qgqp_b_id=652bf4c98a74e210088f372a17d4e27b; st_nvi=ulN5JAj9FUocz3p4klMME9f20; emshistory=%5B%22603777%22%5D; nid18=010d039dd427dc4d187090491f47d7ad; nid18_create_time=1764582801999; gviem=gSdeY51VWSuTzM3kWaagtf560; gviem_create_time=1764582801999; st_si=55269775884615; st_pvi=66803244437563; st_sp=2025-11-19%2014%3A19%3A16; st_inirUrl=https%3A%2F%2Fso.eastmoney.com%2Fnews%2Fs; st_sn=2; st_psi=20251201223210488-118000300905-0940816858; st_asi=delete",
"host": "search-api-web.eastmoney.com",
"pragma": "no-cache",
"referer": "https://so.eastmoney.com/news/s?keyword=603777",
"sec-ch-ua": '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
"sec-fetch-dest": "script",
"sec-fetch-mode": "no-cors",
"sec-fetch-site": "same-site",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
}
r = requests.get(url, params=params, headers=headers)
data_text = r.text
data_json = json.loads(
data_text.strip("jQuery35101792940631092459_1764599530165(")[:-1]
)
temp_df = pd.DataFrame(data_json["result"]["cmsArticleWebOld"])
temp_df["url"] = "http://finance.eastmoney.com/a/" + temp_df["code"] + ".html"
temp_df.rename(
columns={
"date": "发布时间",
"mediaName": "文章来源",
"code": "-",
"title": "新闻标题",
"content": "新闻内容",
"url": "新闻链接",
"image": "-",
},
inplace=True,
)
temp_df["关键词"] = symbol
temp_df = temp_df[
[
"关键词",
"新闻标题",
"新闻内容",
"发布时间",
"文章来源",
"新闻链接",
]
]
temp_df["新闻标题"] = (
temp_df["新闻标题"]
.str.replace(r"\(<em>", "", regex=True)
.str.replace(r"</em>\)", "", regex=True)
)
temp_df["新闻标题"] = (
temp_df["新闻标题"]
.str.replace(r"<em>", "", regex=True)
.str.replace(r"</em>", "", regex=True)
)
temp_df["新闻内容"] = (
temp_df["新闻内容"]
.str.replace(r"\(<em>", "", regex=True)
.str.replace(r"</em>\)", "", regex=True)
)
temp_df["新闻内容"] = (
temp_df["新闻内容"]
.str.replace(r"<em>", "", regex=True)
.str.replace(r"</em>", "", regex=True)
)
temp_df["新闻内容"] = temp_df["新闻内容"].str.replace(r"\u3000", "", regex=True)
temp_df["新闻内容"] = temp_df["新闻内容"].str.replace(r"\r\n", " ", regex=True)
return temp_df
if __name__ == "__main__":
stock_news_em_df = stock_news_em(symbol="603777")
print(stock_news_em_df)