stock-tracker
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
Date: 2020/10/18 12:54
|
||||
Desc:
|
||||
"""
|
||||
@@ -0,0 +1,462 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
# !/usr/bin/env python
|
||||
"""
|
||||
Date: 2026/4/12 17:00
|
||||
Desc: 百度股市通-经济数据
|
||||
https://finance.baidu.com/calendar
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
from curl_cffi import requests
|
||||
|
||||
|
||||
def _get_baidu_cookie(headers: dict) -> str:
|
||||
"""
|
||||
安全获取百度股市通所需的 Cookie
|
||||
:param headers: 基础请求头
|
||||
:return: 格式化的 Cookie字符串
|
||||
:raises ValueError: 当无法获取必要Cookie时
|
||||
:raises ConnectionError: 网络请求失败时
|
||||
"""
|
||||
try:
|
||||
# 使用Session保持Cookie上下文
|
||||
with requests.Session() as session:
|
||||
session.headers.update(headers)
|
||||
|
||||
# 第一步:获取基础Cookie (BAIDUID系列)
|
||||
resp1 = session.get(
|
||||
"https://finance.baidu.com/calendar",
|
||||
impersonate="chrome110",
|
||||
timeout=10,
|
||||
)
|
||||
resp1.raise_for_status()
|
||||
|
||||
# 验证必要Cookie
|
||||
baiduid = resp1.cookies.get("BAIDUID")
|
||||
baiduid_bfess = resp1.cookies.get("BAIDUID_BFESS")
|
||||
if not all([baiduid, baiduid_bfess]):
|
||||
raise ValueError("Missing BAIDUID cookies in first response")
|
||||
|
||||
# 第二步:提取并请求hm.js
|
||||
hm_pattern = r"https://hm\.baidu\.com/hm\.js\?\w+"
|
||||
hm_match = re.search(hm_pattern, resp1.text)
|
||||
if not hm_match:
|
||||
# 尝试备用正则模式
|
||||
hm_match = re.search(r"//hm\.baidu\.com/hm\.js\?\w+", resp1.text)
|
||||
if not hm_match:
|
||||
raise ValueError("Failed to extract hm.js URL from response")
|
||||
|
||||
hm_url = (
|
||||
"https:" + hm_match.group()
|
||||
if hm_match.group().startswith("//")
|
||||
else hm_match.group()
|
||||
)
|
||||
|
||||
# 第二步请求 (自动携带第一步的Cookie)
|
||||
resp2 = session.get(hm_url, impersonate="chrome110", timeout=10)
|
||||
resp2.raise_for_status()
|
||||
|
||||
# 验证必要Cookie
|
||||
hmac_count = resp2.cookies.get("HMACCOUNT")
|
||||
hmac_count_bfess = resp2.cookies.get("HMACCOUNT_BFESS")
|
||||
if not all([hmac_count, hmac_count_bfess]):
|
||||
raise ValueError("Missing HMACCOUNT cookies in second response")
|
||||
|
||||
# 安全拼接Cookie
|
||||
return (
|
||||
f"BAIDUID={baiduid}; "
|
||||
f"BAIDUID_BFESS={baiduid_bfess}; "
|
||||
f"HMACCOUNT={hmac_count}; "
|
||||
f"HMACCOUNT_BFESS={hmac_count_bfess}"
|
||||
)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
raise ConnectionError(f"Network request failed: {str(e)}") from e
|
||||
except re.error as e:
|
||||
raise ValueError(f"Regex pattern error: {str(e)}") from e
|
||||
|
||||
|
||||
def _baidu_finance_calendar(
|
||||
date: str, cate: str, process_func, cookie: str = None
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
百度股市通日历数据基础函数(支持分页)
|
||||
:param date: 查询日期 (格式: YYYYMMDD)
|
||||
:param cate: 数据类别 ("economic_data" 或 "notify_suspend")
|
||||
:param process_func: 数据处理函数
|
||||
:param cookie: cookie
|
||||
:return: 处理后的DataFrame
|
||||
"""
|
||||
# 日期格式转换
|
||||
formatted_date = "-".join([date[:4], date[4:6], date[6:]])
|
||||
|
||||
# 构建请求参数
|
||||
base_params = {
|
||||
"start_date": formatted_date,
|
||||
"end_date": formatted_date,
|
||||
"pn": "0",
|
||||
"rn": "100", # 每页100条
|
||||
"cate": cate,
|
||||
"finClientType": "pc",
|
||||
}
|
||||
|
||||
# 构建请求头
|
||||
headers = {
|
||||
"accept": "application/vnd.finance-web.v1+json",
|
||||
"accept-encoding": "gzip, deflate, br, zstd",
|
||||
"accept-language": "en,zh-CN;q=0.9,zh;q=0.8",
|
||||
"cache-control": "no-cache",
|
||||
"origin": "https://finance.baidu.com",
|
||||
"pragma": "no-cache",
|
||||
"priority": "u=1, i",
|
||||
"referer": "https://finance.baidu.com/",
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/142.0.0.0 Safari/537.36",
|
||||
}
|
||||
|
||||
# 在_baidu_finance_calendar函数中替换原代码块
|
||||
if cookie is None:
|
||||
try:
|
||||
cookie = _get_baidu_cookie(headers.copy()) # 保护原始headers
|
||||
except Exception as e:
|
||||
# 可降级处理或保留原始行为
|
||||
raise RuntimeError(f"Failed to obtain Baidu cookies: {str(e)}") from e
|
||||
headers["cookie"] = cookie
|
||||
|
||||
url = "https://finance.pae.baidu.com/sapi/v1/financecalendar"
|
||||
big_df = pd.DataFrame()
|
||||
|
||||
# 获取指定日期的总记录数
|
||||
target_date = formatted_date
|
||||
total_records = 0
|
||||
|
||||
# 第一次请求
|
||||
params = base_params.copy()
|
||||
response = requests.get(
|
||||
url=url, params=params, headers=headers, impersonate="chrome110"
|
||||
)
|
||||
response.raise_for_status()
|
||||
data_json = response.json()
|
||||
|
||||
# 从JSON中提取指定日期的总记录数
|
||||
if "Result" in data_json and "calendarInfo" in data_json["Result"]:
|
||||
calendar_info = data_json["Result"]["calendarInfo"]
|
||||
|
||||
# 查找目标日期的记录
|
||||
for item in calendar_info:
|
||||
if item.get("date") == target_date:
|
||||
total_records = item.get("total", 0)
|
||||
break
|
||||
|
||||
# 计算总页数 (每页100条)
|
||||
total_pages = math.ceil(total_records / 100) if total_records > 0 else 1
|
||||
|
||||
# 处理所有页码
|
||||
for page in range(total_pages):
|
||||
if page > 0: # 第一页已在前面获取
|
||||
params = base_params.copy()
|
||||
params["pn"] = str(page)
|
||||
response = requests.get(url=url, params=params, headers=headers)
|
||||
response.raise_for_status()
|
||||
data_json = response.json()
|
||||
|
||||
# 提取并处理指定日期的数据
|
||||
if "Result" in data_json and "calendarInfo" in data_json["Result"]:
|
||||
for item in data_json["Result"]["calendarInfo"]:
|
||||
if item.get("date") == target_date and item.get("list"):
|
||||
processed_df = process_func(item["list"])
|
||||
big_df = pd.concat([big_df, processed_df], ignore_index=True)
|
||||
|
||||
return big_df
|
||||
|
||||
|
||||
def _process_economic_data(data_list: list) -> pd.DataFrame:
|
||||
"""处理经济数据"""
|
||||
if not data_list:
|
||||
return pd.DataFrame()
|
||||
temp_df = pd.DataFrame(data_list)
|
||||
rename_dict = {
|
||||
"date": "日期",
|
||||
"time": "时间",
|
||||
"title": "事件",
|
||||
"star": "重要性",
|
||||
"formerVal": "前值",
|
||||
"pubVal": "公布",
|
||||
"region": "地区",
|
||||
"indicateVal": "预期",
|
||||
"country": "国家",
|
||||
"timePeriod": "统计周期",
|
||||
}
|
||||
temp_df.rename(columns=rename_dict, inplace=True)
|
||||
required_cols = ["公布", "预期", "前值", "重要性"]
|
||||
for col in required_cols:
|
||||
if col not in temp_df.columns:
|
||||
temp_df[col] = None
|
||||
available_cols = []
|
||||
for col in [
|
||||
"日期",
|
||||
"时间",
|
||||
"国家",
|
||||
"地区",
|
||||
"事件",
|
||||
"统计周期",
|
||||
"公布",
|
||||
"预期",
|
||||
"前值",
|
||||
"重要性",
|
||||
]:
|
||||
if col in temp_df.columns:
|
||||
available_cols.append(col)
|
||||
if available_cols:
|
||||
temp_df = temp_df[available_cols]
|
||||
for col in ["公布", "预期", "前值", "重要性"]:
|
||||
if col in temp_df.columns:
|
||||
temp_df[col] = pd.to_numeric(temp_df[col], errors="coerce")
|
||||
if "日期" in temp_df.columns:
|
||||
temp_df["日期"] = pd.to_datetime(temp_df["日期"], errors="coerce").dt.date
|
||||
|
||||
return temp_df
|
||||
|
||||
|
||||
def _process_suspend_data(data_list: list) -> pd.DataFrame:
|
||||
"""处理停复牌数据 - 根据实际JSON结构精确修正"""
|
||||
if not data_list:
|
||||
return pd.DataFrame()
|
||||
temp_df = pd.DataFrame(data_list)
|
||||
rename_dict = {
|
||||
"code": "股票代码",
|
||||
"name": "股票简称",
|
||||
"exchange": "交易所代码",
|
||||
"start": "停牌时间",
|
||||
"reason": "停牌事项说明",
|
||||
"marketValue": "市值",
|
||||
"date": "公告日期",
|
||||
"time": "公告时间",
|
||||
"type": "证券类型",
|
||||
"market": "市场类型",
|
||||
"isSkip": "是否跳过",
|
||||
"end": "复牌时间",
|
||||
}
|
||||
temp_df.rename(columns=rename_dict, inplace=True)
|
||||
if "复牌时间" not in temp_df.columns:
|
||||
temp_df["复牌时间"] = "-"
|
||||
temp_df = temp_df[
|
||||
[
|
||||
"股票代码",
|
||||
"股票简称",
|
||||
"交易所代码",
|
||||
"停牌时间",
|
||||
"复牌时间",
|
||||
"停牌事项说明",
|
||||
"市值",
|
||||
"公告日期",
|
||||
"公告时间",
|
||||
"证券类型",
|
||||
"市场类型",
|
||||
"是否跳过",
|
||||
]
|
||||
]
|
||||
return temp_df
|
||||
|
||||
|
||||
def news_economic_baidu(date: str = "20251126", cookie: str = None) -> pd.DataFrame:
|
||||
"""
|
||||
百度股市通-经济数据
|
||||
https://finance.baidu.com/calendar
|
||||
:param date: 查询日期 (格式: YYYYMMDD)
|
||||
:param cookie: cookie
|
||||
:return: 经济数据 pd.DataFrame
|
||||
"""
|
||||
return _baidu_finance_calendar(
|
||||
date=date,
|
||||
cate="economic_data",
|
||||
process_func=_process_economic_data,
|
||||
cookie=cookie,
|
||||
)
|
||||
|
||||
|
||||
def news_trade_notify_suspend_baidu(
|
||||
date: str = "20251126", cookie: str = None
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
百度股市通-交易提醒-停复牌
|
||||
https://finance.baidu.com/calendar
|
||||
:param date: 查询日期 (格式: YYYYMMDD)
|
||||
:param cookie: cookie
|
||||
:return: 停复牌数据DataFrame
|
||||
"""
|
||||
return _baidu_finance_calendar(
|
||||
date=date,
|
||||
cate="notify_suspend",
|
||||
process_func=_process_suspend_data,
|
||||
cookie=cookie,
|
||||
)
|
||||
|
||||
|
||||
def _process_dividend_data(data_list: list) -> pd.DataFrame:
|
||||
"""处理分红派息数据"""
|
||||
if not data_list:
|
||||
return pd.DataFrame()
|
||||
|
||||
temp_df = pd.DataFrame(data_list)
|
||||
|
||||
# 字段映射
|
||||
rename_dict = {
|
||||
"code": "股票代码",
|
||||
"market": "-", # 这个字段在最终结果中会被删除
|
||||
"exchange": "交易所",
|
||||
"name": "股票简称",
|
||||
"diviDate": "除权日",
|
||||
"date": "报告期",
|
||||
"diviCash": "分红",
|
||||
"shareDivide": "送股",
|
||||
"transfer": "转增",
|
||||
"physical": "实物",
|
||||
}
|
||||
temp_df.rename(columns=rename_dict, inplace=True)
|
||||
|
||||
# 确保必要列存在
|
||||
if "分红" not in temp_df.columns:
|
||||
temp_df["分红"] = "-"
|
||||
if "实物" not in temp_df.columns:
|
||||
temp_df["实物"] = "-"
|
||||
if "送股" not in temp_df.columns:
|
||||
temp_df["送股"] = "-"
|
||||
if "转增" not in temp_df.columns:
|
||||
temp_df["转增"] = "-"
|
||||
|
||||
# 选择需要的列
|
||||
temp_df = temp_df[
|
||||
[
|
||||
"股票代码",
|
||||
"除权日",
|
||||
"分红",
|
||||
"送股",
|
||||
"转增",
|
||||
"实物",
|
||||
"交易所",
|
||||
"股票简称",
|
||||
"报告期",
|
||||
]
|
||||
]
|
||||
|
||||
# 日期格式转换
|
||||
if "除权日" in temp_df.columns:
|
||||
temp_df["除权日"] = pd.to_datetime(temp_df["除权日"], errors="coerce").dt.date
|
||||
if "报告期" in temp_df.columns:
|
||||
temp_df["报告期"] = pd.to_datetime(temp_df["报告期"], errors="coerce").dt.date
|
||||
|
||||
return temp_df
|
||||
|
||||
|
||||
def news_trade_notify_dividend_baidu(
|
||||
date: str = "20251126", cookie: str = None
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
百度股市通-交易提醒-分红派息
|
||||
https://finance.baidu.com/calendar
|
||||
:param date: 查询日期 (格式: YYYYMMDD)
|
||||
:param cookie: cookie
|
||||
:return: 交易提醒-分红派息DataFrame
|
||||
"""
|
||||
return _baidu_finance_calendar(
|
||||
date=date,
|
||||
cate="notify_divide",
|
||||
process_func=_process_dividend_data,
|
||||
cookie=cookie,
|
||||
)
|
||||
|
||||
|
||||
def _process_report_data(data_list: list) -> pd.DataFrame:
|
||||
"""处理财报发行数据 - 根据实际JSON结构精确修正"""
|
||||
if not data_list:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 创建DataFrame
|
||||
temp_df = pd.DataFrame(data_list)
|
||||
|
||||
# 精确字段映射 (根据提供的JSON结构)
|
||||
rename_dict = {
|
||||
"code": "股票代码",
|
||||
"name": "股票简称",
|
||||
"exchange": "交易所",
|
||||
"reportType": "财报类型",
|
||||
"time": "发布时间",
|
||||
"marketValue": "市值",
|
||||
"capitalization": "总市值",
|
||||
"date": "发布日期",
|
||||
}
|
||||
temp_df.rename(columns=rename_dict, inplace=True)
|
||||
|
||||
# 确保必要列存在
|
||||
if "财报类型" not in temp_df.columns:
|
||||
temp_df["财报类型"] = "-"
|
||||
if "发布时间" not in temp_df.columns:
|
||||
temp_df["发布时间"] = "-"
|
||||
if "市值" not in temp_df.columns and "总市值" in temp_df.columns:
|
||||
temp_df["市值"] = temp_df["总市值"]
|
||||
|
||||
# 选择并排序列
|
||||
available_cols = []
|
||||
for col in [
|
||||
"股票代码",
|
||||
"股票简称",
|
||||
"交易所",
|
||||
"财报类型",
|
||||
"发布时间",
|
||||
"市值",
|
||||
"发布日期",
|
||||
]:
|
||||
if col in temp_df.columns:
|
||||
available_cols.append(col)
|
||||
|
||||
if available_cols:
|
||||
temp_df = temp_df[available_cols]
|
||||
else:
|
||||
# 如果没有匹配的列,返回空DataFrame
|
||||
return pd.DataFrame()
|
||||
|
||||
# 类型转换
|
||||
if "市值" in temp_df.columns:
|
||||
temp_df["市值"] = pd.to_numeric(temp_df["市值"], errors="coerce")
|
||||
|
||||
if "发布日期" in temp_df.columns:
|
||||
temp_df["发布日期"] = pd.to_datetime(
|
||||
temp_df["发布日期"], errors="coerce"
|
||||
).dt.date
|
||||
|
||||
return temp_df
|
||||
|
||||
|
||||
def news_report_time_baidu(date: str = "20251126", cookie: str = None) -> pd.DataFrame:
|
||||
"""
|
||||
百度股市通-财报发行
|
||||
https://finance.baidu.com/calendar
|
||||
:param date: 查询日期 (格式: YYYYMMDD)
|
||||
:param cookie: cookie
|
||||
:return: 财报发行DataFrame
|
||||
"""
|
||||
return _baidu_finance_calendar(
|
||||
date=date, cate="report_time", process_func=_process_report_data, cookie=cookie
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
news_economic_baidu_df = news_economic_baidu(date="20251126")
|
||||
print(news_economic_baidu_df)
|
||||
|
||||
news_trade_notify_suspend_baidu_df = news_trade_notify_suspend_baidu(
|
||||
date="20251126"
|
||||
)
|
||||
print(news_trade_notify_suspend_baidu_df)
|
||||
|
||||
news_trade_notify_dividend_baidu_df = news_trade_notify_dividend_baidu(
|
||||
date="20251126"
|
||||
)
|
||||
print(news_trade_notify_dividend_baidu_df)
|
||||
|
||||
news_report_time_baidu_df = news_report_time_baidu(date="20251126")
|
||||
print(news_report_time_baidu_df)
|
||||
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
Date: 2024/4/25 17:00
|
||||
Desc: 新闻联播文字稿
|
||||
https://tv.cctv.com/lm/xwlb
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def news_cctv(date: str = "20240424") -> pd.DataFrame:
|
||||
"""
|
||||
新闻联播文字稿
|
||||
https://tv.cctv.com/lm/xwlb
|
||||
:param date: 需要获取数据的日期; 目前 20160203 年后
|
||||
:type date: str
|
||||
:return: 新闻联播文字稿
|
||||
:rtype: pandas.DataFrame
|
||||
"""
|
||||
if int(date) <= int("20130708"):
|
||||
url = f"https://cctv.cntv.cn/lm/xinwenlianbo/{date}.shtml"
|
||||
r = requests.get(url)
|
||||
r.encoding = "gbk"
|
||||
raw_list = re.findall(r"title_array_01\((.*)", r.text)
|
||||
page_url = [
|
||||
re.findall("(http.*)", item)[0].split("'")[0] for item in raw_list[1:]
|
||||
]
|
||||
title_list = []
|
||||
content_list = []
|
||||
headers = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
|
||||
"image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "no-cache",
|
||||
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
|
||||
"Host": "tv.cctv.com",
|
||||
"Pragma": "no-cache",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/92.0.4515.159 Safari/537.36",
|
||||
}
|
||||
for page in tqdm(page_url, leave=False):
|
||||
try:
|
||||
r = requests.get(page, headers=headers)
|
||||
r.encoding = "utf-8"
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
title = soup.find("h3").text
|
||||
content = soup.find("div", attrs={"class": "cnt_bd"}).text
|
||||
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
|
||||
content_list.append(
|
||||
content.strip()
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("(新闻联播):")
|
||||
.strip()
|
||||
.replace("\n", " ")
|
||||
)
|
||||
except: # noqa: E722
|
||||
continue
|
||||
temp_df = pd.DataFrame(
|
||||
data=[[date] * len(title_list), title_list, content_list],
|
||||
index=["date", "title", "content"],
|
||||
).T
|
||||
return temp_df
|
||||
|
||||
elif int(date) < int("20160203"):
|
||||
url = f"https://cctv.cntv.cn/lm/xinwenlianbo/{date}.shtml"
|
||||
r = requests.get(url)
|
||||
r.encoding = "utf-8"
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
page_url = [
|
||||
item.find("a")["href"]
|
||||
for item in soup.find(
|
||||
name="div", attrs={"id": "contentELMT1368521805488378"}
|
||||
).find_all("li")[1:]
|
||||
]
|
||||
title_list = []
|
||||
content_list = []
|
||||
headers = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
|
||||
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "no-cache",
|
||||
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
|
||||
"Host": "tv.cctv.com",
|
||||
"Pragma": "no-cache",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/92.0.4515.159 Safari/537.36",
|
||||
}
|
||||
for page in tqdm(page_url, leave=False):
|
||||
try:
|
||||
r = requests.get(page, headers=headers)
|
||||
r.encoding = "utf-8"
|
||||
soup = BeautifulSoup(r.text, features="lxml")
|
||||
title = soup.find("h3").text
|
||||
content = soup.find(name="div", attrs={"class": "cnt_bd"}).text
|
||||
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
|
||||
content_list.append(
|
||||
content.strip()
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("(新闻联播):")
|
||||
.strip()
|
||||
.replace("\n", " ")
|
||||
)
|
||||
except: # noqa: E722
|
||||
continue
|
||||
temp_df = pd.DataFrame(
|
||||
data=[[date] * len(title_list), title_list, content_list],
|
||||
index=["date", "title", "content"],
|
||||
).T
|
||||
return temp_df
|
||||
elif int(date) > int("20160203"):
|
||||
url = f"https://tv.cctv.com/lm/xwlb/day/{date}.shtml"
|
||||
r = requests.get(url)
|
||||
r.encoding = "utf-8"
|
||||
soup = BeautifulSoup(r.text, "lxml")
|
||||
page_url = [item.find("a")["href"] for item in soup.find_all("li")[1:]]
|
||||
title_list = []
|
||||
content_list = []
|
||||
headers = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
|
||||
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Cache-Control": "no-cache",
|
||||
"Cookie": "cna=DLYSGBDthG4CAbRVCNxSxGT6",
|
||||
"Host": "tv.cctv.com",
|
||||
"Pragma": "no-cache",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/92.0.4515.159 Safari/537.36",
|
||||
}
|
||||
for page in tqdm(page_url, leave=False):
|
||||
try:
|
||||
r = requests.get(page, headers=headers)
|
||||
r.encoding = "utf-8"
|
||||
soup = BeautifulSoup(r.text, features="lxml")
|
||||
if soup.find("h3"):
|
||||
title = soup.find("h3").text
|
||||
else:
|
||||
title = soup.find(name="div", attrs={"class": "tit"}).text
|
||||
if soup.find(name="div", attrs={"class": "cnt_bd"}):
|
||||
content = soup.find(name="div", attrs={"class": "cnt_bd"}).text
|
||||
else:
|
||||
content = soup.find(
|
||||
name="div", attrs={"class": "content_area"}
|
||||
).text
|
||||
title_list.append(title.strip("[视频]").strip().replace("\n", " "))
|
||||
content_list.append(
|
||||
content.strip()
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("央视网消息(新闻联播):")
|
||||
.strip("(新闻联播):")
|
||||
.strip()
|
||||
.replace("\n", " ")
|
||||
)
|
||||
except: # noqa: E722
|
||||
continue
|
||||
temp_df = pd.DataFrame(
|
||||
data=[[date] * len(title_list), title_list, content_list],
|
||||
index=["date", "title", "content"],
|
||||
).T
|
||||
return temp_df
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
news_cctv_df = news_cctv(date="20240424")
|
||||
print(news_cctv_df)
|
||||
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding:utf-8 -*-
|
||||
"""
|
||||
Date: 2025/12/23 17:15
|
||||
Desc: 个股新闻数据
|
||||
https://so.eastmoney.com/news/s?keyword=603777
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pandas as pd
|
||||
from curl_cffi import requests
|
||||
|
||||
|
||||
def stock_news_em(symbol: str = "603777") -> pd.DataFrame:
|
||||
"""
|
||||
东方财富-个股新闻-最近 100 条新闻
|
||||
https://so.eastmoney.com/news/s?keyword=603777
|
||||
:param symbol: 股票代码
|
||||
:type symbol: str
|
||||
:return: 个股新闻
|
||||
:rtype: pandas.DataFrame
|
||||
"""
|
||||
url = "https://search-api-web.eastmoney.com/search/jsonp"
|
||||
inner_param = {
|
||||
"uid": "",
|
||||
"keyword": symbol,
|
||||
"type": ["cmsArticleWebOld"],
|
||||
"client": "web",
|
||||
"clientType": "web",
|
||||
"clientVersion": "curr",
|
||||
"param": {
|
||||
"cmsArticleWebOld": {
|
||||
"searchScope": "default",
|
||||
"sort": "default",
|
||||
"pageIndex": 1,
|
||||
"pageSize": 10,
|
||||
"preTag": "<em>",
|
||||
"postTag": "</em>",
|
||||
}
|
||||
},
|
||||
}
|
||||
params = {
|
||||
"cb": "jQuery35101792940631092459_1764599530165",
|
||||
"param": json.dumps(inner_param, ensure_ascii=False), # 保留中文,
|
||||
"_": "1764599530176",
|
||||
}
|
||||
headers = {
|
||||
"accept": "*/*",
|
||||
"accept-encoding": "gzip, deflate, br, zstd",
|
||||
"accept-language": "en,zh-CN;q=0.9,zh;q=0.8",
|
||||
"cache-control": "no-cache",
|
||||
"connection": "keep-alive",
|
||||
"cookie": "qgqp_b_id=652bf4c98a74e210088f372a17d4e27b; st_nvi=ulN5JAj9FUocz3p4klMME9f20; emshistory=%5B%22603777%22%5D; nid18=010d039dd427dc4d187090491f47d7ad; nid18_create_time=1764582801999; gviem=gSdeY51VWSuTzM3kWaagtf560; gviem_create_time=1764582801999; st_si=55269775884615; st_pvi=66803244437563; st_sp=2025-11-19%2014%3A19%3A16; st_inirUrl=https%3A%2F%2Fso.eastmoney.com%2Fnews%2Fs; st_sn=2; st_psi=20251201223210488-118000300905-0940816858; st_asi=delete",
|
||||
"host": "search-api-web.eastmoney.com",
|
||||
"pragma": "no-cache",
|
||||
"referer": "https://so.eastmoney.com/news/s?keyword=603777",
|
||||
"sec-ch-ua": '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Windows"',
|
||||
"sec-fetch-dest": "script",
|
||||
"sec-fetch-mode": "no-cors",
|
||||
"sec-fetch-site": "same-site",
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
|
||||
}
|
||||
r = requests.get(url, params=params, headers=headers)
|
||||
data_text = r.text
|
||||
data_json = json.loads(
|
||||
data_text.strip("jQuery35101792940631092459_1764599530165(")[:-1]
|
||||
)
|
||||
temp_df = pd.DataFrame(data_json["result"]["cmsArticleWebOld"])
|
||||
temp_df["url"] = "http://finance.eastmoney.com/a/" + temp_df["code"] + ".html"
|
||||
temp_df.rename(
|
||||
columns={
|
||||
"date": "发布时间",
|
||||
"mediaName": "文章来源",
|
||||
"code": "-",
|
||||
"title": "新闻标题",
|
||||
"content": "新闻内容",
|
||||
"url": "新闻链接",
|
||||
"image": "-",
|
||||
},
|
||||
inplace=True,
|
||||
)
|
||||
temp_df["关键词"] = symbol
|
||||
temp_df = temp_df[
|
||||
[
|
||||
"关键词",
|
||||
"新闻标题",
|
||||
"新闻内容",
|
||||
"发布时间",
|
||||
"文章来源",
|
||||
"新闻链接",
|
||||
]
|
||||
]
|
||||
temp_df["新闻标题"] = (
|
||||
temp_df["新闻标题"]
|
||||
.str.replace(r"\(<em>", "", regex=True)
|
||||
.str.replace(r"</em>\)", "", regex=True)
|
||||
)
|
||||
temp_df["新闻标题"] = (
|
||||
temp_df["新闻标题"]
|
||||
.str.replace(r"<em>", "", regex=True)
|
||||
.str.replace(r"</em>", "", regex=True)
|
||||
)
|
||||
temp_df["新闻内容"] = (
|
||||
temp_df["新闻内容"]
|
||||
.str.replace(r"\(<em>", "", regex=True)
|
||||
.str.replace(r"</em>\)", "", regex=True)
|
||||
)
|
||||
temp_df["新闻内容"] = (
|
||||
temp_df["新闻内容"]
|
||||
.str.replace(r"<em>", "", regex=True)
|
||||
.str.replace(r"</em>", "", regex=True)
|
||||
)
|
||||
temp_df["新闻内容"] = temp_df["新闻内容"].str.replace(r"\u3000", "", regex=True)
|
||||
temp_df["新闻内容"] = temp_df["新闻内容"].str.replace(r"\r\n", " ", regex=True)
|
||||
return temp_df
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stock_news_em_df = stock_news_em(symbol="603777")
|
||||
print(stock_news_em_df)
|
||||
Reference in New Issue
Block a user