From 1f9931da7751484c5f0b27e480ec329bf71b5d06 Mon Sep 17 00:00:00 2001 From: Sakurasan <26715255+Sakurasan@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:03:19 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E7=83=AD=E7=82=B9?= =?UTF-8?q?=E8=82=A1=E8=BF=BD=E8=B8=AA=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=87=BA?= =?UTF-8?q?/=E5=AF=BC=E5=85=A5=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/README_HOTSPOT.md | 152 +++++++++++++++++++++ scripts/docker_hotspot.sh | 236 +++++++++++++++++++++++++++++++++ scripts/export_hotspot_data.py | 213 +++++++++++++++++++++++++++++ scripts/hotspot_backup.sh | 110 +++++++++++++++ 4 files changed, 711 insertions(+) create mode 100644 scripts/README_HOTSPOT.md create mode 100755 scripts/docker_hotspot.sh create mode 100755 scripts/export_hotspot_data.py create mode 100755 scripts/hotspot_backup.sh diff --git a/scripts/README_HOTSPOT.md b/scripts/README_HOTSPOT.md new file mode 100644 index 0000000..7596598 --- /dev/null +++ b/scripts/README_HOTSPOT.md @@ -0,0 +1,152 @@ +# 热点股追踪数据导出/导入工具 + +## 功能说明 + +本工具用于导出和导入热点股追踪的历史数据，包括： +- 每日题材涨幅前20 +- 每日核心股票（覆盖多个题材的股票） +- 每日核心股票与题材的关联关系 + +## 快速使用（Docker环境） + +### 1. 导出8月20日的数据 + +```bash +./scripts/docker_hotspot.sh export 2026-08-20 +``` + +### 2. 导入数据 + +```bash +# 导入数据到容器 +./scripts/docker_hotspot.sh import hotspot_2026-08-20.json +``` + +### 3. 查看数据库中的日期 + +```bash +./scripts/docker_hotspot.sh list +``` + +### 4. 备份整个数据库 + +```bash +./scripts/docker_hotspot.sh backup +``` + +### 5. 进入Python shell + +```bash +./scripts/docker_hotspot.sh shell +``` + +## 本地开发环境 + +如果在本地开发环境中，可以使用以下命令： + +```bash +# 导出数据 +python3 scripts/export_hotspot_data.py export --date 2026-08-20 + +# 导入数据 +python3 scripts/export_hotspot_data.py import --file hotspot_2026-08-20.json + +# 列出日期 +python3 scripts/export_hotspot_data.py list +``` + +## 文件说明 + +### Docker脚本 (`docker_hotspot.sh`) + +- `export [日期]`: 从容器导出数据（默认8月20日） +- `import <文件>`: 导入数据文件到容器 +- `list`: 列出所有日期 +- `backup`: 备份整个数据库 +- `shell`: 进入容器的Python shell + +### Python脚本 (`export_hotspot_data.py`) + +- `export --date YYYY-MM-DD`: 导出指定日期的数据 +- `import --file <文件路径>`: 导入数据文件 +- `list`: 列出数据库中所有有数据的日期 + +### Shell脚本 (`hotspot_backup.sh`) + +- `export [日期]`: 导出数据（默认8月20日） +- `import <文件>`: 导入数据文件 +- `list`: 列出所有日期 +- `backup`: 备份整个数据库 + +## 数据格式 + +导出的JSON文件包含以下字段： + +```json +{ + "version": "1.0", + "trade_date": "2026-08-11", + "themes": [...], // 题材数据 + "core_stocks": [...], // 核心股票数据 + "stock_themes": [...], // 股票-题材关联 + "stats": { + "theme_count": 20, + "core_stock_count": 193, + "stock_theme_count": 1205 + } +} +``` + +## 注意事项 + +1. **容器运行**：使用Docker脚本前，请确保容器正在运行 +2. **幂等性**：导入时使用 `INSERT OR IGNORE`，重复导入不会产生重复数据 +3. **数据完整性**：导入前会检查目标日期是否已有数据 +4. **备份建议**：导入前建议先备份数据库 +5. **日期格式**：日期格式必须为 `YYYY-MM-DD`（如 `2026-08-20`） + +## 常见问题 + +### Q: 容器未运行怎么办？ + +A: 启动容器： + +```bash +docker-compose up -d +``` + +### Q: 如何查看容器是否运行？ + +A: 运行以下命令： + +```bash +docker ps | grep auv +``` + +### Q: 如何手动采集8月20日的数据？ + +A: 在容器内运行： + +```bash +docker exec -it auv python3 -c " +import asyncio +from services.daily_collector import collect_daily +asyncio.run(collect_daily('2026-08-20')) +" +``` + +### Q: 如何查看数据库中的日期？ + +A: 运行 `./scripts/docker_hotspot.sh list` + +### Q: 如何备份数据库？ + +A: 运行 `./scripts/docker_hotspot.sh backup` + +### Q: 导入时出现错误怎么办？ + +A: 检查JSON文件格式是否正确，确保包含必要的字段。可以使用以下命令验证JSON： + +```bash +python3 -m json.tool hotspot_2026-08-20.json +``` diff --git a/scripts/docker_hotspot.sh b/scripts/docker_hotspot.sh new file mode 100755 index 0000000..6a384d0 --- /dev/null +++ b/scripts/docker_hotspot.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# 从Docker容器导出热点股数据 + +set -e + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +CONTAINER_NAME="auv" +DB_PATH="/app/backend/data/stock_data.db" + +# 检查容器是否运行 +if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo -e "${RED}❌ 容器 ${CONTAINER_NAME} 未运行${NC}" + exit 1 +fi + +# 检查数据库是否存在 +if ! docker exec "$CONTAINER_NAME" test -f "$DB_PATH"; then + echo -e "${RED}❌ 数据库文件不存在：$DB_PATH${NC}" + exit 1 +fi + +# 显示帮助 +show_help() { + echo "用法：" + echo " $0 export [日期] - 导出指定日期的数据（默认：2026-08-20）" + echo " $0 import <文件> - 导入数据文件到容器" + echo " $0 list - 列出所有有数据的日期" + echo " $0 backup - 备份整个数据库" + echo " $0 shell - 进入容器的Python shell" + echo "" + echo "示例：" + echo " $0 export 2026-08-20" + echo " $0 import hotspot_2026-08-20.json" + echo " $0 backup" +} + +# 导出数据 +export_data() { + local date="${1:-2026-08-20}" + local output_file="hotspot_${date}.json" + + echo -e "${YELLOW}📤 从容器导出 $date 的热点股数据...${NC}" + + # 在容器内执行导出命令 + docker exec "$CONTAINER_NAME" python3 -c " +import sys +sys.path.insert(0, '/app') +from database import get_connection, dict_from_row +import json + +date = '$date' +conn = get_connection() +try: + themes = conn.execute('SELECT * FROM daily_top_themes WHERE trade_date = ? ORDER BY rank ASC', (date,)).fetchall() + core_stocks = conn.execute('SELECT * FROM daily_core_stocks WHERE trade_date = ? ORDER BY rank ASC', (date,)).fetchall() + stock_themes = conn.execute('SELECT * FROM daily_core_stock_themes WHERE trade_date = ?', (date,)).fetchall() + + data = { + 'version': '1.0', + 'trade_date': date, + 'themes': [dict_from_row(r) for r in themes], + 'core_stocks': [dict_from_row(r) for r in core_stocks], + 'stock_themes': [dict_from_row(r) for r in stock_themes], + } + data['stats'] = { + 'theme_count': len(data['themes']), + 'core_stock_count': len(data['core_stocks']), + 'stock_theme_count': len(data['stock_themes']), + } + + print(json.dumps(data, ensure_ascii=False, indent=2)) +finally: + conn.close() +" > "$output_file" + + if [ -f "$output_file" ]; then + echo -e "${GREEN}✅ 导出完成：$output_file${NC}" + echo "文件大小：$(du -h "$output_file" | cut -f1)" + else + echo -e "${RED}❌ 导出失败${NC}" + exit 1 + fi +} + +# 导入数据 +import_data() { + local file="$1" + + if [ -z "$file" ]; then + echo -e "${RED}❌ 请指定要导入的文件${NC}" + exit 1 + fi + + if [ ! -f "$file" ]; then + echo -e "${RED}❌ 文件不存在：$file${NC}" + exit 1 + fi + + echo -e "${YELLOW}📥 导入数据到容器：$file${NC}" + + # 复制文件到容器 + docker cp "$file" "$CONTAINER_NAME:/tmp/import_data.json" + + # 在容器内执行导入 + docker exec "$CONTAINER_NAME" python3 -c " +import sys +import json +sys.path.insert(0, '/app') +from database import get_connection + +with open('/tmp/import_data.json', 'r', encoding='utf-8') as f: + data = json.load(f) + +date = data['trade_date'] +print(f'📅 导入日期：{date}') +print(f' 题材数量：{len(data[\"themes\"])}') +print(f' 核心股票：{len(data[\"core_stocks\"])}') +print(f' 股票-题材关联：{len(data[\"stock_themes\"])}') + +conn = get_connection() +try: + for item in data['themes']: + item.pop('id', None) + item.pop('created_at', None) + conn.execute( + 'INSERT OR IGNORE INTO daily_top_themes (trade_date, theme_code, theme_name, bf3, hot_rank, rank) VALUES (?,?,?,?,?,?)', + (item['trade_date'], item['theme_code'], item['theme_name'], item['bf3'], item['hot_rank'], item['rank']), + ) + + for item in data['core_stocks']: + item.pop('id', None) + item.pop('created_at', None) + conn.execute( + 'INSERT OR IGNORE INTO daily_core_stocks (trade_date, stock_code, stock_name, f3, cover_count, rank) VALUES (?,?,?,?,?,?)', + (item['trade_date'], item['stock_code'], item['stock_name'], item['f3'], item['cover_count'], item['rank']), + ) + + for item in data['stock_themes']: + item.pop('id', None) + item.pop('created_at', None) + conn.execute( + 'INSERT OR IGNORE INTO daily_core_stock_themes (trade_date, stock_code, theme_code, theme_name) VALUES (?,?,?,?)', + (item['trade_date'], item['stock_code'], item['theme_code'], item['theme_name']), + ) + + conn.commit() + print(f'\\n✅ 导入成功：{date}') +except Exception as e: + conn.rollback() + print(f'\\n❌ 导入失败：{e}') +finally: + conn.close() +" +} + +# 列出日期 +list_dates() { + echo -e "${YELLOW}📅 列出容器中的日期...${NC}" + docker exec "$CONTAINER_NAME" python3 -c " +import sys +sys.path.insert(0, '/app') +from database import get_connection + +conn = get_connection() +try: + rows = conn.execute('SELECT DISTINCT trade_date FROM daily_core_stocks ORDER BY trade_date DESC').fetchall() + if rows: + print('📅 数据库中的日期：') + for row in rows: + print(f' {row[\"trade_date\"]}') + else: + print('📭 数据库中暂无数据') +finally: + conn.close() +" +} + +# 备份数据库 +backup_database() { + local timestamp=$(date +%Y%m%d_%H%M%S) + local backup_file="stock_data_${timestamp}.db" + + echo -e "${YELLOW}💾 备份数据库...${NC}" + docker cp "$CONTAINER_NAME:$DB_PATH" "$backup_file" + + if [ -f "$backup_file" ]; then + echo -e "${GREEN}✅ 备份完成：$backup_file${NC}" + echo "文件大小：$(du -h "$backup_file" | cut -f1)" + else + echo -e "${RED}❌ 备份失败${NC}" + exit 1 + fi +} + +# 进入Python shell +enter_shell() { + echo -e "${YELLOW}🐍 进入容器Python shell...${NC}" + docker exec -it "$CONTAINER_NAME" python3 -c " +import sys +sys.path.insert(0, '/app') +from database import get_connection +print('Python shell 已启动') +print('可用变量：') +print(' conn - 数据库连接') +print(' get_connection - 获取新连接函数') +print() +conn = get_connection() +" +} + +# 主逻辑 +case "${1:-help}" in + export) + export_data "$2" + ;; + import) + import_data "$2" + ;; + list) + list_dates + ;; + backup) + backup_database + ;; + shell) + enter_shell + ;; + *) + show_help + ;; +esac diff --git a/scripts/export_hotspot_data.py b/scripts/export_hotspot_data.py new file mode 100755 index 0000000..4471b3f --- /dev/null +++ b/scripts/export_hotspot_data.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""热点股追踪数据导出/导入脚本 + +用法： + 导出8月20日数据： + python scripts/export_hotspot_data.py export --date 2026-08-20 + + 导入数据： + python scripts/export_hotspot_data.py import --file hotspot_2026-08-20.json + + 查看数据库中有哪些日期的数据： + python scripts/export_hotspot_data.py list +""" + +import argparse +import json +import os +import sys +from datetime import datetime +from typing import Optional + +# 添加项目路径 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend")) + +from database import get_connection, dict_from_row + + +def export_data(date: str, output_file: Optional[str] = None) -> dict: + """导出指定日期的热点股追踪数据""" + conn = get_connection() + try: + # 1. 导出每日题材前20 + themes = conn.execute( + "SELECT * FROM daily_top_themes WHERE trade_date = ? ORDER BY rank ASC", + (date,), + ).fetchall() + + # 2. 导出每日核心股票 + core_stocks = conn.execute( + "SELECT * FROM daily_core_stocks WHERE trade_date = ? ORDER BY rank ASC", + (date,), + ).fetchall() + + # 3. 导出每日核心股票与题材的关联 + stock_themes = conn.execute( + "SELECT * FROM daily_core_stock_themes WHERE trade_date = ?", + (date,), + ).fetchall() + + data = { + "version": "1.0", + "export_time": datetime.now().isoformat(), + "trade_date": date, + "themes": [dict_from_row(r) for r in themes], + "core_stocks": [dict_from_row(r) for r in core_stocks], + "stock_themes": [dict_from_row(r) for r in stock_themes], + } + + # 统计信息 + data["stats"] = { + "theme_count": len(data["themes"]), + "core_stock_count": len(data["core_stocks"]), + "stock_theme_count": len(data["stock_themes"]), + } + + # 保存到文件 + if output_file is None: + output_file = f"hotspot_{date}.json" + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + print(f"✅ 导出成功：{output_file}") + print(f" 题材数量：{data['stats']['theme_count']}") + print(f" 核心股票：{data['stats']['core_stock_count']}") + print(f" 股票-题材关联：{data['stats']['stock_theme_count']}") + + return data + finally: + conn.close() + + +def import_data(input_file: str, dry_run: bool = False) -> bool: + """导入热点股追踪数据""" + if not os.path.exists(input_file): + print(f"❌ 文件不存在：{input_file}") + return False + + with open(input_file, "r", encoding="utf-8") as f: + data = json.load(f) + + # 验证数据格式 + required_keys = ["version", "trade_date", "themes", "core_stocks", "stock_themes"] + if not all(k in data for k in required_keys): + print("❌ 数据格式错误：缺少必要字段") + return False + + date = data["trade_date"] + print(f"📅 导入日期：{date}") + print(f" 题材数量：{len(data['themes'])}") + print(f" 核心股票：{len(data['core_stocks'])}") + print(f" 股票-题材关联：{len(data['stock_themes'])}") + + if dry_run: + print("\n🔍 试运行模式，不写入数据库") + return True + + # 检查是否已存在 + conn = get_connection() + try: + existing = conn.execute( + "SELECT 1 FROM daily_core_stocks WHERE trade_date = ? LIMIT 1", + (date,), + ).fetchone() + + if existing: + print(f"\n⚠️ {date} 的数据已存在，将跳过重复数据（使用 INSERT OR IGNORE）") + finally: + conn.close() + + # 写入数据库 + conn = get_connection() + try: + # 导入每日题材前20 + for item in data["themes"]: + # 移除自动生成的id和created_at + item.pop("id", None) + item.pop("created_at", None) + conn.execute( + "INSERT OR IGNORE INTO daily_top_themes (trade_date, theme_code, theme_name, bf3, hot_rank, rank) VALUES (?,?,?,?,?,?)", + (item["trade_date"], item["theme_code"], item["theme_name"], item["bf3"], item["hot_rank"], item["rank"]), + ) + + # 导入每日核心股票 + for item in data["core_stocks"]: + # 移除自动生成的id和created_at + item.pop("id", None) + item.pop("created_at", None) + conn.execute( + "INSERT OR IGNORE INTO daily_core_stocks (trade_date, stock_code, stock_name, f3, cover_count, rank) VALUES (?,?,?,?,?,?)", + (item["trade_date"], item["stock_code"], item["stock_name"], item["f3"], item["cover_count"], item["rank"]), + ) + + # 导入每日核心股票与题材的关联 + for item in data["stock_themes"]: + # 移除自动生成的id和created_at + item.pop("id", None) + item.pop("created_at", None) + conn.execute( + "INSERT OR IGNORE INTO daily_core_stock_themes (trade_date, stock_code, theme_code, theme_name) VALUES (?,?,?,?)", + (item["trade_date"], item["stock_code"], item["theme_code"], item["theme_name"]), + ) + + conn.commit() + print(f"\n✅ 导入成功：{date}") + return True + except Exception as e: + conn.rollback() + print(f"\n❌ 导入失败：{e}") + return False + finally: + conn.close() + + +def list_dates() -> list[str]: + """列出数据库中所有有数据的日期""" + conn = get_connection() + try: + rows = conn.execute( + "SELECT DISTINCT trade_date FROM daily_core_stocks ORDER BY trade_date DESC" + ).fetchall() + return [row["trade_date"] for row in rows] + finally: + conn.close() + + +def main(): + parser = argparse.ArgumentParser(description="热点股追踪数据导出/导入工具") + subparsers = parser.add_subparsers(dest="command", help="可用命令") + + # 导出命令 + export_parser = subparsers.add_parser("export", help="导出数据") + export_parser.add_argument("--date", required=True, help="交易日期 (YYYY-MM-DD)") + export_parser.add_argument("--output", "-o", help="输出文件路径") + + # 导入命令 + import_parser = subparsers.add_parser("import", help="导入数据") + import_parser.add_argument("--file", "-f", required=True, help="输入文件路径") + import_parser.add_argument("--dry-run", action="store_true", help="试运行，不写入数据库") + + # 列表命令 + subparsers.add_parser("list", help="列出所有有数据的日期") + + args = parser.parse_args() + + if args.command == "export": + export_data(args.date, args.output) + elif args.command == "import": + import_data(args.file, args.dry_run) + elif args.command == "list": + dates = list_dates() + if dates: + print("📅 数据库中的日期：") + for d in dates: + print(f" {d}") + else: + print("📭 数据库中暂无数据") + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/scripts/hotspot_backup.sh b/scripts/hotspot_backup.sh new file mode 100755 index 0000000..5d95afd --- /dev/null +++ b/scripts/hotspot_backup.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# 热点股追踪数据导出/导入快捷脚本 + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +PYTHON_SCRIPT="$SCRIPT_DIR/export_hotspot_data.py" + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +show_help() { + echo "用法：" + echo " $0 export [日期] - 导出指定日期的数据（默认：2026-08-20）" + echo " $0 import <文件> - 导入数据文件" + echo " $0 list - 列出所有有数据的日期" + echo " $0 backup - 备份整个数据库" + echo "" + echo "示例：" + echo " $0 export 2026-08-20" + echo " $0 import hotspot_2026-08-20.json" + echo " $0 backup" +} + +export_data() { + local date="${1:-2026-08-20}" + local output_file="hotspot_${date}.json" + + echo -e "${YELLOW}📤 导出 $date 的热点股数据...${NC}" + python3 "$PYTHON_SCRIPT" export --date "$date" --output "$output_file" + + if [ -f "$output_file" ]; then + echo -e "${GREEN}✅ 导出完成：$output_file${NC}" + echo "文件大小：$(du -h "$output_file" | cut -f1)" + else + echo -e "${RED}❌ 导出失败${NC}" + exit 1 + fi +} + +import_data() { + local file="$1" + + if [ -z "$file" ]; then + echo -e "${RED}❌ 请指定要导入的文件${NC}" + exit 1 + fi + + if [ ! -f "$file" ]; then + echo -e "${RED}❌ 文件不存在：$file${NC}" + exit 1 + fi + + echo -e "${YELLOW}📥 导入数据：$file${NC}" + python3 "$PYTHON_SCRIPT" import --file "$file" +} + +list_dates() { + echo -e "${YELLOW}📅 列出数据库中的日期...${NC}" + python3 "$PYTHON_SCRIPT" list +} + +backup_database() { + local db_path="$PROJECT_DIR/backend/data/stock_data.db" + local backup_dir="$PROJECT_DIR/backups" + local timestamp=$(date +%Y%m%d_%H%M%S) + local backup_file="$backup_dir/stock_data_${timestamp}.db" + + if [ ! -f "$db_path" ]; then + echo -e "${RED}❌ 数据库文件不存在：$db_path${NC}" + exit 1 + fi + + # 创建备份目录 + mkdir -p "$backup_dir" + + # 复制数据库 + cp "$db_path" "$backup_file" + + if [ -f "$backup_file" ]; then + echo -e "${GREEN}✅ 数据库备份完成：$backup_file${NC}" + echo "文件大小：$(du -h "$backup_file" | cut -f1)" + else + echo -e "${RED}❌ 备份失败${NC}" + exit 1 + fi +} + +# 主逻辑 +case "${1:-help}" in + export) + export_data "$2" + ;; + import) + import_data "$2" + ;; + list) + list_dates + ;; + backup) + backup_database + ;; + *) + show_help + ;; +esac