feat: 添加热点股追踪数据导出/导入脚本

This commit is contained in:
Sakurasan
2026-08-21 19:03:44 +08:00
parent bb2be17ae2
commit 1f9931da77
4 changed files with 711 additions and 0 deletions
+152
View File
@@ -0,0 +1,152 @@
# 热点股追踪数据导出/导入工具
## 功能说明
本工具用于导出和导入热点股追踪的历史数据,包括:
- 每日题材涨幅前20
- 每日核心股票(覆盖多个题材的股票)
- 每日核心股票与题材的关联关系
## 快速使用(Docker环境)
### 1. 导出8月20日的数据
```bash
./scripts/docker_hotspot.sh export 2026-08-20
```
### 2. 导入数据
```bash
# 导入数据到容器
./scripts/docker_hotspot.sh import hotspot_2026-08-20.json
```
### 3. 查看数据库中的日期
```bash
./scripts/docker_hotspot.sh list
```
### 4. 备份整个数据库
```bash
./scripts/docker_hotspot.sh backup
```
### 5. 进入Python shell
```bash
./scripts/docker_hotspot.sh shell
```
## 本地开发环境
如果在本地开发环境中,可以使用以下命令:
```bash
# 导出数据
python3 scripts/export_hotspot_data.py export --date 2026-08-20
# 导入数据
python3 scripts/export_hotspot_data.py import --file hotspot_2026-08-20.json
# 列出日期
python3 scripts/export_hotspot_data.py list
```
## 文件说明
### Docker脚本 (`docker_hotspot.sh`)
- `export [日期]`: 从容器导出数据(默认8月20日)
- `import <文件>`: 导入数据文件到容器
- `list`: 列出所有日期
- `backup`: 备份整个数据库
- `shell`: 进入容器的Python shell
### Python脚本 (`export_hotspot_data.py`)
- `export --date YYYY-MM-DD`: 导出指定日期的数据
- `import --file <文件路径>`: 导入数据文件
- `list`: 列出数据库中所有有数据的日期
### Shell脚本 (`hotspot_backup.sh`)
- `export [日期]`: 导出数据(默认8月20日)
- `import <文件>`: 导入数据文件
- `list`: 列出所有日期
- `backup`: 备份整个数据库
## 数据格式
导出的JSON文件包含以下字段:
```json
{
"version": "1.0",
"trade_date": "2026-08-11",
"themes": [...], // 题材数据
"core_stocks": [...], // 核心股票数据
"stock_themes": [...], // 股票-题材关联
"stats": {
"theme_count": 20,
"core_stock_count": 193,
"stock_theme_count": 1205
}
}
```
## 注意事项
1. **容器运行**:使用Docker脚本前,请确保容器正在运行
2. **幂等性**:导入时使用 `INSERT OR IGNORE`,重复导入不会产生重复数据
3. **数据完整性**:导入前会检查目标日期是否已有数据
4. **备份建议**:导入前建议先备份数据库
5. **日期格式**:日期格式必须为 `YYYY-MM-DD`(如 `2026-08-20`)
## 常见问题
### Q: 容器未运行怎么办?
A: 启动容器:
```bash
docker-compose up -d
```
### Q: 如何查看容器是否运行?
A: 运行以下命令:
```bash
docker ps | grep auv
```
### Q: 如何手动采集8月20日的数据?
A: 在容器内运行:
```bash
docker exec -it auv python3 -c "
import asyncio
from services.daily_collector import collect_daily
asyncio.run(collect_daily('2026-08-20'))
"
```
### Q: 如何查看数据库中的日期?
A: 运行 `./scripts/docker_hotspot.sh list`
### Q: 如何备份数据库?
A: 运行 `./scripts/docker_hotspot.sh backup`
### Q: 导入时出现错误怎么办?
A: 检查JSON文件格式是否正确,确保包含必要的字段。可以使用以下命令验证JSON:
```bash
python3 -m json.tool hotspot_2026-08-20.json
```
+236
View File
@@ -0,0 +1,236 @@
#!/bin/bash
# 从Docker容器导出热点股数据
set -e
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
CONTAINER_NAME="auv"
DB_PATH="/app/backend/data/stock_data.db"
# 检查容器是否运行
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
echo -e "${RED}❌ 容器 ${CONTAINER_NAME} 未运行${NC}"
exit 1
fi
# 检查数据库是否存在
if ! docker exec "$CONTAINER_NAME" test -f "$DB_PATH"; then
echo -e "${RED}❌ 数据库文件不存在:$DB_PATH${NC}"
exit 1
fi
# 显示帮助
show_help() {
echo "用法:"
echo " $0 export [日期] - 导出指定日期的数据(默认:2026-08-20)"
echo " $0 import <文件> - 导入数据文件到容器"
echo " $0 list - 列出所有有数据的日期"
echo " $0 backup - 备份整个数据库"
echo " $0 shell - 进入容器的Python shell"
echo ""
echo "示例:"
echo " $0 export 2026-08-20"
echo " $0 import hotspot_2026-08-20.json"
echo " $0 backup"
}
# 导出数据
export_data() {
local date="${1:-2026-08-20}"
local output_file="hotspot_${date}.json"
echo -e "${YELLOW}📤 从容器导出 $date 的热点股数据...${NC}"
# 在容器内执行导出命令
docker exec "$CONTAINER_NAME" python3 -c "
import sys
sys.path.insert(0, '/app')
from database import get_connection, dict_from_row
import json
date = '$date'
conn = get_connection()
try:
themes = conn.execute('SELECT * FROM daily_top_themes WHERE trade_date = ? ORDER BY rank ASC', (date,)).fetchall()
core_stocks = conn.execute('SELECT * FROM daily_core_stocks WHERE trade_date = ? ORDER BY rank ASC', (date,)).fetchall()
stock_themes = conn.execute('SELECT * FROM daily_core_stock_themes WHERE trade_date = ?', (date,)).fetchall()
data = {
'version': '1.0',
'trade_date': date,
'themes': [dict_from_row(r) for r in themes],
'core_stocks': [dict_from_row(r) for r in core_stocks],
'stock_themes': [dict_from_row(r) for r in stock_themes],
}
data['stats'] = {
'theme_count': len(data['themes']),
'core_stock_count': len(data['core_stocks']),
'stock_theme_count': len(data['stock_themes']),
}
print(json.dumps(data, ensure_ascii=False, indent=2))
finally:
conn.close()
" > "$output_file"
if [ -f "$output_file" ]; then
echo -e "${GREEN}✅ 导出完成:$output_file${NC}"
echo "文件大小:$(du -h "$output_file" | cut -f1)"
else
echo -e "${RED}❌ 导出失败${NC}"
exit 1
fi
}
# 导入数据
import_data() {
local file="$1"
if [ -z "$file" ]; then
echo -e "${RED}❌ 请指定要导入的文件${NC}"
exit 1
fi
if [ ! -f "$file" ]; then
echo -e "${RED}❌ 文件不存在:$file${NC}"
exit 1
fi
echo -e "${YELLOW}📥 导入数据到容器:$file${NC}"
# 复制文件到容器
docker cp "$file" "$CONTAINER_NAME:/tmp/import_data.json"
# 在容器内执行导入
docker exec "$CONTAINER_NAME" python3 -c "
import sys
import json
sys.path.insert(0, '/app')
from database import get_connection
with open('/tmp/import_data.json', 'r', encoding='utf-8') as f:
data = json.load(f)
date = data['trade_date']
print(f'📅 导入日期:{date}')
print(f' 题材数量:{len(data[\"themes\"])}')
print(f' 核心股票:{len(data[\"core_stocks\"])}')
print(f' 股票-题材关联:{len(data[\"stock_themes\"])}')
conn = get_connection()
try:
for item in data['themes']:
item.pop('id', None)
item.pop('created_at', None)
conn.execute(
'INSERT OR IGNORE INTO daily_top_themes (trade_date, theme_code, theme_name, bf3, hot_rank, rank) VALUES (?,?,?,?,?,?)',
(item['trade_date'], item['theme_code'], item['theme_name'], item['bf3'], item['hot_rank'], item['rank']),
)
for item in data['core_stocks']:
item.pop('id', None)
item.pop('created_at', None)
conn.execute(
'INSERT OR IGNORE INTO daily_core_stocks (trade_date, stock_code, stock_name, f3, cover_count, rank) VALUES (?,?,?,?,?,?)',
(item['trade_date'], item['stock_code'], item['stock_name'], item['f3'], item['cover_count'], item['rank']),
)
for item in data['stock_themes']:
item.pop('id', None)
item.pop('created_at', None)
conn.execute(
'INSERT OR IGNORE INTO daily_core_stock_themes (trade_date, stock_code, theme_code, theme_name) VALUES (?,?,?,?)',
(item['trade_date'], item['stock_code'], item['theme_code'], item['theme_name']),
)
conn.commit()
print(f'\\n✅ 导入成功:{date}')
except Exception as e:
conn.rollback()
print(f'\\n❌ 导入失败:{e}')
finally:
conn.close()
"
}
# 列出日期
list_dates() {
echo -e "${YELLOW}📅 列出容器中的日期...${NC}"
docker exec "$CONTAINER_NAME" python3 -c "
import sys
sys.path.insert(0, '/app')
from database import get_connection
conn = get_connection()
try:
rows = conn.execute('SELECT DISTINCT trade_date FROM daily_core_stocks ORDER BY trade_date DESC').fetchall()
if rows:
print('📅 数据库中的日期:')
for row in rows:
print(f' {row[\"trade_date\"]}')
else:
print('📭 数据库中暂无数据')
finally:
conn.close()
"
}
# 备份数据库
backup_database() {
local timestamp=$(date +%Y%m%d_%H%M%S)
local backup_file="stock_data_${timestamp}.db"
echo -e "${YELLOW}💾 备份数据库...${NC}"
docker cp "$CONTAINER_NAME:$DB_PATH" "$backup_file"
if [ -f "$backup_file" ]; then
echo -e "${GREEN}✅ 备份完成:$backup_file${NC}"
echo "文件大小:$(du -h "$backup_file" | cut -f1)"
else
echo -e "${RED}❌ 备份失败${NC}"
exit 1
fi
}
# 进入Python shell
enter_shell() {
echo -e "${YELLOW}🐍 进入容器Python shell...${NC}"
docker exec -it "$CONTAINER_NAME" python3 -c "
import sys
sys.path.insert(0, '/app')
from database import get_connection
print('Python shell 已启动')
print('可用变量:')
print(' conn - 数据库连接')
print(' get_connection - 获取新连接函数')
print()
conn = get_connection()
"
}
# 主逻辑
case "${1:-help}" in
export)
export_data "$2"
;;
import)
import_data "$2"
;;
list)
list_dates
;;
backup)
backup_database
;;
shell)
enter_shell
;;
*)
show_help
;;
esac
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""热点股追踪数据导出/导入脚本
用法:
导出8月20日数据:
python scripts/export_hotspot_data.py export --date 2026-08-20
导入数据:
python scripts/export_hotspot_data.py import --file hotspot_2026-08-20.json
查看数据库中有哪些日期的数据:
python scripts/export_hotspot_data.py list
"""
import argparse
import json
import os
import sys
from datetime import datetime
from typing import Optional
# 添加项目路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "backend"))
from database import get_connection, dict_from_row
def export_data(date: str, output_file: Optional[str] = None) -> dict:
"""导出指定日期的热点股追踪数据"""
conn = get_connection()
try:
# 1. 导出每日题材前20
themes = conn.execute(
"SELECT * FROM daily_top_themes WHERE trade_date = ? ORDER BY rank ASC",
(date,),
).fetchall()
# 2. 导出每日核心股票
core_stocks = conn.execute(
"SELECT * FROM daily_core_stocks WHERE trade_date = ? ORDER BY rank ASC",
(date,),
).fetchall()
# 3. 导出每日核心股票与题材的关联
stock_themes = conn.execute(
"SELECT * FROM daily_core_stock_themes WHERE trade_date = ?",
(date,),
).fetchall()
data = {
"version": "1.0",
"export_time": datetime.now().isoformat(),
"trade_date": date,
"themes": [dict_from_row(r) for r in themes],
"core_stocks": [dict_from_row(r) for r in core_stocks],
"stock_themes": [dict_from_row(r) for r in stock_themes],
}
# 统计信息
data["stats"] = {
"theme_count": len(data["themes"]),
"core_stock_count": len(data["core_stocks"]),
"stock_theme_count": len(data["stock_themes"]),
}
# 保存到文件
if output_file is None:
output_file = f"hotspot_{date}.json"
with open(output_file, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
print(f"✅ 导出成功:{output_file}")
print(f" 题材数量:{data['stats']['theme_count']}")
print(f" 核心股票:{data['stats']['core_stock_count']}")
print(f" 股票-题材关联:{data['stats']['stock_theme_count']}")
return data
finally:
conn.close()
def import_data(input_file: str, dry_run: bool = False) -> bool:
"""导入热点股追踪数据"""
if not os.path.exists(input_file):
print(f"❌ 文件不存在:{input_file}")
return False
with open(input_file, "r", encoding="utf-8") as f:
data = json.load(f)
# 验证数据格式
required_keys = ["version", "trade_date", "themes", "core_stocks", "stock_themes"]
if not all(k in data for k in required_keys):
print("❌ 数据格式错误:缺少必要字段")
return False
date = data["trade_date"]
print(f"📅 导入日期:{date}")
print(f" 题材数量:{len(data['themes'])}")
print(f" 核心股票:{len(data['core_stocks'])}")
print(f" 股票-题材关联:{len(data['stock_themes'])}")
if dry_run:
print("\n🔍 试运行模式,不写入数据库")
return True
# 检查是否已存在
conn = get_connection()
try:
existing = conn.execute(
"SELECT 1 FROM daily_core_stocks WHERE trade_date = ? LIMIT 1",
(date,),
).fetchone()
if existing:
print(f"\n⚠️ {date} 的数据已存在,将跳过重复数据(使用 INSERT OR IGNORE)")
finally:
conn.close()
# 写入数据库
conn = get_connection()
try:
# 导入每日题材前20
for item in data["themes"]:
# 移除自动生成的id和created_at
item.pop("id", None)
item.pop("created_at", None)
conn.execute(
"INSERT OR IGNORE INTO daily_top_themes (trade_date, theme_code, theme_name, bf3, hot_rank, rank) VALUES (?,?,?,?,?,?)",
(item["trade_date"], item["theme_code"], item["theme_name"], item["bf3"], item["hot_rank"], item["rank"]),
)
# 导入每日核心股票
for item in data["core_stocks"]:
# 移除自动生成的id和created_at
item.pop("id", None)
item.pop("created_at", None)
conn.execute(
"INSERT OR IGNORE INTO daily_core_stocks (trade_date, stock_code, stock_name, f3, cover_count, rank) VALUES (?,?,?,?,?,?)",
(item["trade_date"], item["stock_code"], item["stock_name"], item["f3"], item["cover_count"], item["rank"]),
)
# 导入每日核心股票与题材的关联
for item in data["stock_themes"]:
# 移除自动生成的id和created_at
item.pop("id", None)
item.pop("created_at", None)
conn.execute(
"INSERT OR IGNORE INTO daily_core_stock_themes (trade_date, stock_code, theme_code, theme_name) VALUES (?,?,?,?)",
(item["trade_date"], item["stock_code"], item["theme_code"], item["theme_name"]),
)
conn.commit()
print(f"\n✅ 导入成功:{date}")
return True
except Exception as e:
conn.rollback()
print(f"\n❌ 导入失败:{e}")
return False
finally:
conn.close()
def list_dates() -> list[str]:
"""列出数据库中所有有数据的日期"""
conn = get_connection()
try:
rows = conn.execute(
"SELECT DISTINCT trade_date FROM daily_core_stocks ORDER BY trade_date DESC"
).fetchall()
return [row["trade_date"] for row in rows]
finally:
conn.close()
def main():
parser = argparse.ArgumentParser(description="热点股追踪数据导出/导入工具")
subparsers = parser.add_subparsers(dest="command", help="可用命令")
# 导出命令
export_parser = subparsers.add_parser("export", help="导出数据")
export_parser.add_argument("--date", required=True, help="交易日期 (YYYY-MM-DD)")
export_parser.add_argument("--output", "-o", help="输出文件路径")
# 导入命令
import_parser = subparsers.add_parser("import", help="导入数据")
import_parser.add_argument("--file", "-f", required=True, help="输入文件路径")
import_parser.add_argument("--dry-run", action="store_true", help="试运行,不写入数据库")
# 列表命令
subparsers.add_parser("list", help="列出所有有数据的日期")
args = parser.parse_args()
if args.command == "export":
export_data(args.date, args.output)
elif args.command == "import":
import_data(args.file, args.dry_run)
elif args.command == "list":
dates = list_dates()
if dates:
print("📅 数据库中的日期:")
for d in dates:
print(f" {d}")
else:
print("📭 数据库中暂无数据")
else:
parser.print_help()
if __name__ == "__main__":
main()
+110
View File
@@ -0,0 +1,110 @@
#!/bin/bash
# 热点股追踪数据导出/导入快捷脚本
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
PYTHON_SCRIPT="$SCRIPT_DIR/export_hotspot_data.py"
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
show_help() {
echo "用法:"
echo " $0 export [日期] - 导出指定日期的数据(默认:2026-08-20)"
echo " $0 import <文件> - 导入数据文件"
echo " $0 list - 列出所有有数据的日期"
echo " $0 backup - 备份整个数据库"
echo ""
echo "示例:"
echo " $0 export 2026-08-20"
echo " $0 import hotspot_2026-08-20.json"
echo " $0 backup"
}
export_data() {
local date="${1:-2026-08-20}"
local output_file="hotspot_${date}.json"
echo -e "${YELLOW}📤 导出 $date 的热点股数据...${NC}"
python3 "$PYTHON_SCRIPT" export --date "$date" --output "$output_file"
if [ -f "$output_file" ]; then
echo -e "${GREEN}✅ 导出完成:$output_file${NC}"
echo "文件大小:$(du -h "$output_file" | cut -f1)"
else
echo -e "${RED}❌ 导出失败${NC}"
exit 1
fi
}
import_data() {
local file="$1"
if [ -z "$file" ]; then
echo -e "${RED}❌ 请指定要导入的文件${NC}"
exit 1
fi
if [ ! -f "$file" ]; then
echo -e "${RED}❌ 文件不存在:$file${NC}"
exit 1
fi
echo -e "${YELLOW}📥 导入数据:$file${NC}"
python3 "$PYTHON_SCRIPT" import --file "$file"
}
list_dates() {
echo -e "${YELLOW}📅 列出数据库中的日期...${NC}"
python3 "$PYTHON_SCRIPT" list
}
backup_database() {
local db_path="$PROJECT_DIR/backend/data/stock_data.db"
local backup_dir="$PROJECT_DIR/backups"
local timestamp=$(date +%Y%m%d_%H%M%S)
local backup_file="$backup_dir/stock_data_${timestamp}.db"
if [ ! -f "$db_path" ]; then
echo -e "${RED}❌ 数据库文件不存在:$db_path${NC}"
exit 1
fi
# 创建备份目录
mkdir -p "$backup_dir"
# 复制数据库
cp "$db_path" "$backup_file"
if [ -f "$backup_file" ]; then
echo -e "${GREEN}✅ 数据库备份完成:$backup_file${NC}"
echo "文件大小:$(du -h "$backup_file" | cut -f1)"
else
echo -e "${RED}❌ 备份失败${NC}"
exit 1
fi
}
# 主逻辑
case "${1:-help}" in
export)
export_data "$2"
;;
import)
import_data "$2"
;;
list)
list_dates
;;
backup)
backup_database
;;
*)
show_help
;;
esac