diff --git a/.gitignore b/.gitignore index 01de6aab3..a4ae29559 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,11 @@ log/* data/* !data/.gitkeep -conf/.env +**/.env .pyc __pycache__/ .cursorignore .cursor +/conf/*/* +!/conf/group/* +server.log diff --git a/.qoder/settings.json b/.qoder/settings.json new file mode 100644 index 000000000..0ee38cfd2 --- /dev/null +++ b/.qoder/settings.json @@ -0,0 +1,14 @@ +{ + "permissions": { + "ask": [ + "Read(!./**)", + "Edit(!./**)" + ], + "allow": [ + "Read(./**)", + "Edit(./**)" + ] + }, + "memoryImport": {}, + "monitoring": {} +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..3c16b4348 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,128 @@ +# CLAUDE.md + +本文件为 Claude Code (claude.ai/code) 在此代码库中工作时提供指导。 + +## 开发命令 + +### Docker 开发 +```bash +# 启动所有服务 +docker-compose up -d + +# 停止服务 +docker-compose stop + +# 查看服务状态 +docker-compose ps + +# 重启服务 +docker-compose restart + +# 清理容器 +docker-compose rm +``` + +### 本地开发 +```bash +# 启动 Flask API 服务器（端口 5001） +python api.py + +# 启动 Streamlit 仪表板（端口 5002） +streamlit run ui.py --server.port=5002 --server.address=0.0.0.0 +``` + +### 测试 +```bash +# 运行所有测试 +python -m unittest discover -s test -p "test_*.py" -v + +# 运行特定模块测试 +python -m unittest test.biz.queue.test_whitelist -v +``` + +### 代码库审查工具 +```bash +# 交互式完整代码库审查 +python -m biz.cmd.review +``` + +## 架构概览 + +### 核心组件 +- **Flask API 服务器** (`api.py`)：处理 GitLab/GitHub webhook 和审查请求 +- **Streamlit 仪表板** (`ui.py`)：审查统计和日志的可视化界面 +- **多容器部署**：Supervisor 管理的进程，使用 Redis Queue 进行异步处理 + +### 业务逻辑结构 (`biz/`) +- **`cmd/`**：不同审查功能的命令处理器 + - `review.py`：主要审查命令编排器 + - `func/`：专门的审查功能（分支、复杂度、目录、MySQL 审查） +- **`gitlab/`** 和 **`github/`**：平台特定的 webhook 处理器和 API 集成 +- **`llm/`**：多 LLM 客户端实现，支持 DeepSeek、智谱AI、OpenAI、通义千问和 Ollama +- **`queue/`**：使用 Redis Queue worker 的异步作业处理 +- **`service/`**：核心业务服务和审查编排 +- **`utils/`**：日志、配置、即时消息通知和报告的实用工具 +- **`entity/`**：数据模型和实体 + +### 配置系统 +多级配置，优先级：**项目 > 命名空间 > 全局** +- 全局配置：`conf/.env` +- 命名空间级：`conf/{namespace}/.env` +- 项目级：`conf/{namespace}/{project_name}/.env` + +### 数据库选项 +- **SQLite**（默认）：`data/data.db` - 适合小团队 +- **MySQL**：通过环境配置，适合大型部署 + +## 关键开发模式 + +### 事件驱动的 Webhook 处理 +1. GitLab/GitHub 发送 webhook 事件到 Flask API (`/review/webhook`) +2. 事件通过 Redis Queue 排队进行异步处理 +3. Worker 进程处理不同事件类型（合并请求、推送） +4. LLM 集成生成代码审查内容 +5. 结果发布回 GitLab/GitHub 并通过即时消息通知发送 + +### 多 LLM 集成 +- 支持 5 个 LLM 提供商的统一接口 +- 可按项目/命名空间配置 +- Token 管理和响应解析 +- 不同的审查风格：专业、讽刺、温和、幽默 + +### 即时消息通知系统 +- **钉钉**：Markdown 消息 +- **企业微信**：文本消息支持@提及（推送事件）或 markdown +- **飞书**：应用消息 +- **自定义 webhook**：用于扩展性 + +### 审查控制机制 +- **白名单系统**：命名空间和项目级过滤 +- **提交消息过滤**：基于正则表达式的触发控制 +- **受保护分支过滤**：可选的审查限制 +- **文件类型过滤**：可配置的支持扩展名 + +## 环境配置 + +复制 `conf/.env.dist` 到 `conf/.env` 并配置： +- LLM 提供商设置（选择一个：deepseek, openai, zhipuai, qwen, ollama） +- GitLab/GitHub 访问令牌 +- 即时消息 webhook URL（可选） +- 数据库设置（SQLite/MySQL） +- 审查行为选项（风格、限制、过滤器） + +## 端口配置 +- **Flask API**：5001 +- **Streamlit 仪表板**：5002 +- **Ollama**：11434（如果外部使用） + +## 测试结构 +测试文件镜像 `biz/` 目录结构： +- `test/biz/gitlab/test_webhook_handler.py`：Webhook 处理测试 +- `test/biz/queue/test_whitelist_isolation.py`：配置过滤测试 +- 基于 pytest 的全面测试，包含 64+ 个测试文件 + +## 生产部署 +- 多阶段 Docker 构建，使用 supervisor 进程管理 +- 分离的应用和 worker 容器目标 +- 持久数据、日志和配置的卷挂载 +- 健康检查和自动重启策略 \ No newline at end of file diff --git a/README.md b/README.md index 975a207f9..09b216869 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,25 @@ ## 功能 - 🚀 多模型支持 - - 兼容 DeepSeek、ZhipuAI、OpenAI、通义千问 和 Ollama，想用哪个就用哪个。 + - 兼容 DeepSeek、ZhipuAI、OpenAI、通义千问、Ollama 和 Kimi，想用哪个就用哪个。 - 📢 消息即时推送 - 审查结果一键直达 钉钉、企业微信 或 飞书，代码问题无处可藏！ + - 🆕 **企微增强**：支持 text 消息格式，可 @commit 者，并展示 AI Review 评分和详情链接！ - 📅 自动化日报生成 - 基于 GitLab & GitHub & Gitea Commit 记录，自动整理每日开发进展，谁在摸鱼、谁在卷，一目了然 😼。 - 📊 可视化 Dashboard - 集中展示所有 Code Review 记录，项目统计、开发者统计，数据说话，甩锅无门！ - 🎭 Review Style 任你选 - 专业型 🤵：严谨细致，正式专业。 - - 讽刺型 😈：毒舌吐槽，专治不服（"这代码是用脚写的吗？"） - - 绅士型 🌸：温柔建议，如沐春风（"或许这里可以再优化一下呢~"） - - 幽默型 🤪：搞笑点评，快乐改码（"这段 if-else 比我的相亲经历还曲折！"） + - 讽刺型 😈：毒舌吐槽，专治不服（“这代码是用脚写的吗？”） + - 绅士型 🌸：温柔建议，如沐春风（“或许这里可以再优化一下呢~”） + - 幽默型 🤪：搞笑点评，快乐改码（“这段 if-else 比我的相亲经历还曲折！”） +- 🎯 **多级配置系统** + - 支持项目级别、命名空间级别、全局配置，优先级：项目 > 命名空间 > 全局 + - 可为不同项目配置独立的 LLM、Review 风格、Prompt 模板等 +- 🎯 **白名单控制** + - 支持按命名空间或项目配置 Review 白名单，精准控制哪些项目允许进行代码审查 + - 支持 commit message 规则过滤，仅匹配特定 message 时才触发 Review **效果图:** @@ -56,7 +63,7 @@ cp conf/.env.dist conf/.env - 编辑 conf/.env 文件，配置以下关键参数： ```bash -#大模型供应商配置,支持 zhipuai , openai , deepseek 和 ollama +#大模型供应商配置,支持 zhipuai , openai , deepseek , ollama 和 kimi LLM_PROVIDER=deepseek #DeepSeek @@ -69,6 +76,14 @@ SUPPORTED_EXTENSIONS=.java,.py,.php,.yml,.vue,.go,.c,.cpp,.h,.js,.css,.md,.sql DINGTALK_ENABLED=0 DINGTALK_WEBHOOK_URL={YOUR_WDINGTALK_WEBHOOK_URL} +#企业微信消息推送 +WECOM_ENABLED=0 +WECOM_WEBHOOK_URL={YOUR_WECOM_WEBHOOK_URL} +# Push事件是否使用text消息类型（支持@人）：1=启用，0=使用markdown（默认） +PUSH_WECOM_USE_TEXT_MSG=0 +# 日报专用webhook（可选） +# WECOM_WEBHOOK_URL_DAILY_REPORT={YOUR_DAILY_REPORT_WEBHOOK_URL} + #Gitlab配置 GITLAB_ACCESS_TOKEN={YOUR_GITLAB_ACCESS_TOKEN} ``` @@ -77,6 +92,10 @@ GITLAB_ACCESS_TOKEN={YOUR_GITLAB_ACCESS_TOKEN} ```bash docker-compose up -d +docker-compose stop +docker-compose rm +docker-compose ps +docker-compose restart ``` **3. 验证部署** @@ -174,6 +193,143 @@ streamlit run ui.py --server.port=5002 --server.address=0.0.0.0 企业微信和飞书推送配置类似，具体参见 [常见问题](doc/faq.md) +## 高级配置 + +### 多级配置系统 + +支持为不同项目或命名空间配置独立的审查规则： + +```bash +# 全局配置：conf/.env +LLM_PROVIDER=deepseek +REVIEW_STYLE=professional + +# 命名空间级别：conf/{namespace}/.env +# 项目级别：conf/{namespace}/{project_name}/.env +# 优先级：项目级别 > 命名空间级别 > 全局配置 +``` + +### Review 白名单 + +控制哪些项目允许进行代码审查： + +```bash +# 开启白名单功能 +REVIEW_WHITELIST_ENABLED=1 + +# 配置白名单（支持命名空间或完整项目路径） +# 示例1：按命名空间 +REVIEW_WHITELIST=asset,frontend + +# 示例2：按项目路径 +REVIEW_WHITELIST=asset/asset-batch-center,frontend/web-app + +# 示例3：混合配置 +REVIEW_WHITELIST=asset,frontend/web-app,backend/api-gateway +``` + +### Commit Message 过滤 + +仅当 commit message 匹配指定规则时才触发 Review： + +```bash +# 开启 commit message 检查 +PUSH_COMMIT_MESSAGE_CHECK_ENABLED=1 + +# 配置匹配规则（支持正则表达式） +PUSH_COMMIT_MESSAGE_CHECK_PATTERN=review +# 或者：PUSH_COMMIT_MESSAGE_CHECK_PATTERN=\[review\] +# 或者：PUSH_COMMIT_MESSAGE_CHECK_PATTERN=(review|codereview) +``` + +### 企业微信 @人功能 + +Push 事件支持 text 消息格式，可 @commit 者： + +```bash +# 启用 text 消息类型（支持@人） +PUSH_WECOM_USE_TEXT_MSG=1 +``` + +### 日报专用推送配置 + +支持为日报功能配置独立的 webhook，与 push/merge 事件通知分开： + +```bash +# 企业微信日报专用 webhook（可选） +WECOM_WEBHOOK_URL_DAILY_REPORT=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=daily-report-key + +# 钉钉日报专用 webhook（可选） +DINGTALK_WEBHOOK_URL_DAILY_REPORT=https://oapi.dingtalk.com/robot/send?access_token=daily-report-token + +# 飞书日报专用 webhook（可选） +FEISHU_WEBHOOK_URL_DAILY_REPORT=https://open.feishu.cn/open-apis/bot/v2/hook/daily-report-hook +``` + +**说明**： +- 日报专用配置仅使用全局默认配置（`conf/.env`），不查找项目或命名空间级别配置 +- 如果未配置专用 webhook，则使用默认的 `{PLATFORM}_WEBHOOK_URL` +- 可以将日报推送到管理群，而 push/merge 事件推送到开发群 + +### 其他高级配置 + +```bash +# 仅对保护分支的合并请求进行 Review +MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED=1 + +# Review 风格：professional | sarcastic | gentle | humorous +REVIEW_STYLE=professional + +# 每次 Review 的最大 Token 限制 +REVIEW_MAX_TOKENS=10000 +``` + +### 数据库配置 + +系统支持 SQLite 和 MySQL 两种数据库存储方式，默认使用 SQLite。 + +#### 使用 SQLite（默认） + +```bash +# 数据库类型（默认：sqlite） +DB_TYPE=sqlite + +# SQLite 数据库文件路径（默认：data/data.db） +DB_FILE=data/data.db +``` + +#### 使用 MySQL + +```bash +# 数据库类型 +DB_TYPE=mysql + +# MySQL 数据库配置 +DB_HOST=localhost +DB_PORT=3306 +DB_USER=root +DB_PASSWORD=your_password +DB_NAME=ai_codereview +``` + +**MySQL 初始化**： + +1. 创建数据库： +```sql +CREATE DATABASE ai_codereview CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +``` + +2. 导入表结构（可选，系统会自动创建）： +```bash +mysql -u root -p ai_codereview < doc/mysql_schema.sql +``` + +**说明**： +- SQLite：适合小型团队或个人使用，无需额外配置数据库服务 +- MySQL：适合中大型团队，支持更高的并发性能和数据规模 +- 切换数据库类型前，建议备份现有数据 +- 详细使用说明请参见 [数据库使用指南](doc/database.md) + ## 其它 **1.如何对整个代码库进行Review?** @@ -186,7 +342,21 @@ python -m biz.cmd.review 运行后，请按照命令行中的提示进行操作即可。 -**2.其它问题** +**2.如何运行测试？** + +项目的所有测试代码统一存放在 `test/` 目录下，组织结构与 `biz/` 目录对应。 + +```bash +# 运行所有测试 +python -m unittest discover -s test -p "test_*.py" -v + +# 运行特定模块的测试 +python -m unittest test.biz.queue.test_whitelist -v +``` + +详细的测试说明请参见 [test/README.md](test/README.md) + +**3.其它问题** 参见 [常见问题](doc/faq.md) diff --git a/api.py b/api.py index f2c7f937f..c8d236ee4 100644 --- a/api.py +++ b/api.py @@ -38,37 +38,92 @@ def home(): """ -@api_app.route('/review/daily_report', methods=['GET']) -def daily_report(): +def generate_daily_report_core(): + """ + 日报生成核心逻辑，供Flask路由和定时任务共同调用 + :return: (report_text, error_message) + """ + logger.info("开始生成日报...") + # 获取当前日期0点和23点59分59秒的时间戳 - start_time = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp() - end_time = datetime.now().replace(hour=23, minute=59, second=59, microsecond=0).timestamp() + start_time = int(datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp()) + end_time = int(datetime.now().replace(hour=23, minute=59, second=59, microsecond=0).timestamp()) + + logger.info(f"查询时间范围: {start_time} - {end_time}") try: if push_review_enabled: + logger.info("PUSH_REVIEW_ENABLED=true, 获取push review日志") df = ReviewService().get_push_review_logs(updated_at_gte=start_time, updated_at_lte=end_time) else: + logger.info("PUSH_REVIEW_ENABLED=false, 获取merge request review日志") df = ReviewService().get_mr_review_logs(updated_at_gte=start_time, updated_at_lte=end_time) if df.empty: - logger.info("No data to process.") - return jsonify({'message': 'No data to process.'}), 200 + logger.info("没有找到相关数据.") + return None, "No data to process." + + logger.info(f"获取到 {len(df)} 条原始记录") + # 去重：基于 (author, message) 组合 df_unique = df.drop_duplicates(subset=["author", "commit_messages"]) + logger.info(f"去重后剩余 {len(df_unique)} 条记录") + # 按照 author 排序 df_sorted = df_unique.sort_values(by="author") + logger.info("数据已按作者排序") + # 转换为适合生成日报的格式 commits = df_sorted.to_dict(orient="records") - # 生成日报内容 + logger.info(f"转换为 {len(commits)} 条提交记录用于日报生成") + + # 生成日报内容(Reporter会从环境变量读取LLM配置) + logger.info("开始调用LLM生成日报内容...") report_txt = Reporter().generate_report(json.dumps(commits)) - # 发送钉钉通知 - notifier.send_notification(content=report_txt, msg_type="markdown", title="代码提交日报") + logger.info("LLM日报内容生成完成") + + # 发送IM通知，使用 msg_category='daily_report' 来使用独立的日报webhook + logger.info("开始发送IM通知...") + notifier.send_notification( + content=report_txt, + msg_type="markdown", + title="代码提交日报", + msg_category="daily_report", + project_config=None # 日报是全局任务,使用默认配置 + ) + logger.info("IM通知发送完成") - # 返回生成的日报内容 - return json.dumps(report_txt, ensure_ascii=False, indent=4) + return report_txt, None except Exception as e: - logger.error(f"Failed to generate daily report: {e}") - return jsonify({'message': f"Failed to generate daily report: {e}"}), 500 + logger.error(f"❌ Failed to generate daily report: {e}", exc_info=True) + return None, str(e) + + +@api_app.route('/review/daily_report', methods=['GET']) +def daily_report(): + """ + 日报生成Flask路由接口 + """ + report_txt, error = generate_daily_report_core() + + if error: + return jsonify({'message': error}), 500 if report_txt is None else 200 + + # 返回生成的日报内容 + return json.dumps(report_txt, ensure_ascii=False, indent=4) + + +def daily_report_scheduled(): + """ + 定时任务专用的日报生成函数（不依赖Flask应用上下文） + """ + logger.info("⏰ Scheduled daily report task started...") + report_txt, error = generate_daily_report_core() + + if error and report_txt is None: + logger.error(f"❌ Scheduled daily report failed: {error}") + else: + logger.info("✅ Scheduled daily report generated successfully") def setup_scheduler(): @@ -81,21 +136,33 @@ def setup_scheduler(): cron_parts = crontab_expression.split() cron_minute, cron_hour, cron_day, cron_month, cron_day_of_week = cron_parts + logger.info(f"Configuring scheduler with cron expression: {crontab_expression}") + logger.info(f"Parsed cron: minute={cron_minute}, hour={cron_hour}, day={cron_day}, month={cron_month}, day_of_week={cron_day_of_week}") + # Schedule the task based on the crontab expression - scheduler.add_job( - daily_report, + # 使用 daily_report_scheduled 而不是 daily_report，避免在定时任务中调用Flask路由 + job = scheduler.add_job( + daily_report_scheduled, trigger=CronTrigger( minute=cron_minute, hour=cron_hour, day=cron_day, month=cron_month, day_of_week=cron_day_of_week - ) + ), + id='daily_report_job' ) # Start the scheduler scheduler.start() logger.info("Scheduler started successfully.") + + # Log next run time + next_run = job.next_run_time + if next_run: + logger.info(f"Next scheduled run time: {next_run}") + else: + logger.warning("Could not determine next run time for the scheduled job") # Shut down the scheduler when exiting the app atexit.register(lambda: scheduler.shutdown()) @@ -239,4 +306,5 @@ def handle_gitea_webhook(event_type, data): # 启动Flask API服务 port = int(os.environ.get('SERVER_PORT', 5001)) - api_app.run(host='0.0.0.0', port=port) + # 使用 use_reloader=False 避免在开发模式下调度器被初始化两次 + api_app.run(host='0.0.0.0', port=port, use_reloader=False) diff --git a/biz/entity/review_entity.py b/biz/entity/review_entity.py index 890848cb9..41d0b8f6b 100644 --- a/biz/entity/review_entity.py +++ b/biz/entity/review_entity.py @@ -1,7 +1,10 @@ +from typing import Optional, Dict + + class MergeRequestReviewEntity: def __init__(self, project_name: str, author: str, source_branch: str, target_branch: str, updated_at: int, commits: list, score: float, url: str, review_result: str, url_slug: str, webhook_data: dict, - additions: int, deletions: int, last_commit_id: str): + additions: int, deletions: int, last_commit_id: str, project_config: Optional[Dict[str, str]] = None): self.project_name = project_name self.author = author self.source_branch = source_branch @@ -16,6 +19,7 @@ def __init__(self, project_name: str, author: str, source_branch: str, target_br self.additions = additions self.deletions = deletions self.last_commit_id = last_commit_id + self.project_config = project_config or {} # 项目专属配置 @property def commit_messages(self): @@ -25,7 +29,7 @@ def commit_messages(self): class PushReviewEntity: def __init__(self, project_name: str, author: str, branch: str, updated_at: int, commits: list, score: float, - review_result: str, url_slug: str, webhook_data: dict, additions: int, deletions: int): + review_result: str, url_slug: str, webhook_data: dict, additions: int, deletions: int, note_url: str = '', project_config: Optional[Dict[str, str]] = None): self.project_name = project_name self.author = author self.branch = branch @@ -37,6 +41,8 @@ def __init__(self, project_name: str, author: str, branch: str, updated_at: int, self.webhook_data = webhook_data self.additions = additions self.deletions = deletions + self.note_url = note_url # AI Review结果的URL + self.project_config = project_config or {} # 项目专属配置 @property def commit_messages(self): diff --git a/biz/event/event_manager.py b/biz/event/event_manager.py index 2c4454d51..aeb85be44 100644 --- a/biz/event/event_manager.py +++ b/biz/event/event_manager.py @@ -31,36 +31,86 @@ def on_merge_request_reviewed(mr_review_entity: MergeRequestReviewEntity): {mr_review_entity.review_result} """ + # 从 entity 中获取 project_config，如果没有则传递 None + project_config = getattr(mr_review_entity, 'project_config', None) notifier.send_notification(content=im_msg, msg_type='markdown', title='Merge Request Review', project_name=mr_review_entity.project_name, url_slug=mr_review_entity.url_slug, - webhook_data=mr_review_entity.webhook_data) + webhook_data=mr_review_entity.webhook_data, project_config=project_config) # 记录到数据库 ReviewService().insert_mr_review_log(mr_review_entity) def on_push_reviewed(entity: PushReviewEntity): - # 发送IM消息通知 - im_msg = f"### 🚀 {entity.project_name}: Push\n\n" - im_msg += "#### 提交记录:\n" - + # 从项目配置中获取：是否使用text消息类型 + use_text_msg = entity.project_config.get('PUSH_WECOM_USE_TEXT_MSG', '0') == '1' + msg_type = 'text' if use_text_msg else 'markdown' + + # 提取commit者用于@（text和markdown都支持） + mentioned_list = None + authors = set() for commit in entity.commits: - message = commit.get('message', '').strip() - author = commit.get('author', 'Unknown Author') - timestamp = commit.get('timestamp', '') - url = commit.get('url', '#') - im_msg += ( - f"- **提交信息**: {message}\n" - f"- **提交者**: {author}\n" - f"- **时间**: {timestamp}\n" - f"- [查看提交详情]({url})\n\n" - ) - - if entity.review_result: - im_msg += f"#### AI Review 结果: \n {entity.review_result}\n\n" - notifier.send_notification(content=im_msg, msg_type='markdown',title=f"{entity.project_name} Push Event", - project_name=entity.project_name, url_slug=entity.url_slug, - webhook_data=entity.webhook_data) + author = commit.get('author', '') + if author: + authors.add(author) + mentioned_list = list(authors) if authors else None + + # 发送IM消息通知 + if msg_type == 'text': + # Text消息，提交信息保持详细，Review结果仅显示评分和链接 + im_msg = f"🚀 {entity.project_name}: Push\n\n" + im_msg += "提交记录:\n" + for commit in entity.commits: + message = commit.get('message', '').strip() + author = commit.get('author', 'Unknown Author') + timestamp = commit.get('timestamp', '') + url = commit.get('url', '#') + im_msg += ( + f"- 提交信息: {message}\n" + f" 提交者: {author}\n" + f" 时间: {timestamp}\n" + f" 查看详情: {url}\n\n" + ) + + if entity.review_result and entity.score > 0: + im_msg += f"AI Review 结果:\n" + im_msg += f"评分: {entity.score:.1f}/100\n" + if entity.note_url: + im_msg += f"查看详情: {entity.note_url}" + else: + # Markdown消息 + im_msg = f"### 🚀 {entity.project_name}: Push\n\n" + im_msg += "#### 提交记录:\n" + + for commit in entity.commits: + message = commit.get('message', '').strip() + author = commit.get('author', 'Unknown Author') + timestamp = commit.get('timestamp', '') + url = commit.get('url', '#') + im_msg += ( + f"- **提交信息**: {message}\n" + f"- **提交者**: {author}\n" + f"- **时间**: {timestamp}\n" + f"- [查看提交详情]({url})\n\n" + ) + + if entity.review_result: + im_msg += f"#### AI Review 结果:\n" + im_msg += f"- **评分**: {entity.score:.1f}\n" + if entity.note_url: + im_msg += f"- [查看详情]({entity.note_url})\n\n" + im_msg += f"{entity.review_result}\n\n" + + notifier.send_notification( + content=im_msg, + msg_type=msg_type, + title=f"{entity.project_name} Push Event", + project_name=entity.project_name, + url_slug=entity.url_slug, + webhook_data=entity.webhook_data, + mentioned_list=mentioned_list, + project_config=entity.project_config + ) # 记录到数据库 ReviewService().insert_push_review_log(entity) diff --git a/biz/github/webhook_handler.py b/biz/github/webhook_handler.py index 8eb735fb9..fe3ae85e6 100644 --- a/biz/github/webhook_handler.py +++ b/biz/github/webhook_handler.py @@ -1,6 +1,7 @@ import os import re import time +from typing import Optional, Dict import requests import fnmatch @@ -8,13 +9,16 @@ -def filter_changes(changes: list): +def filter_changes(changes: list, project_config: Optional[Dict[str, str]] = None): ''' 过滤数据，只保留支持的文件类型以及必要的字段信息 专门处理GitHub格式的变更 + :param changes: 变更列表 + :param project_config: 项目专属配置字典 ''' - # 从环境变量中获取支持的文件扩展名 - supported_extensions = os.getenv('SUPPORTED_EXTENSIONS', '.java,.py,.php').split(',') + # 从项目配置中获取支持的文件扩展名 + project_config = project_config or {} + supported_extensions = project_config.get('SUPPORTED_EXTENSIONS', '.java,.py,.php').split(',') # 筛选出未被删除的文件 not_deleted_changes = [] @@ -239,13 +243,13 @@ def add_push_notes(self, message: str): # 添加评论到 GitHub Push 请求的提交中（此处假设是在最后一次提交上添加注释） if not self.commit_list: logger.warn("No commits found to add notes to.") - return + return '' # 获取最后一个提交的ID last_commit_id = self.commit_list[-1].get('id') if not last_commit_id: logger.error("Last commit ID not found.") - return + return '' url = f"https://api.github.com/repos/{self.repo_full_name}/commits/{last_commit_id}/comments" headers = { @@ -259,9 +263,13 @@ def add_push_notes(self, message: str): logger.debug(f"Add comment to commit {last_commit_id}: {response.status_code}, {response.text}") if response.status_code == 201: logger.info("Comment successfully added to push commit.") + # 返回commit的URL，用户可以在这里查看评论 + commit_url = self.commit_list[-1].get('url', '') + return commit_url else: logger.error(f"Failed to add comment: {response.status_code}") logger.error(response.text) + return '' def __repository_commits(self, sha: str = "", per_page: int = 100, page: int = 1): # 获取仓库提交信息 diff --git a/biz/gitlab/webhook_handler.py b/biz/gitlab/webhook_handler.py index 5bb837582..dfa56d5a9 100644 --- a/biz/gitlab/webhook_handler.py +++ b/biz/gitlab/webhook_handler.py @@ -1,6 +1,7 @@ import os import re import time +from typing import Optional, Dict from urllib.parse import urljoin import fnmatch import requests @@ -8,12 +9,15 @@ from biz.utils.log import logger -def filter_changes(changes: list): +def filter_changes(changes: list, project_config: Optional[Dict[str, str]] = None): ''' 过滤数据，只保留支持的文件类型以及必要的字段信息 + :param changes: 变更列表 + :param project_config: 项目专属配置字典 ''' - # 从环境变量中获取支持的文件扩展名 - supported_extensions = os.getenv('SUPPORTED_EXTENSIONS', '.java,.py,.php').split(',') + # 从项目配置中获取支持的文件扩展名 + project_config = project_config or {} + supported_extensions = project_config.get('SUPPORTED_EXTENSIONS', '.java,.py,.php').split(',') filter_deleted_files_changes = [change for change in changes if not change.get("deleted_file")] @@ -73,6 +77,21 @@ def parse_merge_request_event(self): self.project_id = merge_request.get('target_project_id') self.action = merge_request.get('action') + def is_author_excluded(self, excluded_users: Optional[list] = None) -> bool: + """ + 检查MR的作者是否在排除列表中 + :param excluded_users: 排除的用户名列表，如 ['howbuyscm', 'admin'] + :return: True表示作者在排除列表中，False表示不在 + """ + if not excluded_users: + excluded_users = ['howbuyscm'] # 默认排除用户 + + author_username = self.webhook_data.get('user', {}).get('username', '') + if author_username in excluded_users: + logger.info(f"MR author '{author_username}' is in excluded users list. Skipping review.") + return True + return False + def get_merge_request_changes(self) -> list: # 检查是否为 Merge Request Hook 事件 if self.event_type != 'merge_request': @@ -215,13 +234,13 @@ def add_push_notes(self, message: str): # 添加评论到 GitLab Push 请求的提交中（此处假设是在最后一次提交上添加注释） if not self.commit_list: logger.warn("No commits found to add notes to.") - return + return '' # 获取最后一个提交的ID last_commit_id = self.commit_list[-1].get('id') if not last_commit_id: logger.error("Last commit ID not found.") - return + return '' url = urljoin(f"{self.gitlab_url}/", f"api/v4/projects/{self.project_id}/repository/commits/{last_commit_id}/comments") @@ -236,9 +255,13 @@ def add_push_notes(self, message: str): logger.debug(f"Add comment to commit {last_commit_id}: {response.status_code}, {response.text}") if response.status_code == 201: logger.info("Comment successfully added to push commit.") + # 返回commit的URL，用户可以在这里查看评论 + commit_url = self.commit_list[-1].get('url', '') + return commit_url else: logger.error(f"Failed to add comment: {response.status_code}") logger.error(response.text) + return '' def __repository_commits(self, ref_name: str = "", since: str = "", until: str = "", pre_page: int = 100, page: int = 1): diff --git a/biz/llm/client/base.py b/biz/llm/client/base.py index 69ea86910..241bf3e87 100644 --- a/biz/llm/client/base.py +++ b/biz/llm/client/base.py @@ -7,12 +7,30 @@ class BaseClient: """ Base class for chat models client. """ + + def __init__(self, config: Optional[Dict[str, str]] = None): + """ + 初始化LLM客户端 + :param config: 项目专属配置字典，优先级高于全局环境变量 + """ + self.config = config or {} + + def get_config(self, key: str, default: Optional[str] = None) -> Optional[str]: + """ + 获取配置项，优先从config字典读取，若不存在则从环境变量读取 + :param key: 配置键 + :param default: 默认值 + :return: 配置值 + """ + import os + # 优先从config字典读取，其次从环境变量读取，最后使用默认值 + return self.config.get(key) or os.getenv(key, default) def ping(self) -> bool: """Ping the model to check connectivity.""" try: result = self.completions(messages=[{"role": "user", "content": '请仅返回 "ok"。'}]) - return result and result.strip() == "ok" + return bool(result and result.strip() == "ok") except Exception: logger.error("尝试连接LLM失败， {e}") return False diff --git a/biz/llm/client/deepseek.py b/biz/llm/client/deepseek.py index 9cd63b5d7..06771eb72 100644 --- a/biz/llm/client/deepseek.py +++ b/biz/llm/client/deepseek.py @@ -9,14 +9,15 @@ class DeepSeekClient(BaseClient): - def __init__(self, api_key: str = None): - self.api_key = api_key or os.getenv("DEEPSEEK_API_KEY") - self.base_url = os.getenv("DEEPSEEK_API_BASE_URL", "https://api.deepseek.com") + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.api_key = api_key or self.get_config("DEEPSEEK_API_KEY") + self.base_url = self.get_config("DEEPSEEK_API_BASE_URL", "https://api.deepseek.com") if not self.api_key: raise ValueError("API key is required. Please provide it or set it in the environment variables.") self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) # DeepSeek supports OpenAI API SDK - self.default_model = os.getenv("DEEPSEEK_API_MODEL", "deepseek-chat") + self.default_model = self.get_config("DEEPSEEK_API_MODEL", "deepseek-chat") def completions(self, messages: List[Dict[str, str]], @@ -24,18 +25,20 @@ def completions(self, ) -> str: try: model = model or self.default_model + if not model: + model = "deepseek-chat" logger.debug(f"Sending request to DeepSeek API. Model: {model}, Messages: {messages}") completion = self.client.chat.completions.create( model=model, - messages=messages + messages=messages # type: ignore ) if not completion or not completion.choices: logger.error("Empty response from DeepSeek API") return "AI服务返回为空，请稍后重试" - return completion.choices[0].message.content + return completion.choices[0].message.content or "" except Exception as e: logger.error(f"DeepSeek API error: {str(e)}") diff --git a/biz/llm/client/kimi.py b/biz/llm/client/kimi.py new file mode 100644 index 000000000..0d5fa5241 --- /dev/null +++ b/biz/llm/client/kimi.py @@ -0,0 +1,52 @@ + +import os +from typing import Dict, List, Optional + +from openai import OpenAI + +from biz.llm.client.base import BaseClient +from biz.llm.types import NotGiven, NOT_GIVEN +from biz.utils.log import logger + + +class KimiClient(BaseClient): + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.api_key = api_key or self.get_config("KIMI_API_KEY") + self.base_url = self.get_config("KIMI_API_BASE_URL", "https://api.moonshot.cn/v1") + if not self.api_key: + raise ValueError("API key is required. Please provide it or set it in the environment variables.") + + self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) + self.default_model = self.get_config("KIMI_API_MODEL", "kimi-k2-turbo-preview") + + def completions(self, + messages: List[Dict[str, str]], + model: Optional[str] | NotGiven = NOT_GIVEN, + ) -> str: + try: + model = model or self.default_model + if not model: + model = "kimi-k2-turbo-preview" + logger.debug(f"Sending request to Kimi API. Model: {model}, Messages: {messages}") + + completion = self.client.chat.completions.create( + model=model, + messages=messages # type: ignore + ) + + if not completion or not completion.choices: + logger.error("Empty response from Kimi API") + return "AI服务返回为空，请稍后重试" + + return completion.choices[0].message.content or "" + + except Exception as e: + logger.error(f"Kimi API error: {str(e)}") + # 检查是否是认证错误 + if "401" in str(e): + return "Kimi API认证失败，请检查API密钥是否正确" + elif "404" in str(e): + return "Kimi API接口未找到，请检查API地址是否正确" + else: + return f"调用Kimi API时出错: {str(e)}" diff --git a/biz/llm/client/ollama_client.py b/biz/llm/client/ollama_client.py index 1574f9ac5..cac26ac49 100644 --- a/biz/llm/client/ollama_client.py +++ b/biz/llm/client/ollama_client.py @@ -10,9 +10,10 @@ class OllamaClient(BaseClient): - def __init__(self, api_key: str = None): - self.default_model = self.default_model = os.getenv("OLLAMA_API_MODEL", "deepseek-r1-8k:14b") - self.base_url = os.getenv("OLLAMA_API_BASE_URL", "http://127.0.0.1:11434") + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.default_model = self.get_config("OLLAMA_API_MODEL", "deepseek-r1-8k:14b") + self.base_url = self.get_config("OLLAMA_API_BASE_URL", "http://127.0.0.1:11434") self.client = Client( host=self.base_url, ) @@ -40,6 +41,7 @@ def completions(self, messages: List[Dict[str, str]], model: Optional[str] | NotGiven = NOT_GIVEN, ) -> str: - response: ChatResponse = self.client.chat(model or self.default_model, messages) + model = model or self.default_model or "deepseek-r1-8k:14b" + response: ChatResponse = self.client.chat(model, messages) # type: ignore content = response['message']['content'] return self._extract_content(content) diff --git a/biz/llm/client/openai.py b/biz/llm/client/openai.py index 69d35f172..78705bcc3 100644 --- a/biz/llm/client/openai.py +++ b/biz/llm/client/openai.py @@ -8,22 +8,23 @@ class OpenAIClient(BaseClient): - def __init__(self, api_key: str = None): - self.api_key = api_key or os.getenv("OPENAI_API_KEY") - self.base_url = os.getenv("OPENAI_API_BASE_URL", "https://api.openai.com") + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.api_key = api_key or self.get_config("OPENAI_API_KEY") + self.base_url = self.get_config("OPENAI_API_BASE_URL", "https://api.openai.com") if not self.api_key: raise ValueError("API key is required. Please provide it or set it in the environment variables.") self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) - self.default_model = os.getenv("OPENAI_API_MODEL", "gpt-4o-mini") + self.default_model = self.get_config("OPENAI_API_MODEL", "gpt-4o-mini") def completions(self, messages: List[Dict[str, str]], model: Optional[str] | NotGiven = NOT_GIVEN, ) -> str: - model = model or self.default_model + model = model or self.default_model or "gpt-4o-mini" completion = self.client.chat.completions.create( model=model, - messages=messages, + messages=messages, # type: ignore ) - return completion.choices[0].message.content + return completion.choices[0].message.content or "" diff --git a/biz/llm/client/qwen.py b/biz/llm/client/qwen.py index 14e03a9dd..3c4bc9bcf 100644 --- a/biz/llm/client/qwen.py +++ b/biz/llm/client/qwen.py @@ -8,24 +8,25 @@ class QwenClient(BaseClient): - def __init__(self, api_key: str = None): - self.api_key = api_key or os.getenv("QWEN_API_KEY") - self.base_url = os.getenv("QWEN_API_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1") + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.api_key = api_key or self.get_config("QWEN_API_KEY") + self.base_url = self.get_config("QWEN_API_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1") if not self.api_key: raise ValueError("API key is required. Please provide it or set it in the environment variables.") self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) - self.default_model = os.getenv("QWEN_API_MODEL", "qwen-coder-plus") + self.default_model = self.get_config("QWEN_API_MODEL", "qwen-coder-plus") self.extra_body={"enable_thinking": False} def completions(self, messages: List[Dict[str, str]], model: Optional[str] | NotGiven = NOT_GIVEN, ) -> str: - model = model or self.default_model + model = model or self.default_model or "qwen-coder-plus" completion = self.client.chat.completions.create( model=model, - messages=messages, + messages=messages, # type: ignore extra_body=self.extra_body, ) - return completion.choices[0].message.content + return completion.choices[0].message.content or "" diff --git a/biz/llm/client/zhipuai.py b/biz/llm/client/zhipuai.py index 0790cd97f..23d0738f1 100644 --- a/biz/llm/client/zhipuai.py +++ b/biz/llm/client/zhipuai.py @@ -8,21 +8,22 @@ class ZhipuAIClient(BaseClient): - def __init__(self, api_key: str = None): - self.api_key = api_key or os.getenv("ZHIPUAI_API_KEY") + def __init__(self, api_key: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__(config) + self.api_key = api_key or self.get_config("ZHIPUAI_API_KEY") if not self.api_key: raise ValueError("API key is required. Please provide it or set it in the environment variables.") - self.client = ZhipuAI(api_key=api_key) - self.default_model = os.getenv("ZHIPUAI_API_MODEL", "GLM-4-Flash") + self.client = ZhipuAI(api_key=self.api_key) + self.default_model = self.get_config("ZHIPUAI_API_MODEL", "GLM-4-Flash") def completions(self, messages: List[Dict[str, str]], model: Optional[str] | NotGiven = NOT_GIVEN, ) -> str: - model = model or self.default_model + model = model or self.default_model or "GLM-4-Flash" completion = self.client.chat.completions.create( model=model, - messages=messages, + messages=messages, # type: ignore ) - return completion.choices[0].message.content + return completion.choices[0].message.content or "" # type: ignore diff --git a/biz/llm/factory.py b/biz/llm/factory.py index 453ac3a3d..765f8d104 100644 --- a/biz/llm/factory.py +++ b/biz/llm/factory.py @@ -1,7 +1,9 @@ import os +from typing import Dict, Optional from biz.llm.client.base import BaseClient from biz.llm.client.deepseek import DeepSeekClient +from biz.llm.client.kimi import KimiClient from biz.llm.client.ollama_client import OllamaClient from biz.llm.client.openai import OpenAIClient from biz.llm.client.qwen import QwenClient @@ -11,14 +13,23 @@ class Factory: @staticmethod - def getClient(provider: str = None) -> BaseClient: - provider = provider or os.getenv("LLM_PROVIDER", "openai") + def getClient(provider: Optional[str] = None, config: Optional[Dict[str, str]] = None) -> BaseClient: + """ + 获取LLM客户端 + :param provider: 提供商名称 + :param config: 项目专属配置字典，优先级高于全局环境变量 + :return: LLM客户端实例 + """ + config = config or {} + # 优先从 config 读取，其次从 provider 参数，最后从环境变量读取 + provider = provider or config.get("LLM_PROVIDER") or os.environ.get("LLM_PROVIDER", "openai") chat_model_providers = { - 'zhipuai': lambda: ZhipuAIClient(), - 'openai': lambda: OpenAIClient(), - 'deepseek': lambda: DeepSeekClient(), - 'qwen': lambda: QwenClient(), - 'ollama': lambda : OllamaClient() + 'zhipuai': lambda: ZhipuAIClient(config=config), + 'openai': lambda: OpenAIClient(config=config), + 'deepseek': lambda: DeepSeekClient(config=config), + 'qwen': lambda: QwenClient(config=config), + 'ollama': lambda : OllamaClient(config=config), + 'kimi': lambda: KimiClient(config=config) } provider_func = chat_model_providers.get(provider) diff --git a/biz/queue/worker.py b/biz/queue/worker.py index 132487b49..84324f06c 100644 --- a/biz/queue/worker.py +++ b/biz/queue/worker.py @@ -1,7 +1,11 @@ import os +import os +import re import traceback from datetime import datetime +from typing import Dict, Optional + from biz.entity.review_entity import MergeRequestReviewEntity, PushReviewEntity from biz.event.event_manager import event_manager from biz.gitlab.webhook_handler import filter_changes, MergeRequestHandler, PushHandler @@ -10,14 +14,89 @@ PushHandler as GiteaPushHandler from biz.service.review_service import ReviewService from biz.utils.code_reviewer import CodeReviewer +from biz.utils.config_loader import config_loader from biz.utils.im import notifier from biz.utils.log import logger +def check_project_whitelist(project_path: str, project_config: Optional[Dict[str, str]] = None) -> bool: + """ + 检查项目是否在白名单中 + :param project_path: 项目路径，格式为 namespace/project_name（如：asset/asset-batch-center） + :param project_config: 项目专属配置字典，优先级高于全局环境变量 + :return: True表示在白名单中，False表示不在白名单中 + """ + # 全局开关始终从os.environ读取 + whitelist_enabled = os.environ.get('REVIEW_WHITELIST_ENABLED', '0') == '1' + if not whitelist_enabled: + # 白名单功能未开启，所有项目都允许 + return True + + # 优先从project_config读取白名单列表 + whitelist_str = '' + if project_config: + whitelist_str = project_config.get('REVIEW_WHITELIST', '') + + # 降级到全局环境变量 + if not whitelist_str: + whitelist_str = os.environ.get('REVIEW_WHITELIST', '') + if not whitelist_str: + logger.warning('白名单功能已开启但REVIEW_WHITELIST配置为空，将拒绝所有项目的Review') + return False + + # 解析白名单配置（逗号分隔） + whitelist_items = [item.strip() for item in whitelist_str.split(',') if item.strip()] + + if not project_path: + logger.warning('项目路径为空，无法进行白名单检查') + return False + + # 提取命名空间和项目名 + if '/' in project_path: + namespace = project_path.split('/', 1)[0] + else: + # 如果没有/，则整个project_path就是命名空间 + namespace = project_path + + # 检查是否在白名单中 + for item in whitelist_items: + # 完全匹配项目路径（如：asset/asset-batch-center） + if item == project_path: + logger.info(f'项目 {project_path} 在白名单中（完全匹配：{item}）') + return True + # 匹配命名空间（如：asset） + if '/' not in item and item == namespace: + logger.info(f'项目 {project_path} 在白名单中（命名空间匹配：{item}）') + return True + + logger.info(f'项目 {project_path} 不在白名单中，跳过Review。白名单配置：{whitelist_str}') + return False + + def handle_push_event(webhook_data: dict, gitlab_token: str, gitlab_url: str, gitlab_url_slug: str): - push_review_enabled = os.environ.get('PUSH_REVIEW_ENABLED', '0') == '1' + # 初始化project_config为None，确保在异常处理中可以访问 + project_config = None try: + # 提取项目路径 + project_path = webhook_data.get('project', {}).get('path_with_namespace', '') + logger.info(f'Project path: {project_path}') + + # 加载项目专属配置（不修改全局环境变量） + project_config = config_loader.get_config(project_path=project_path) + + # 检查白名单（传递project_config确保配置隔离） + if not check_project_whitelist(project_path, project_config=project_config): + logger.info(f'项目 {project_path} 不在白名单中，跳过Push Review') + return + logger.info(f'项目 {project_path} 使用独立配置上下文') + + # 从项目配置中读取 GITLAB_ACCESS_TOKEN + gitlab_token = project_config.get('GITLAB_ACCESS_TOKEN') or gitlab_token + + # 检查是否启用Push Review + push_review_enabled = project_config.get('PUSH_REVIEW_ENABLED', '0') == '1' + handler = PushHandler(webhook_data, gitlab_token, gitlab_url) logger.info('Push Hook event received') commits = handler.get_push_commits() @@ -25,28 +104,45 @@ def handle_push_event(webhook_data: dict, gitlab_token: str, gitlab_url: str, gi logger.error('Failed to get commits') return - review_result = None + # 检查是否启用了commit message检查 + commit_message_check_enabled = project_config.get('PUSH_COMMIT_MESSAGE_CHECK_ENABLED', '0') == '1' + if commit_message_check_enabled: + # 获取检查规则（支持正则表达式） + check_pattern = project_config.get('PUSH_COMMIT_MESSAGE_CHECK_PATTERN', 'review') + try: + # 检查所有commits的message是否匹配正则表达式 + pattern = re.compile(check_pattern, re.IGNORECASE) + has_match = any(pattern.search(commit.get('message', '')) for commit in commits) + if not has_match: + logger.info(f'Commits message中未匹配到指定规则 "{check_pattern}"，跳过本次审查。') + return + logger.info(f'Commits message匹配规则 "{check_pattern}"，继续执行审查。') + except re.error as e: + logger.error(f'正则表达式 "{check_pattern}" 格式错误: {e}，跳过检查继续执行。') + + review_result = "" score = 0 additions = 0 deletions = 0 + note_url = '' # 存储AI Review结果的URL if push_review_enabled: # 获取PUSH的changes changes = handler.get_push_changes() logger.info('changes: %s', changes) - changes = filter_changes(changes) + changes = filter_changes(changes, project_config) if not changes: logger.info('未检测到PUSH代码的修改,修改文件可能不满足SUPPORTED_EXTENSIONS。') review_result = "关注的文件没有修改" if len(changes) > 0: commits_text = ';'.join(commit.get('message', '').strip() for commit in commits) - review_result = CodeReviewer().review_and_strip_code(str(changes), commits_text) + review_result = CodeReviewer(project_path=project_path, config=project_config).review_and_strip_code(str(changes), commits_text) score = CodeReviewer.parse_review_score(review_text=review_result) for item in changes: additions += item['additions'] deletions += item['deletions'] # 将review结果提交到Gitlab的 notes - handler.add_push_notes(f'Auto Review Result: \n{review_result}') + note_url = handler.add_push_notes(f'Auto Review Result: \n{review_result}') event_manager['push_reviewed'].send(PushReviewEntity( project_name=webhook_data['project']['name'], @@ -60,11 +156,17 @@ def handle_push_event(webhook_data: dict, gitlab_token: str, gitlab_url: str, gi webhook_data=webhook_data, additions=additions, deletions=deletions, + note_url=note_url, + project_config=project_config, )) except Exception as e: error_message = f'服务出现未知错误: {str(e)}\n{traceback.format_exc()}' - notifier.send_notification(content=error_message) + # 尝试获取project_config，如果异常发生在配置加载之前则为None + try: + notifier.send_notification(content=error_message, project_config=project_config) + except NameError: + notifier.send_notification(content=error_message) logger.error('出现未知错误: %s', error_message) @@ -77,18 +179,44 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url :param gitlab_url_slug: :return: ''' - merge_review_only_protected_branches = os.environ.get('MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED', '0') == '1' + # 初始化project_config为None，确保在异常处理中可以访问 + project_config = None try: + # 提取项目路径 + project_path = webhook_data.get('project', {}).get('path_with_namespace', '') + logger.info(f'Project path: {project_path}') + + # 加载项目专属配置（不修改全局环境变量） + project_config = config_loader.get_config(project_path=project_path) + + # 检查白名单（传递project_config确保配置隔离） + if not check_project_whitelist(project_path, project_config=project_config): + logger.info(f'项目 {project_path} 不在白名单中，跳过Merge Request Review') + return + logger.info(f'项目 {project_path} 使用独立配置上下文') + + # 从项目配置中读取 GITLAB_ACCESS_TOKEN + gitlab_token = project_config.get('GITLAB_ACCESS_TOKEN') or gitlab_token + + # 检查是否仅review protected branches + merge_review_only_protected_branches = project_config.get('MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED', '0') == '1' + # 解析Webhook数据 handler = MergeRequestHandler(webhook_data, gitlab_token, gitlab_url) logger.info('Merge Request Hook event received') + # 检查MR作者是否在排除列表中 + excluded_users = project_config.get('MERGE_REVIEW_EXCLUDED_USERS', 'howbuyscm').split(',') + excluded_users = [user.strip() for user in excluded_users if user.strip()] + if handler.is_author_excluded(excluded_users): + return + # 新增：判断是否为draft（草稿）MR object_attributes = webhook_data.get('object_attributes', {}) is_draft = object_attributes.get('draft') or object_attributes.get('work_in_progress') if is_draft: msg = f"[通知] MR为草稿（draft），未触发AI审查。\n项目: {webhook_data['project']['name']}\n作者: {webhook_data['user']['username']}\n源分支: {object_attributes.get('source_branch')}\n目标分支: {object_attributes.get('target_branch')}\n链接: {object_attributes.get('url')}" - notifier.send_notification(content=msg) + notifier.send_notification(content=msg, project_config=project_config) logger.info("MR为draft，仅发送通知，不触发AI review。") return @@ -108,7 +236,8 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url source_branch = object_attributes.get('source_branch', '') target_branch = object_attributes.get('target_branch', '') - if ReviewService.check_mr_last_commit_id_exists(project_name, source_branch, target_branch, last_commit_id): + # 创建ReviewService实例并调用方法 + if ReviewService().check_mr_last_commit_id_exists(project_name, source_branch, target_branch, last_commit_id): logger.info(f"Merge Request with last_commit_id {last_commit_id} already exists, skipping review for {project_name}.") return @@ -116,7 +245,7 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url # 获取Merge Request的changes changes = handler.get_merge_request_changes() logger.info('changes: %s', changes) - changes = filter_changes(changes) + changes = filter_changes(changes, project_config) if not changes: logger.info('未检测到有关代码的修改,修改文件可能不满足SUPPORTED_EXTENSIONS。') return @@ -135,7 +264,7 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url # review 代码 commits_text = ';'.join(commit['title'] for commit in commits) - review_result = CodeReviewer().review_and_strip_code(str(changes), commits_text) + review_result = CodeReviewer(project_path=project_path, config=project_config).review_and_strip_code(str(changes), commits_text) # 将review结果提交到Gitlab的 notes handler.add_merge_request_notes(f'Auto Review Result: \n{review_result}') @@ -157,17 +286,42 @@ def handle_merge_request_event(webhook_data: dict, gitlab_token: str, gitlab_url additions=additions, deletions=deletions, last_commit_id=last_commit_id, + project_config=project_config, ) ) except Exception as e: error_message = f'AI Code Review 服务出现未知错误: {str(e)}\n{traceback.format_exc()}' - notifier.send_notification(content=error_message) + # 尝试获取project_config，如果异常发生在配置加载之前则为None + try: + notifier.send_notification(content=error_message, project_config=project_config) + except NameError: + notifier.send_notification(content=error_message) logger.error('出现未知错误: %s', error_message) def handle_github_push_event(webhook_data: dict, github_token: str, github_url: str, github_url_slug: str): - push_review_enabled = os.environ.get('PUSH_REVIEW_ENABLED', '0') == '1' + # 初始化project_config为None，确保在异常处理中可以访问 + project_config = None try: + # 提取项目路径 + project_path = webhook_data.get('repository', {}).get('full_name', '') + logger.info(f'Project path: {project_path}') + + # 加载项目专属配置（不修改全局环境变量） + project_config = config_loader.get_config(project_path=project_path) + + # 检查白名单（传递project_config确保配置隔离） + if not check_project_whitelist(project_path, project_config=project_config): + logger.info(f'项目 {project_path} 不在白名单中，跳过GitHub Push Review') + return + logger.info(f'项目 {project_path} 使用独立配置上下文') + + # 从项目配置中读取 GITHUB_ACCESS_TOKEN + github_token = project_config.get('GITHUB_ACCESS_TOKEN') or github_token + + # 检查是否启用Push Review + push_review_enabled = project_config.get('PUSH_REVIEW_ENABLED', '0') == '1' + handler = GithubPushHandler(webhook_data, github_token, github_url) logger.info('GitHub Push event received') commits = handler.get_push_commits() @@ -175,28 +329,45 @@ def handle_github_push_event(webhook_data: dict, github_token: str, github_url: logger.error('Failed to get commits') return - review_result = None + # 检查是否启用了commit message检查 + commit_message_check_enabled = project_config.get('PUSH_COMMIT_MESSAGE_CHECK_ENABLED', '0') == '1' + if commit_message_check_enabled: + # 获取检查规则（支持正则表达式） + check_pattern = project_config.get('PUSH_COMMIT_MESSAGE_CHECK_PATTERN', 'review') + try: + # 检查所有commits的message是否匹配正则表达式 + pattern = re.compile(check_pattern, re.IGNORECASE) + has_match = any(pattern.search(commit.get('message', '')) for commit in commits) + if not has_match: + logger.info(f'Commits message中未匹配到指定规则 "{check_pattern}"，跳过本次审查。') + return + logger.info(f'Commits message匹配规则 "{check_pattern}"，继续执行审查。') + except re.error as e: + logger.error(f'正则表达式 "{check_pattern}" 格式错误: {e}，跳过检查继续执行。') + + review_result = "" score = 0 additions = 0 deletions = 0 + note_url = '' # 存储AI Review结果的URL if push_review_enabled: # 获取PUSH的changes changes = handler.get_push_changes() logger.info('changes: %s', changes) - changes = filter_github_changes(changes) + changes = filter_github_changes(changes, project_config) if not changes: logger.info('未检测到PUSH代码的修改,修改文件可能不满足SUPPORTED_EXTENSIONS。') review_result = "关注的文件没有修改" if len(changes) > 0: commits_text = ';'.join(commit.get('message', '').strip() for commit in commits) - review_result = CodeReviewer().review_and_strip_code(str(changes), commits_text) + review_result = CodeReviewer(project_path=project_path, config=project_config).review_and_strip_code(str(changes), commits_text) score = CodeReviewer.parse_review_score(review_text=review_result) for item in changes: additions += item.get('additions', 0) deletions += item.get('deletions', 0) # 将review结果提交到GitHub的 notes - handler.add_push_notes(f'Auto Review Result: \n{review_result}') + note_url = handler.add_push_notes(f'Auto Review Result: \n{review_result}') event_manager['push_reviewed'].send(PushReviewEntity( project_name=webhook_data['repository']['name'], @@ -210,11 +381,17 @@ def handle_github_push_event(webhook_data: dict, github_token: str, github_url: webhook_data=webhook_data, additions=additions, deletions=deletions, + note_url=note_url, + project_config=project_config, )) except Exception as e: error_message = f'服务出现未知错误: {str(e)}\n{traceback.format_exc()}' - notifier.send_notification(content=error_message) + # 尝试获取project_config，如果异常发生在配置加载之前则为None + try: + notifier.send_notification(content=error_message, project_config=project_config) + except NameError: + notifier.send_notification(content=error_message) logger.error('出现未知错误: %s', error_message) @@ -227,8 +404,28 @@ def handle_github_pull_request_event(webhook_data: dict, github_token: str, gith :param github_url_slug: :return: ''' - merge_review_only_protected_branches = os.environ.get('MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED', '0') == '1' + # 初始化project_config为None，确保在异常处理中可以访问 + project_config = None try: + # 提取项目路径 + project_path = webhook_data.get('repository', {}).get('full_name', '') + logger.info(f'Project path: {project_path}') + + # 加载项目专属配置（不修改全局环境变量） + project_config = config_loader.get_config(project_path=project_path) + + # 检查白名单（传递project_config确保配置隔离） + if not check_project_whitelist(project_path, project_config=project_config): + logger.info(f'项目 {project_path} 不在白名单中，跳过GitHub Pull Request Review') + return + logger.info(f'项目 {project_path} 使用独立配置上下文') + + # 从项目配置中读取 GITHUB_ACCESS_TOKEN + github_token = project_config.get('GITHUB_ACCESS_TOKEN') or github_token + + # 检查是否仅review protected branches + merge_review_only_protected_branches = project_config.get('MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED', '0') == '1' + # 解析Webhook数据 handler = GithubPullRequestHandler(webhook_data, github_token, github_url) logger.info('GitHub Pull Request event received') @@ -248,7 +445,8 @@ def handle_github_pull_request_event(webhook_data: dict, github_token: str, gith source_branch = webhook_data['pull_request']['head']['ref'] target_branch = webhook_data['pull_request']['base']['ref'] - if ReviewService.check_mr_last_commit_id_exists(project_name, source_branch, target_branch, github_last_commit_id): + # 创建ReviewService实例并调用方法 + if ReviewService().check_mr_last_commit_id_exists(project_name, source_branch, target_branch, github_last_commit_id): logger.info(f"Pull Request with last_commit_id {github_last_commit_id} already exists, skipping review for {project_name}.") return @@ -256,7 +454,7 @@ def handle_github_pull_request_event(webhook_data: dict, github_token: str, gith # 获取Pull Request的changes changes = handler.get_pull_request_changes() logger.info('changes: %s', changes) - changes = filter_github_changes(changes) + changes = filter_github_changes(changes, project_config) if not changes: logger.info('未检测到有关代码的修改,修改文件可能不满足SUPPORTED_EXTENSIONS。') return @@ -275,7 +473,7 @@ def handle_github_pull_request_event(webhook_data: dict, github_token: str, gith # review 代码 commits_text = ';'.join(commit['title'] for commit in commits) - review_result = CodeReviewer().review_and_strip_code(str(changes), commits_text) + review_result = CodeReviewer(project_path=project_path, config=project_config).review_and_strip_code(str(changes), commits_text) # 将review结果提交到GitHub的 notes handler.add_pull_request_notes(f'Auto Review Result: \n{review_result}') @@ -297,11 +495,16 @@ def handle_github_pull_request_event(webhook_data: dict, github_token: str, gith additions=additions, deletions=deletions, last_commit_id=github_last_commit_id, + project_config=project_config, )) except Exception as e: error_message = f'服务出现未知错误: {str(e)}\n{traceback.format_exc()}' - notifier.send_notification(content=error_message) + # 尝试获取project_config，如果异常发生在配置加载之前则为None + try: + notifier.send_notification(content=error_message, project_config=project_config) + except NameError: + notifier.send_notification(content=error_message) logger.error('出现未知错误: %s', error_message) diff --git a/biz/service/db/__init__.py b/biz/service/db/__init__.py new file mode 100644 index 000000000..9b4eb5eed --- /dev/null +++ b/biz/service/db/__init__.py @@ -0,0 +1,6 @@ +from biz.service.db.base_db_service import BaseDBService +from biz.service.db.sqlite_service import SQLiteService +from biz.service.db.mysql_service import MySQLService +from biz.service.db.db_service_factory import DBServiceFactory + +__all__ = ['BaseDBService', 'SQLiteService', 'MySQLService', 'DBServiceFactory'] diff --git a/biz/service/db/base_db_service.py b/biz/service/db/base_db_service.py new file mode 100644 index 000000000..b124dbd91 --- /dev/null +++ b/biz/service/db/base_db_service.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod +from typing import Optional +import pandas as pd +from biz.entity.review_entity import MergeRequestReviewEntity, PushReviewEntity + + +class BaseDBService(ABC): + """数据库服务基类，定义统一的数据库操作接口""" + + @abstractmethod + def init_db(self): + """初始化数据库及表结构""" + pass + + @abstractmethod + def insert_mr_review_log(self, entity: MergeRequestReviewEntity): + """插入合并请求审核日志""" + pass + + @abstractmethod + def get_mr_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的合并请求审核日志""" + pass + + @abstractmethod + def check_mr_last_commit_id_exists(self, project_name: str, source_branch: str, + target_branch: str, last_commit_id: str) -> bool: + """检查指定项目的Merge Request是否已经存在相同的last_commit_id""" + pass + + @abstractmethod + def insert_push_review_log(self, entity: PushReviewEntity): + """插入推送审核日志""" + pass + + @abstractmethod + def get_push_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的推送审核日志""" + pass diff --git a/biz/service/db/db_service_factory.py b/biz/service/db/db_service_factory.py new file mode 100644 index 000000000..ce94781a8 --- /dev/null +++ b/biz/service/db/db_service_factory.py @@ -0,0 +1,72 @@ +import os +from biz.service.db.base_db_service import BaseDBService +from biz.service.db.sqlite_service import SQLiteService +from biz.service.db.mysql_service import MySQLService +from biz.utils.log import logger + + +class DBServiceFactory: + """数据库服务工厂类，根据配置创建相应的数据库服务实例""" + + _instance = None + + @staticmethod + def create_db_service() -> BaseDBService: + """ + 根据环境变量配置创建数据库服务实例 + 支持的数据库类型：sqlite（默认）、mysql + + 环境变量配置： + - DB_TYPE: 数据库类型，可选值：sqlite, mysql + - DB_FILE: SQLite数据库文件路径（仅当DB_TYPE=sqlite时使用） + - DB_HOST: MySQL数据库主机地址（仅当DB_TYPE=mysql时使用） + - DB_PORT: MySQL数据库端口（仅当DB_TYPE=mysql时使用） + - DB_USER: MySQL数据库用户名（仅当DB_TYPE=mysql时使用） + - DB_PASSWORD: MySQL数据库密码（仅当DB_TYPE=mysql时使用） + - DB_NAME: MySQL数据库名称（仅当DB_TYPE=mysql时使用） + + :return: 数据库服务实例 + """ + db_type = os.environ.get('DB_TYPE', 'sqlite').lower() + + if db_type == 'mysql': + # MySQL配置 + host = os.environ.get('DB_HOST', 'localhost') + port = int(os.environ.get('DB_PORT', '3306')) + user = os.environ.get('DB_USER', 'root') + password = os.environ.get('DB_PASSWORD', '') + database = os.environ.get('DB_NAME', 'ai_codereview') + + if not password: + logger.warning("MySQL密码未配置，使用空密码可能存在安全风险") + + logger.info(f"使用MySQL数据库: {host}:{port}/{database}") + return MySQLService( + host=host, + port=port, + user=user, + password=password, + database=database + ) + else: + # SQLite配置（默认） + db_file = os.environ.get('DB_FILE', 'data/data.db') + logger.info(f"使用SQLite数据库: {db_file}") + return SQLiteService(db_file=db_file) + + @classmethod + def get_instance(cls) -> BaseDBService: + """ + 获取数据库服务单例实例 + :return: 数据库服务实例 + """ + if cls._instance is None: + logger.info("创建数据库服务实例") + cls._instance = cls.create_db_service() + cls._instance.init_db() + return cls._instance + + @classmethod + def reset_instance(cls): + """重置单例实例（主要用于测试）""" + cls._instance = None diff --git a/biz/service/db/mysql_service.py b/biz/service/db/mysql_service.py new file mode 100644 index 000000000..9386f7c01 --- /dev/null +++ b/biz/service/db/mysql_service.py @@ -0,0 +1,234 @@ +import pymysql +from typing import Optional +import pandas as pd +from biz.entity.review_entity import MergeRequestReviewEntity, PushReviewEntity +from biz.service.db.base_db_service import BaseDBService +from biz.utils.log import logger + + +class MySQLService(BaseDBService): + """MySQL数据库服务实现""" + + def __init__(self, host: str, port: int, user: str, password: str, database: str): + self.host = host + self.port = port + self.user = user + self.password = password + self.database = database + + def _get_connection(self): + """获取数据库连接""" + return pymysql.connect( + host=self.host, + port=self.port, + user=self.user, + password=self.password, + db=self.database, + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + def init_db(self): + """初始化数据库及表结构""" + try: + conn = self._get_connection() + try: + with conn.cursor() as cursor: + # 创建 mr_review_log 表 + cursor.execute(''' + CREATE TABLE IF NOT EXISTS mr_review_log ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + project_name VARCHAR(255), + author VARCHAR(255), + source_branch VARCHAR(255), + target_branch VARCHAR(255), + updated_at BIGINT, + commit_messages TEXT, + score INT, + url VARCHAR(500), + review_result LONGTEXT, + additions INT DEFAULT 0, + deletions INT DEFAULT 0, + last_commit_id VARCHAR(255) DEFAULT '', + INDEX idx_updated_at (updated_at), + INDEX idx_project_name (project_name), + INDEX idx_author (author) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + ''') + + # 创建 push_review_log 表 + cursor.execute(''' + CREATE TABLE IF NOT EXISTS push_review_log ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + project_name VARCHAR(255), + author VARCHAR(255), + branch VARCHAR(255), + updated_at BIGINT, + commit_messages TEXT, + score INT, + review_result LONGTEXT, + additions INT DEFAULT 0, + deletions INT DEFAULT 0, + INDEX idx_updated_at (updated_at), + INDEX idx_project_name (project_name), + INDEX idx_author (author) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + ''') + + conn.commit() + logger.info("MySQL数据库初始化成功") + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"MySQL数据库初始化失败: {e}") + + def insert_mr_review_log(self, entity: MergeRequestReviewEntity): + """插入合并请求审核日志""" + try: + conn = self._get_connection() + try: + with conn.cursor() as cursor: + cursor.execute(''' + INSERT INTO mr_review_log (project_name, author, source_branch, target_branch, + updated_at, commit_messages, score, url, review_result, additions, deletions, + last_commit_id) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ''', + (entity.project_name, entity.author, entity.source_branch, + entity.target_branch, entity.updated_at, entity.commit_messages, entity.score, + entity.url, entity.review_result, entity.additions, entity.deletions, + entity.last_commit_id)) + conn.commit() + logger.info(f"插入MR审核日志成功: {entity.project_name}#{entity.source_branch}->{entity.target_branch}") + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"插入MR审核日志失败: {e}") + + def get_mr_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的合并请求审核日志""" + try: + conn = self._get_connection() + try: + query = """ + SELECT project_name, author, source_branch, target_branch, updated_at, + commit_messages, score, url, review_result, additions, deletions + FROM mr_review_log + WHERE 1=1 + """ + params = [] + + if authors: + placeholders = ','.join(['%s'] * len(authors)) + query += f" AND author IN ({placeholders})" + params.extend(authors) + + if project_names: + placeholders = ','.join(['%s'] * len(project_names)) + query += f" AND project_name IN ({placeholders})" + params.extend(project_names) + + if updated_at_gte is not None: + query += " AND updated_at >= %s" + params.append(updated_at_gte) + + if updated_at_lte is not None: + query += " AND updated_at <= %s" + params.append(updated_at_lte) + + query += " ORDER BY updated_at DESC" + + df = pd.read_sql_query(sql=query, con=conn, params=params) + logger.info(f"查询MR审核日志成功: 条数={len(df)}, 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return df + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"获取MR审核日志失败: {e}") + return pd.DataFrame() + + def check_mr_last_commit_id_exists(self, project_name: str, source_branch: str, + target_branch: str, last_commit_id: str) -> bool: + """检查指定项目的Merge Request是否已经存在相同的last_commit_id""" + try: + conn = self._get_connection() + try: + with conn.cursor() as cursor: + cursor.execute(''' + SELECT COUNT(*) as count FROM mr_review_log + WHERE project_name = %s AND source_branch = %s AND target_branch = %s AND last_commit_id = %s + ''', (project_name, source_branch, target_branch, last_commit_id)) + result = cursor.fetchone() + exists = result['count'] > 0 if result else False + logger.info(f"检查last_commit_id: {project_name}#{source_branch}->{target_branch}#{last_commit_id}, 结果={exists}") + return exists + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"检查last_commit_id失败: {e}") + return False + + def insert_push_review_log(self, entity: PushReviewEntity): + """插入推送审核日志""" + try: + conn = self._get_connection() + try: + with conn.cursor() as cursor: + cursor.execute(''' + INSERT INTO push_review_log (project_name, author, branch, updated_at, + commit_messages, score, review_result, additions, deletions) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ''', + (entity.project_name, entity.author, entity.branch, + entity.updated_at, entity.commit_messages, entity.score, + entity.review_result, entity.additions, entity.deletions)) + conn.commit() + logger.info(f"插入Push审核日志成功: {entity.project_name}#{entity.branch}") + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"插入Push审核日志失败: {e}") + + def get_push_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的推送审核日志""" + try: + conn = self._get_connection() + try: + query = """ + SELECT project_name, author, branch, updated_at, commit_messages, + score, review_result, additions, deletions + FROM push_review_log + WHERE 1=1 + """ + params = [] + + if authors: + placeholders = ','.join(['%s'] * len(authors)) + query += f" AND author IN ({placeholders})" + params.extend(authors) + + if project_names: + placeholders = ','.join(['%s'] * len(project_names)) + query += f" AND project_name IN ({placeholders})" + params.extend(project_names) + + if updated_at_gte is not None: + query += " AND updated_at >= %s" + params.append(updated_at_gte) + + if updated_at_lte is not None: + query += " AND updated_at <= %s" + params.append(updated_at_lte) + + query += " ORDER BY updated_at DESC" + + df = pd.read_sql_query(sql=query, con=conn, params=params) + logger.info(f"查询Push审核日志成功: 条数={len(df)}, 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return df + finally: + conn.close() + except pymysql.Error as e: + logger.error(f"获取Push审核日志失败: {e}") + return pd.DataFrame() diff --git a/biz/service/db/sqlite_service.py b/biz/service/db/sqlite_service.py new file mode 100644 index 000000000..116f8976b --- /dev/null +++ b/biz/service/db/sqlite_service.py @@ -0,0 +1,219 @@ +import sqlite3 +from typing import Optional +import pandas as pd +from biz.entity.review_entity import MergeRequestReviewEntity, PushReviewEntity +from biz.service.db.base_db_service import BaseDBService +from biz.utils.log import logger + + +class SQLiteService(BaseDBService): + """SQLite数据库服务实现""" + + def __init__(self, db_file: str = "data/data.db"): + self.db_file = db_file + + def init_db(self): + """初始化数据库及表结构""" + try: + with sqlite3.connect(self.db_file) as conn: + cursor = conn.cursor() + cursor.execute(''' + CREATE TABLE IF NOT EXISTS mr_review_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_name TEXT, + author TEXT, + source_branch TEXT, + target_branch TEXT, + updated_at INTEGER, + commit_messages TEXT, + score INTEGER, + url TEXT, + review_result TEXT, + additions INTEGER DEFAULT 0, + deletions INTEGER DEFAULT 0, + last_commit_id TEXT DEFAULT '' + ) + ''') + cursor.execute(''' + CREATE TABLE IF NOT EXISTS push_review_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_name TEXT, + author TEXT, + branch TEXT, + updated_at INTEGER, + commit_messages TEXT, + score INTEGER, + review_result TEXT, + additions INTEGER DEFAULT 0, + deletions INTEGER DEFAULT 0 + ) + ''') + # 确保旧版本的mr_review_log、push_review_log表添加additions、deletions列 + tables = ["mr_review_log", "push_review_log"] + columns = ["additions", "deletions"] + for table in tables: + cursor.execute(f"PRAGMA table_info({table})") + current_columns = [col[1] for col in cursor.fetchall()] + for column in columns: + if column not in current_columns: + cursor.execute(f"ALTER TABLE {table} ADD COLUMN {column} INTEGER DEFAULT 0") + + # 为旧版本的mr_review_log表添加last_commit_id字段 + mr_columns = [ + { + "name": "last_commit_id", + "type": "TEXT", + "default": "''" + } + ] + cursor.execute(f"PRAGMA table_info('mr_review_log')") + current_columns = [col[1] for col in cursor.fetchall()] + for column in mr_columns: + if column.get("name") not in current_columns: + cursor.execute(f"ALTER TABLE mr_review_log ADD COLUMN {column.get('name')} {column.get('type')} " + f"DEFAULT {column.get('default')}") + + conn.commit() + # 添加时间字段索引（默认查询就需要时间范围） + conn.execute('CREATE INDEX IF NOT EXISTS idx_push_review_log_updated_at ON ' + 'push_review_log (updated_at);') + conn.execute('CREATE INDEX IF NOT EXISTS idx_mr_review_log_updated_at ON mr_review_log (updated_at);') + logger.info("SQLite数据库初始化成功") + except sqlite3.DatabaseError as e: + logger.error(f"SQLite数据库初始化失败: {e}") + + def insert_mr_review_log(self, entity: MergeRequestReviewEntity): + """插入合并请求审核日志""" + try: + with sqlite3.connect(self.db_file) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO mr_review_log (project_name,author, source_branch, target_branch, + updated_at, commit_messages, score, url,review_result, additions, deletions, + last_commit_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', + (entity.project_name, entity.author, entity.source_branch, + entity.target_branch, entity.updated_at, entity.commit_messages, entity.score, + entity.url, entity.review_result, entity.additions, entity.deletions, + entity.last_commit_id)) + conn.commit() + logger.info(f"插入MR审核日志成功: {entity.project_name}#{entity.source_branch}->{entity.target_branch}") + except sqlite3.DatabaseError as e: + logger.error(f"插入MR审核日志失败: {e}") + + def get_mr_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的合并请求审核日志""" + try: + with sqlite3.connect(self.db_file) as conn: + query = """ + SELECT project_name, author, source_branch, target_branch, updated_at, commit_messages, score, url, review_result, additions, deletions + FROM mr_review_log + WHERE 1=1 + """ + params = [] + + if authors: + placeholders = ','.join(['?'] * len(authors)) + query += f" AND author IN ({placeholders})" + params.extend(authors) + + if project_names: + placeholders = ','.join(['?'] * len(project_names)) + query += f" AND project_name IN ({placeholders})" + params.extend(project_names) + + if updated_at_gte is not None: + query += " AND updated_at >= ?" + params.append(updated_at_gte) + + if updated_at_lte is not None: + query += " AND updated_at <= ?" + params.append(updated_at_lte) + query += " ORDER BY updated_at DESC" + df = pd.read_sql_query(sql=query, con=conn, params=params) + logger.info(f"查询MR审核日志成功: 条数={len(df)}, 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return df + except sqlite3.DatabaseError as e: + logger.error(f"获取MR审核日志失败: {e}") + return pd.DataFrame() + + def check_mr_last_commit_id_exists(self, project_name: str, source_branch: str, + target_branch: str, last_commit_id: str) -> bool: + """检查指定项目的Merge Request是否已经存在相同的last_commit_id""" + try: + with sqlite3.connect(self.db_file) as conn: + cursor = conn.cursor() + cursor.execute(''' + SELECT COUNT(*) FROM mr_review_log + WHERE project_name = ? AND source_branch = ? AND target_branch = ? AND last_commit_id = ? + ''', (project_name, source_branch, target_branch, last_commit_id)) + count = cursor.fetchone()[0] + logger.info(f"检查last_commit_id: {project_name}#{source_branch}->{target_branch}#{last_commit_id}, 结果={count > 0}") + return count > 0 + except sqlite3.DatabaseError as e: + logger.error(f"检查last_commit_id失败: {e}") + return False + + def insert_push_review_log(self, entity: PushReviewEntity): + """插入推送审核日志""" + try: + with sqlite3.connect(self.db_file) as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO push_review_log (project_name,author, branch, updated_at, commit_messages, score,review_result, additions, deletions) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', + (entity.project_name, entity.author, entity.branch, + entity.updated_at, entity.commit_messages, entity.score, + entity.review_result, entity.additions, entity.deletions)) + conn.commit() + logger.info(f"插入Push审核日志成功: {entity.project_name}#{entity.branch}") + except sqlite3.DatabaseError as e: + logger.error(f"插入Push审核日志失败: {e}") + + def get_push_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: + """获取符合条件的推送审核日志""" + try: + with sqlite3.connect(self.db_file) as conn: + # 基础查询 + query = """ + SELECT project_name, author, branch, updated_at, commit_messages, score, review_result, additions, deletions + FROM push_review_log + WHERE 1=1 + """ + params = [] + + # 动态添加 authors 条件 + if authors: + placeholders = ','.join(['?'] * len(authors)) + query += f" AND author IN ({placeholders})" + params.extend(authors) + + if project_names: + placeholders = ','.join(['?'] * len(project_names)) + query += f" AND project_name IN ({placeholders})" + params.extend(project_names) + + # 动态添加 updated_at_gte 条件 + if updated_at_gte is not None: + query += " AND updated_at >= ?" + params.append(updated_at_gte) + + # 动态添加 updated_at_lte 条件 + if updated_at_lte is not None: + query += " AND updated_at <= ?" + params.append(updated_at_lte) + + # 按 updated_at 降序排序 + query += " ORDER BY updated_at DESC" + + # 执行查询 + df = pd.read_sql_query(sql=query, con=conn, params=params) + logger.info(f"查询Push审核日志成功: 条数={len(df)}, 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return df + except sqlite3.DatabaseError as e: + logger.error(f"获取Push审核日志失败: {e}") + return pd.DataFrame() diff --git a/biz/service/review_service.py b/biz/service/review_service.py index 887d6fca8..f7fed31f8 100644 --- a/biz/service/review_service.py +++ b/biz/service/review_service.py @@ -1,218 +1,52 @@ -import sqlite3 - +from typing import Optional import pandas as pd from biz.entity.review_entity import MergeRequestReviewEntity, PushReviewEntity +from biz.service.db.db_service_factory import DBServiceFactory +from biz.utils.log import logger class ReviewService: - DB_FILE = "data/data.db" - - @staticmethod - def init_db(): - """初始化数据库及表结构""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - cursor = conn.cursor() - cursor.execute(''' - CREATE TABLE IF NOT EXISTS mr_review_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - project_name TEXT, - author TEXT, - source_branch TEXT, - target_branch TEXT, - updated_at INTEGER, - commit_messages TEXT, - score INTEGER, - url TEXT, - review_result TEXT, - additions INTEGER DEFAULT 0, - deletions INTEGER DEFAULT 0, - last_commit_id TEXT DEFAULT '' - ) - ''') - cursor.execute(''' - CREATE TABLE IF NOT EXISTS push_review_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - project_name TEXT, - author TEXT, - branch TEXT, - updated_at INTEGER, - commit_messages TEXT, - score INTEGER, - review_result TEXT, - additions INTEGER DEFAULT 0, - deletions INTEGER DEFAULT 0 - ) - ''') - # 确保旧版本的mr_review_log、push_review_log表添加additions、deletions列 - tables = ["mr_review_log", "push_review_log"] - columns = ["additions", "deletions"] - for table in tables: - cursor.execute(f"PRAGMA table_info({table})") - current_columns = [col[1] for col in cursor.fetchall()] - for column in columns: - if column not in current_columns: - cursor.execute(f"ALTER TABLE {table} ADD COLUMN {column} INTEGER DEFAULT 0") - - # 为旧版本的mr_review_log表添加last_commit_id字段 - mr_columns = [ - { - "name": "last_commit_id", - "type": "TEXT", - "default": "''" - } - ] - cursor.execute(f"PRAGMA table_info('mr_review_log')") - current_columns = [col[1] for col in cursor.fetchall()] - for column in mr_columns: - if column.get("name") not in current_columns: - cursor.execute(f"ALTER TABLE mr_review_log ADD COLUMN {column.get('name')} {column.get('type')} " - f"DEFAULT {column.get('default')}") - - conn.commit() - # 添加时间字段索引（默认查询就需要时间范围） - conn.execute('CREATE INDEX IF NOT EXISTS idx_push_review_log_updated_at ON ' - 'push_review_log (updated_at);') - conn.execute('CREATE INDEX IF NOT EXISTS idx_mr_review_log_updated_at ON mr_review_log (updated_at);') - except sqlite3.DatabaseError as e: - print(f"Database initialization failed: {e}") - - @staticmethod - def insert_mr_review_log(entity: MergeRequestReviewEntity): + """审核服务类，提供统一的数据库操作接口""" + + def __init__(self): + """初始化审核服务，获取数据库服务实例""" + self._db_service = DBServiceFactory.get_instance() + logger.info("创建ReviewService实例") + + def init_db(self): + """初始化数据库及表结构（兼容旧版本调用）""" + logger.info("初始化数据库") + self._db_service.init_db() + + def insert_mr_review_log(self, entity: MergeRequestReviewEntity): """插入合并请求审核日志""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO mr_review_log (project_name,author, source_branch, target_branch, - updated_at, commit_messages, score, url,review_result, additions, deletions, - last_commit_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ''', - (entity.project_name, entity.author, entity.source_branch, - entity.target_branch, entity.updated_at, entity.commit_messages, entity.score, - entity.url, entity.review_result, entity.additions, entity.deletions, - entity.last_commit_id)) - conn.commit() - except sqlite3.DatabaseError as e: - print(f"Error inserting review log: {e}") + logger.info(f"插入MR审核日志: {entity.project_name}#{entity.source_branch}->{entity.target_branch}, 评分={entity.score}") + self._db_service.insert_mr_review_log(entity) - @staticmethod - def get_mr_review_logs(authors: list = None, project_names: list = None, updated_at_gte: int = None, - updated_at_lte: int = None) -> pd.DataFrame: + def get_mr_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: """获取符合条件的合并请求审核日志""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - query = """ - SELECT project_name, author, source_branch, target_branch, updated_at, commit_messages, score, url, review_result, additions, deletions - FROM mr_review_log - WHERE 1=1 - """ - params = [] - - if authors: - placeholders = ','.join(['?'] * len(authors)) - query += f" AND author IN ({placeholders})" - params.extend(authors) - - if project_names: - placeholders = ','.join(['?'] * len(project_names)) - query += f" AND project_name IN ({placeholders})" - params.extend(project_names) - - if updated_at_gte is not None: - query += " AND updated_at >= ?" - params.append(updated_at_gte) + logger.info(f"查询MR审核日志: 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return self._db_service.get_mr_review_logs(authors, project_names, updated_at_gte, updated_at_lte) - if updated_at_lte is not None: - query += " AND updated_at <= ?" - params.append(updated_at_lte) - query += " ORDER BY updated_at DESC" - df = pd.read_sql_query(sql=query, con=conn, params=params) - return df - except sqlite3.DatabaseError as e: - print(f"Error retrieving review logs: {e}") - return pd.DataFrame() - - @staticmethod - def check_mr_last_commit_id_exists(project_name: str, source_branch: str, target_branch: str, last_commit_id: str) -> bool: + def check_mr_last_commit_id_exists(self, project_name: str, source_branch: str, target_branch: str, last_commit_id: str) -> bool: """检查指定项目的Merge Request是否已经存在相同的last_commit_id""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - cursor = conn.cursor() - cursor.execute(''' - SELECT COUNT(*) FROM mr_review_log - WHERE project_name = ? AND source_branch = ? AND target_branch = ? AND last_commit_id = ? - ''', (project_name, source_branch, target_branch, last_commit_id)) - count = cursor.fetchone()[0] - return count > 0 - except sqlite3.DatabaseError as e: - print(f"Error checking last_commit_id: {e}") - return False + logger.info(f"检查last_commit_id: {project_name}#{source_branch}->{target_branch}#{last_commit_id}") + return self._db_service.check_mr_last_commit_id_exists(project_name, source_branch, target_branch, last_commit_id) - @staticmethod - def insert_push_review_log(entity: PushReviewEntity): + def insert_push_review_log(self, entity: PushReviewEntity): """插入推送审核日志""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - cursor = conn.cursor() - cursor.execute(''' - INSERT INTO push_review_log (project_name,author, branch, updated_at, commit_messages, score,review_result, additions, deletions) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ''', - (entity.project_name, entity.author, entity.branch, - entity.updated_at, entity.commit_messages, entity.score, - entity.review_result, entity.additions, entity.deletions)) - conn.commit() - except sqlite3.DatabaseError as e: - print(f"Error inserting review log: {e}") + logger.info(f"插入Push审核日志: {entity.project_name}#{entity.branch}, 评分={entity.score}") + self._db_service.insert_push_review_log(entity) - @staticmethod - def get_push_review_logs(authors: list = None, project_names: list = None, updated_at_gte: int = None, - updated_at_lte: int = None) -> pd.DataFrame: + def get_push_review_logs(self, authors: Optional[list] = None, project_names: Optional[list] = None, + updated_at_gte: Optional[int] = None, updated_at_lte: Optional[int] = None) -> pd.DataFrame: """获取符合条件的推送审核日志""" - try: - with sqlite3.connect(ReviewService.DB_FILE) as conn: - # 基础查询 - query = """ - SELECT project_name, author, branch, updated_at, commit_messages, score, review_result, additions, deletions - FROM push_review_log - WHERE 1=1 - """ - params = [] - - # 动态添加 authors 条件 - if authors: - placeholders = ','.join(['?'] * len(authors)) - query += f" AND author IN ({placeholders})" - params.extend(authors) - - if project_names: - placeholders = ','.join(['?'] * len(project_names)) - query += f" AND project_name IN ({placeholders})" - params.extend(project_names) - - # 动态添加 updated_at_gte 条件 - if updated_at_gte is not None: - query += " AND updated_at >= ?" - params.append(updated_at_gte) - - # 动态添加 updated_at_lte 条件 - if updated_at_lte is not None: - query += " AND updated_at <= ?" - params.append(updated_at_lte) - - # 按 updated_at 降序排序 - query += " ORDER BY updated_at DESC" - - # 执行查询 - df = pd.read_sql_query(sql=query, con=conn, params=params) - return df - except sqlite3.DatabaseError as e: - print(f"Error retrieving push review logs: {e}") - return pd.DataFrame() + logger.info(f"查询Push审核日志: 条件=[authors={authors}, projects={project_names}, time_range={updated_at_gte}-{updated_at_lte}]") + return self._db_service.get_push_review_logs(authors, project_names, updated_at_gte, updated_at_lte) -# Initialize database -ReviewService.init_db() +# 初始化数据库（向后兼容） +logger.info("初始化数据库服务") +DBServiceFactory.get_instance().init_db() diff --git a/biz/utils/code_reviewer.py b/biz/utils/code_reviewer.py index a277ac59e..f8fddb3c9 100644 --- a/biz/utils/code_reviewer.py +++ b/biz/utils/code_reviewer.py @@ -1,12 +1,13 @@ import abc import os import re -from typing import Dict, Any, List +from typing import Dict, Any, List, Optional import yaml from jinja2 import Template from biz.llm.factory import Factory +from biz.utils.config_loader import config_loader from biz.utils.log import logger from biz.utils.token_util import count_tokens, truncate_text_by_tokens @@ -14,29 +15,32 @@ class BaseReviewer(abc.ABC): """代码审查基类""" - def __init__(self, prompt_key: str): - self.client = Factory().getClient() - self.prompts = self._load_prompts(prompt_key, os.getenv("REVIEW_STYLE", "professional")) + def __init__(self, prompt_key: str, app_name: Optional[str] = None, project_path: Optional[str] = None, config: Optional[Dict[str, str]] = None): + self.config = config or {} # 项目专属配置 + self.client = Factory().getClient(config=self.config) + self.app_name = app_name + self.project_path = project_path + # 从config中读取REVIEW_STYLE（已包含默认值） + review_style = self.config.get("REVIEW_STYLE", "professional") + self.prompts = self._load_prompts(prompt_key, review_style) def _load_prompts(self, prompt_key: str, style="professional") -> Dict[str, Any]: """加载提示词配置""" - prompt_templates_file = "conf/prompt_templates.yml" try: - # 在打开 YAML 文件时显式指定编码为 UTF-8，避免使用系统默认的 GBK 编码。 - with open(prompt_templates_file, "r", encoding="utf-8") as file: - prompts = yaml.safe_load(file).get(prompt_key, {}) + # 使用ConfigLoader加载Prompt模板 + prompts: dict[Any, Any] = config_loader.load_prompt_template(prompt_key, self.app_name, self.project_path) - # 使用Jinja2渲染模板 - def render_template(template_str: str) -> str: - return Template(template_str).render(style=style) + # 使用Jinja2渲染模板 + def render_template(template_str: str) -> str: + return Template(template_str).render(style=style) - system_prompt = render_template(prompts["system_prompt"]) - user_prompt = render_template(prompts["user_prompt"]) + system_prompt = render_template(prompts["system_prompt"]) + user_prompt = render_template(prompts["user_prompt"]) - return { - "system_message": {"role": "system", "content": system_prompt}, - "user_message": {"role": "user", "content": user_prompt}, - } + return { + "system_message": {"role": "system", "content": system_prompt}, + "user_message": {"role": "user", "content": user_prompt}, + } except (FileNotFoundError, KeyError, yaml.YAMLError) as e: logger.error(f"加载提示词配置失败: {e}") raise Exception(f"提示词配置加载失败: {e}") @@ -57,8 +61,8 @@ def review_code(self, *args, **kwargs) -> str: class CodeReviewer(BaseReviewer): """代码 Diff 级别的审查""" - def __init__(self): - super().__init__("code_review_prompt") + def __init__(self, app_name: Optional[str] = None, project_path: Optional[str] = None, config: Optional[Dict[str, str]] = None): + super().__init__("code_review_prompt", app_name, project_path, config) def review_and_strip_code(self, changes_text: str, commits_text: str = "") -> str: """ @@ -68,8 +72,8 @@ def review_and_strip_code(self, changes_text: str, commits_text: str = "") -> st :param commits_text: :return: """ - # 如果超长，取前REVIEW_MAX_TOKENS个token - review_max_tokens = int(os.getenv("REVIEW_MAX_TOKENS", 10000)) + # 从config中读取REVIEW_MAX_TOKENS（已包含默认值） + review_max_tokens = int(self.config.get("REVIEW_MAX_TOKENS", "10000")) # 如果changes为空,打印日志 if not changes_text: logger.info("代码为空, diffs_text = %", str(changes_text)) diff --git a/biz/utils/config_checker.py b/biz/utils/config_checker.py index cfd028e2d..4d6d169ab 100644 --- a/biz/utils/config_checker.py +++ b/biz/utils/config_checker.py @@ -15,7 +15,7 @@ ] # 允许的 LLM 供应商 -LLM_PROVIDERS = {"zhipuai", "openai", "deepseek", "ollama", "qwen"} +LLM_PROVIDERS = {"zhipuai", "openai", "deepseek", "ollama", "qwen", "kimi"} # 每种供应商必须配置的键 LLM_REQUIRED_KEYS = { @@ -24,6 +24,7 @@ "deepseek": ["DEEPSEEK_API_KEY", "DEEPSEEK_API_MODEL"], "ollama": ["OLLAMA_API_BASE_URL", "OLLAMA_API_MODEL"], "qwen": ["QWEN_API_KEY", "QWEN_API_MODEL"], + "kimi": ["KIMI_API_KEY", "KIMI_API_MODEL"], } @@ -57,8 +58,13 @@ def check_llm_provider(): logger.info(f"LLM 供应商 {llm_provider} 的配置项已设置。") def check_llm_connectivity(): - client = Factory().getClient() - logger.info(f"正在检查 LLM 供应商的连接...") + llm_provider = os.getenv("LLM_PROVIDER") + if not llm_provider: + logger.error("LLM_PROVIDER 未设置，跳过连接检查。") + return + + client = Factory().getClient(provider=llm_provider) + logger.info(f"正在检查 LLM 供应商 {llm_provider} 的连接...") if client.ping(): logger.info("LLM 可以连接成功。") else: diff --git a/biz/utils/config_loader.py b/biz/utils/config_loader.py new file mode 100644 index 000000000..8307913ed --- /dev/null +++ b/biz/utils/config_loader.py @@ -0,0 +1,200 @@ +import os +from pathlib import Path +from typing import Optional, Dict +from dotenv import load_dotenv, dotenv_values +import yaml + +from biz.utils.log import logger + + +class ConfigLoader: + """ + 配置加载器，支持按应用名独立目录配置 + 配置文件优先级：conf/{app_name}/.env > conf/.env + """ + + # 默认配置目录 + DEFAULT_CONF_DIR = "conf" + + # 环境变量文件名 + ENV_FILE_NAME = ".env" + + # Prompt模板文件名 + PROMPT_TEMPLATE_FILE_NAME = "prompt_templates.yml" + + _instance = None + _initialized = False + + def __new__(cls): + """单例模式""" + if cls._instance is None: + cls._instance = super(ConfigLoader, cls).__new__(cls) + return cls._instance + + def __init__(self): + """初始化配置加载器""" + if not ConfigLoader._initialized: + self.app_name = None + ConfigLoader._initialized = True + + def set_app_name(self, app_name: str): + """设置应用名称""" + self.app_name = app_name + logger.info(f"ConfigLoader设置应用名称: {app_name}") + + def get_env_file_path(self, app_name: Optional[str] = None, project_path: Optional[str] = None) -> str: + """ + 获取环境变量配置文件路径 + 优先级：项目级别配置 > 命名空间级别配置 > 默认配置 + :param app_name: 应用名称（URL slug，已废弃，保留用于兼容） + :param project_path: 项目路径（如：asset/asset-batch-center） + :return: 配置文件路径 + """ + if project_path and '/' in project_path: + # 提取命名空间和项目名 + parts = project_path.split('/', 1) + namespace = parts[0] + project_name = parts[1] if len(parts) > 1 else '' + + # 1. 优先查找项目级别配置: conf/{namespace}/{project_name}/.env + if project_name: + project_env_path = Path(self.DEFAULT_CONF_DIR) / namespace / project_name / self.ENV_FILE_NAME + if project_env_path.exists(): + logger.info(msg=f"使用项目级别配置: {project_env_path}") + return str(project_env_path) + else: + logger.debug(f"项目级别配置不存在 ({project_env_path})") + + # 2. 查找命名空间级别配置: conf/{namespace}/.env + namespace_env_path = Path(self.DEFAULT_CONF_DIR) / namespace / self.ENV_FILE_NAME + if namespace_env_path.exists(): + logger.info(msg=f"使用命名空间级别配置: {namespace_env_path}") + return str(namespace_env_path) + else: + logger.debug(f"命名空间级别配置不存在 ({namespace_env_path})") + + # 3. 使用默认配置: conf/.env + default_env_path = Path(self.DEFAULT_CONF_DIR) / self.ENV_FILE_NAME + logger.info(msg=f"使用默认配置: {default_env_path}") + return str(default_env_path) + + def load_env(self, app_name: Optional[str] = None, project_path: Optional[str] = None, override: bool = False): + """ + 加载环境变量配置 + :param app_name: 应用名称（URL slug） + :param project_path: 项目路径（如：asset/asset-batch-center） + :param override: 是否覆盖已存在的环境变量 + """ + env_path = self.get_env_file_path(app_name, project_path) + if os.path.exists(env_path): + load_dotenv(dotenv_path=env_path, override=override) + logger.info(msg=f"成功加载环境变量配置: {env_path}") + else: + logger.warning(f"环境变量配置文件不存在: {env_path}") + + def get_prompt_template_file_path(self, app_name: Optional[str] = None, project_path: Optional[str] = None) -> str: + """ + 获取Prompt模板配置文件路径 + 优先级：项目级别配置 > 命名空间级别配置 > 默认配置 + :param app_name: 应用名称（URL slug，已废弃，保留用于兼容） + :param project_path: 项目路径（如：asset/asset-batch-center） + :return: 配置文件路径 + """ + if project_path and '/' in project_path: + # 提取命名空间和项目名 + parts = project_path.split('/', 1) + namespace = parts[0] + project_name = parts[1] if len(parts) > 1 else '' + + # 1. 优先查找项目级别配置: conf/{namespace}/{project_name}/prompt_templates.yml + if project_name: + project_template_path = Path(self.DEFAULT_CONF_DIR) / namespace / project_name / self.PROMPT_TEMPLATE_FILE_NAME + if project_template_path.exists(): + logger.info(msg=f"使用项目级别Prompt模板: {project_template_path}") + return str(project_template_path) + else: + logger.debug(f"项目级别Prompt模板不存在 ({project_template_path})") + + # 2. 查找命名空间级别配置: conf/{namespace}/prompt_templates.yml + namespace_template_path = Path(self.DEFAULT_CONF_DIR) / namespace / self.PROMPT_TEMPLATE_FILE_NAME + if namespace_template_path.exists(): + logger.info(msg=f"使用命名空间级别Prompt模板: {namespace_template_path}") + return str(namespace_template_path) + else: + logger.debug(f"命名空间级别Prompt模板不存在 ({namespace_template_path})") + + # 3. 使用默认配置: conf/prompt_templates.yml + default_template_path = Path(self.DEFAULT_CONF_DIR) / self.PROMPT_TEMPLATE_FILE_NAME + logger.info(msg=f"使用默认Prompt模板: {default_template_path}") + return str(default_template_path) + + def load_prompt_template(self, prompt_key: str, app_name: Optional[str] = None, project_path: Optional[str] = None) -> dict: + """ + 加载Prompt模板配置 + :param prompt_key: 提示词配置键 + :param app_name: 应用名称（URL slug） + :param project_path: 项目路径（如：asset/asset-batch-center） + :return: 提示词配置字典 + """ + template_path = self.get_prompt_template_file_path(app_name, project_path) + + try: + with open(template_path, "r", encoding="utf-8") as file: + templates = yaml.safe_load(file) + prompts = templates.get(prompt_key, {}) + + if not prompts: + logger.warning(f"在配置文件 {template_path} 中未找到 prompt_key: {prompt_key}") + else: + logger.info(f"成功加载Prompt模板 {prompt_key} from {template_path}") + + return prompts + except (FileNotFoundError, KeyError, yaml.YAMLError) as e: + logger.error(f"加载Prompt模板配置失败: {e}") + raise Exception(f"Prompt模板配置加载失败: {e}") + + def get_config(self, app_name: Optional[str] = None, project_path: Optional[str] = None) -> Dict[str, str]: + """ + 获取项目专属配置字典（不修改全局环境变量） + :param app_name: 应用名称（URL slug，已废弃，保留用于兼容） + :param project_path: 项目路径（如：asset/asset-batch-center） + :return: 配置字典，包含从.env文件加载的所有配置项 + """ + env_path = self.get_env_file_path(app_name, project_path) + + # 首先加载默认配置 + default_env_path = Path(self.DEFAULT_CONF_DIR) / self.ENV_FILE_NAME + config = {} + + # 加载默认配置作为基础 + if os.path.exists(default_env_path): + config.update(dotenv_values(default_env_path)) + logger.debug(f"加载默认配置: {default_env_path}") + + # 如果有项目专属配置，覆盖默认配置 + if env_path != str(default_env_path) and os.path.exists(env_path): + config.update(dotenv_values(env_path)) + logger.info(f"加载项目配置并覆盖默认值: {env_path}") + + # 与全局环境变量合并（全局环境变量优先级最低，仅用于未在配置文件中定义的值） + for key in config.keys(): + if config[key] is None and key in os.environ: + config[key] = os.environ[key] + + return config + + @staticmethod + def create_app_config_dir(app_name: str) -> Path: + """ + 创建应用专属配置目录 + :param app_name: 应用名称（URL slug） + :return: 配置目录路径 + """ + app_conf_dir = Path(ConfigLoader.DEFAULT_CONF_DIR) / app_name + app_conf_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"创建应用专属配置目录: {app_conf_dir}") + return app_conf_dir + + +# 全局配置加载器实例 +config_loader = ConfigLoader() diff --git a/biz/utils/im/dingtalk.py b/biz/utils/im/dingtalk.py index 17c99e24d..7fe931460 100644 --- a/biz/utils/im/dingtalk.py +++ b/biz/utils/im/dingtalk.py @@ -12,18 +12,46 @@ class DingTalkNotifier: - def __init__(self, webhook_url=None): - self.enabled = os.environ.get('DINGTALK_ENABLED', '0') == '1' - self.default_webhook_url = webhook_url or os.environ.get('DINGTALK_WEBHOOK_URL') + def __init__(self, webhook_url=None, project_config=None): + """ + 初始化钉钉通知器 + :param webhook_url: 钉钉机器人webhook地址 + :param project_config: 项目专属配置字典 + """ + self.project_config = project_config or {} + # 优先从 project_config 获取，如果没有则降级到 os.environ + self.enabled = self._get_enabled_status('DINGTALK_ENABLED', '0') + self.default_webhook_url = webhook_url or self.project_config.get('DINGTALK_WEBHOOK_URL') or os.environ.get('DINGTALK_WEBHOOK_URL') + + def _get_enabled_status(self, config_key, default_value='0'): + """获取启用状态，修复逻辑错误""" + enabled = self.project_config.get(config_key) + if enabled is None: + enabled = os.environ.get(config_key, default_value) + return enabled == '1' - def _get_webhook_url(self, project_name=None, url_slug=None): + def _get_webhook_url(self, project_name=None, url_slug=None, msg_category=None): """ 获取项目对应的 Webhook URL :param project_name: 项目名称 :param url_slug: 由 gitlab 项目的 url 转换而来的 slug + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook :return: Webhook URL :raises ValueError: 如果未找到 Webhook URL """ + # 如果指定了消息类别（如日报），只使用全局默认的专用 webhook，不查找项目级别配置 + if msg_category: + category_webhook_key = f"DINGTALK_WEBHOOK_URL_{msg_category.upper()}" + # 优先从 project_config 获取，如果没有则降级到 os.environ + category_webhook_url = self.project_config.get(category_webhook_key) or os.environ.get(category_webhook_key) + if category_webhook_url: + return category_webhook_url + # 如果没有配置专用webhook，降级使用默认webhook + if self.default_webhook_url: + return self.default_webhook_url + else: + raise ValueError(f"未设置消息类别 '{msg_category}' 的专用钉钉 Webhook URL，且未设置默认的钉钉 Webhook URL。") + # 如果未提供 project_name，直接返回默认的 Webhook URL if not project_name: if self.default_webhook_url: @@ -33,30 +61,30 @@ def _get_webhook_url(self, project_name=None, url_slug=None): # 构造目标键 target_key_project = f"DINGTALK_WEBHOOK_URL_{project_name.upper()}" - target_key_url_slug = f"DINGTALK_WEBHOOK_URL_{url_slug.upper()}" + target_key_url_slug = f"DINGTALK_WEBHOOK_URL_{url_slug.upper()}" if url_slug else None - # 遍历环境变量 - for env_key, env_value in os.environ.items(): - env_key_upper = env_key.upper() - if env_key_upper == target_key_project: - return env_value # 找到项目名称对应的 Webhook URL，直接返回 - if env_key_upper == target_key_url_slug: - return env_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 + # 遍历项目配置 + for config_key, config_value in self.project_config.items(): + config_key_upper = config_key.upper() + if config_key_upper == target_key_project: + return config_value # 找到项目名称对应的 Webhook URL，直接返回 + if target_key_url_slug and config_key_upper == target_key_url_slug: + return config_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 - # 如果未找到匹配的环境变量，降级使用全局的 Webhook URL + # 如果未找到匹配的配置项，降级使用全局的 Webhook URL if self.default_webhook_url: return self.default_webhook_url # 如果既未找到匹配项，也没有默认值，抛出异常 raise ValueError(f"未找到项目 '{project_name}' 对应的钉钉Webhook URL，且未设置默认的 Webhook URL。") - def send_message(self, content: str, msg_type='text', title='通知', is_at_all=False, project_name=None, url_slug = None): + def send_message(self, content: str, msg_type='text', title='通知', is_at_all=False, project_name=None, url_slug=None, msg_category=None): if not self.enabled: logger.info("钉钉推送未启用") return try: - post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug) + post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug, msg_category=msg_category) headers = { "Content-Type": "application/json", "Charset": "UTF-8" diff --git a/biz/utils/im/feishu.py b/biz/utils/im/feishu.py index 208f4c748..2f03660ed 100644 --- a/biz/utils/im/feishu.py +++ b/biz/utils/im/feishu.py @@ -4,21 +4,46 @@ class FeishuNotifier: - def __init__(self, webhook_url=None): + def __init__(self, webhook_url=None, project_config=None): """ 初始化飞书通知器 :param webhook_url: 飞书机器人webhook地址 + :param project_config: 项目专属配置字典 """ - self.default_webhook_url = webhook_url or os.environ.get('FEISHU_WEBHOOK_URL', '') - self.enabled = os.environ.get('FEISHU_ENABLED', '0') == '1' + self.project_config = project_config or {} + # 优先从 project_config 获取，如果没有则降级到 os.environ + self.default_webhook_url = webhook_url or self.project_config.get('FEISHU_WEBHOOK_URL', '') or os.environ.get('FEISHU_WEBHOOK_URL', '') + self.enabled = self._get_enabled_status('FEISHU_ENABLED', '0') - def _get_webhook_url(self, project_name=None, url_slug=None): + def _get_enabled_status(self, config_key, default_value='0'): + """获取启用状态，修复逻辑错误""" + enabled = self.project_config.get(config_key) + if enabled is None: + enabled = os.environ.get(config_key, default_value) + return enabled == '1' + + def _get_webhook_url(self, project_name=None, url_slug=None, msg_category=None): """ 获取项目对应的 Webhook URL :param project_name: 项目名称 + :param url_slug: URL slug + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook :return: Webhook URL :raises ValueError: 如果未找到 Webhook URL """ + # 如果指定了消息类别（如日报），只使用全局默认的专用 webhook，不查找项目级别配置 + if msg_category: + category_webhook_key = f"FEISHU_WEBHOOK_URL_{msg_category.upper()}" + # 优先从 project_config 获取，如果没有则降级到 os.environ + category_webhook_url = self.project_config.get(category_webhook_key) or os.environ.get(category_webhook_key) + if category_webhook_url: + return category_webhook_url + # 如果没有配置专用webhook，降级使用默认webhook + if self.default_webhook_url: + return self.default_webhook_url + else: + raise ValueError(f"未设置消息类别 '{msg_category}' 的专用飞书 Webhook URL，且未设置默认的飞书 Webhook URL。") + # 如果未提供 project_name，直接返回默认的 Webhook URL if not project_name: if self.default_webhook_url: @@ -28,24 +53,24 @@ def _get_webhook_url(self, project_name=None, url_slug=None): # 构造目标键 target_key_project = f"FEISHU_WEBHOOK_URL_{project_name.upper()}" - target_key_url_slug = f"FEISHU_WEBHOOK_URL_{url_slug.upper()}" + target_key_url_slug = f"FEISHU_WEBHOOK_URL_{url_slug.upper()}" if url_slug else None - # 遍历环境变量 - for env_key, env_value in os.environ.items(): - env_key_upper = env_key.upper() - if env_key_upper == target_key_project: - return env_value # 找到项目名称对应的 Webhook URL，直接返回 - if env_key_upper == target_key_url_slug: - return env_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 + # 遍历项目配置 + for config_key, config_value in self.project_config.items(): + config_key_upper = config_key.upper() + if config_key_upper == target_key_project: + return config_value # 找到项目名称对应的 Webhook URL，直接返回 + if target_key_url_slug and config_key_upper == target_key_url_slug: + return config_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 - # 如果未找到匹配的环境变量，降级使用全局的 Webhook URL + # 如果未找到匹配的配置项，降级使用全局的 Webhook URL if self.default_webhook_url: return self.default_webhook_url # 如果既未找到匹配项，也没有默认值，抛出异常 raise ValueError(f"未找到项目 '{project_name}' 对应的 Feishu Webhook URL，且未设置默认的 Webhook URL。") - def send_message(self, content, msg_type='text', title=None, is_at_all=False, project_name=None, url_slug=None): + def send_message(self, content, msg_type='text', title=None, is_at_all=False, project_name=None, url_slug=None, msg_category=None): """ 发送飞书消息 :param content: 消息内容 @@ -53,13 +78,15 @@ def send_message(self, content, msg_type='text', title=None, is_at_all=False, pr :param title: 消息标题(markdown类型时使用) :param is_at_all: 是否@所有人 :param project_name: 项目名称 + :param url_slug: URL slug + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook """ if not self.enabled: logger.info("飞书推送未启用") return try: - post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug) + post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug, msg_category=msg_category) if msg_type == 'markdown': data = { "msg_type": "interactive", diff --git a/biz/utils/im/notifier.py b/biz/utils/im/notifier.py index 0fa112eda..5e3cfcb05 100644 --- a/biz/utils/im/notifier.py +++ b/biz/utils/im/notifier.py @@ -5,7 +5,7 @@ def send_notification(content, msg_type='text', title="通知", is_at_all=False, project_name=None, url_slug=None, - webhook_data: dict={}): + webhook_data: dict={}, mentioned_list=None, msg_category=None, project_config=None): """ 发送通知消息到配置的平台(钉钉和企业微信) :param content: 消息内容 @@ -14,24 +14,28 @@ def send_notification(content, msg_type='text', title="通知", is_at_all=False, :param is_at_all: 是否@所有人 :param url_slug: 由gitlab服务器的url地址(如:http://www.gitlab.com)转换成的slug格式，如: www_gitlab_com :param webhook_data: push event、merge event的数据内容 + :param mentioned_list: @指定用户列表，优先于is_at_all（仅企微和部分平台的text类型支持） + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook + :param project_config: 项目专属配置字典 """ # 钉钉推送 - dingtalk_notifier = DingTalkNotifier() + dingtalk_notifier = DingTalkNotifier(project_config=project_config) dingtalk_notifier.send_message(content=content, msg_type=msg_type, title=title, is_at_all=is_at_all, - project_name=project_name, url_slug=url_slug) + project_name=project_name, url_slug=url_slug, msg_category=msg_category) # 企业微信推送 - wecom_notifier = WeComNotifier() + wecom_notifier = WeComNotifier(project_config=project_config) wecom_notifier.send_message(content=content, msg_type=msg_type, title=title, is_at_all=is_at_all, - project_name=project_name, url_slug=url_slug) + project_name=project_name, url_slug=url_slug, mentioned_list=mentioned_list, + msg_category=msg_category) # 飞书推送 - feishu_notifier = FeishuNotifier() + feishu_notifier = FeishuNotifier(project_config=project_config) feishu_notifier.send_message(content=content, msg_type=msg_type, title=title, is_at_all=is_at_all, - project_name=project_name, url_slug=url_slug) + project_name=project_name, url_slug=url_slug, msg_category=msg_category) # 额外自定义webhook通知 - extra_webhook_notifier = ExtraWebhookNotifier() + extra_webhook_notifier = ExtraWebhookNotifier(project_config=project_config) system_data = { "content": content, "msg_type": msg_type, diff --git a/biz/utils/im/webhook.py b/biz/utils/im/webhook.py index f6a6bed59..d63fdf5da 100644 --- a/biz/utils/im/webhook.py +++ b/biz/utils/im/webhook.py @@ -4,13 +4,16 @@ class ExtraWebhookNotifier: - def __init__(self, webhook_url=None): + def __init__(self, webhook_url=None, project_config=None): """ 初始化ExtraWebhook通知器 :param webhook_url: 自定义webhook地址 + :param project_config: 项目专属配置字典 """ - self.default_webhook_url = webhook_url or os.environ.get('EXTRA_WEBHOOK_URL', '') - self.enabled = os.environ.get('EXTRA_WEBHOOK_ENABLED', '0') == '1' + self.project_config = project_config or {} + # 优先从 project_config 获取，如果没有则降级到 os.environ + self.default_webhook_url = webhook_url or self.project_config.get('EXTRA_WEBHOOK_URL', '') or os.environ.get('EXTRA_WEBHOOK_URL', '') + self.enabled = (self.project_config.get('EXTRA_WEBHOOK_ENABLED', '0') or os.environ.get('EXTRA_WEBHOOK_ENABLED', '0')) == '1' def send_message(self, system_data: dict, webhook_data: dict): """ diff --git a/biz/utils/im/wecom.py b/biz/utils/im/wecom.py index 2fbf46778..1c00cdd6a 100644 --- a/biz/utils/im/wecom.py +++ b/biz/utils/im/wecom.py @@ -6,21 +6,44 @@ class WeComNotifier: - def __init__(self, webhook_url=None): + def __init__(self, webhook_url=None, project_config=None): """ 初始化企业微信通知器 :param webhook_url: 企业微信机器人webhook地址 + :param project_config: 项目专属配置字典 """ - self.default_webhook_url = webhook_url or os.environ.get('WECOM_WEBHOOK_URL', '') - self.enabled = os.environ.get('WECOM_ENABLED', '0') == '1' + self.project_config = project_config or {} + # 优先从 project_config 获取，如果没有则降级到 os.environ + self.default_webhook_url = webhook_url or self.project_config.get('WECOM_WEBHOOK_URL', '') or os.environ.get('WECOM_WEBHOOK_URL', '') + + # 修复启用状态判断逻辑 + wecom_enabled = self.project_config.get('WECOM_ENABLED') + if wecom_enabled is None: + wecom_enabled = os.environ.get('WECOM_ENABLED', '0') + self.enabled = wecom_enabled == '1' - def _get_webhook_url(self, project_name=None, url_slug=None): + def _get_webhook_url(self, project_name=None, url_slug=None, msg_category=None): """ 获取项目对应的 Webhook URL :param project_name: 项目名称 + :param url_slug: URL slug + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook :return: Webhook URL :raises ValueError: 如果未找到 Webhook URL """ + # 如果指定了消息类别（如日报），只使用全局默认的专用 webhook，不查找项目级别配置 + if msg_category: + category_webhook_key = f"WECOM_WEBHOOK_URL_{msg_category.upper()}" + # 优先从 project_config 获取，如果没有则降级到 os.environ + category_webhook_url = self.project_config.get(category_webhook_key) or os.environ.get(category_webhook_key) + if category_webhook_url: + return category_webhook_url + # 如果没有配置专用webhook，降级使用默认webhook + if self.default_webhook_url: + return self.default_webhook_url + else: + raise ValueError(f"未设置消息类别 '{msg_category}' 的专用 Webhook URL，且未设置默认的企业微信 Webhook URL。") + # 如果未提供 project_name，直接返回默认的 Webhook URL if not project_name: if self.default_webhook_url: @@ -30,17 +53,17 @@ def _get_webhook_url(self, project_name=None, url_slug=None): # 构造目标键 target_key_project = f"WECOM_WEBHOOK_URL_{project_name.upper()}" - target_key_url_slug = f"WECOM_WEBHOOK_URL_{url_slug.upper()}" + target_key_url_slug = f"WECOM_WEBHOOK_URL_{url_slug.upper()}" if url_slug else None - # 遍历环境变量 - for env_key, env_value in os.environ.items(): - env_key_upper = env_key.upper() - if env_key_upper == target_key_project: - return env_value # 找到项目名称对应的 Webhook URL，直接返回 - if env_key_upper == target_key_url_slug: - return env_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 + # 遍历项目配置 + for config_key, config_value in self.project_config.items(): + config_key_upper = config_key.upper() + if config_key_upper == target_key_project: + return config_value # 找到项目名称对应的 Webhook URL，直接返回 + if target_key_url_slug and config_key_upper == target_key_url_slug: + return config_value # 找到 GitLab URL 对应的 Webhook URL，直接返回 - # 如果未找到匹配的环境变量，降级使用全局的 Webhook URL + # 如果未找到匹配的配置项，降级使用全局的 Webhook URL if self.default_webhook_url: return self.default_webhook_url @@ -49,7 +72,7 @@ def _get_webhook_url(self, project_name=None, url_slug=None): def format_markdown_content(self, content, title=None): """ - 格式化markdown内容以适配企业微信 + 格式化内容以适配企业微信 """ # 处理标题 formatted_content = f"## {title}\n\n" if title else "" @@ -67,7 +90,7 @@ def format_markdown_content(self, content, title=None): return formatted_content def send_message(self, content, msg_type='text', title=None, is_at_all=False, project_name=None, - url_slug=None): + url_slug=None, mentioned_list=None, msg_category=None): """ 发送企业微信消息 :param content: 消息内容 @@ -76,43 +99,81 @@ def send_message(self, content, msg_type='text', title=None, is_at_all=False, pr :param is_at_all: 是否 @所有人 :param project_name: 关联项目名称 :param url_slug: GitLab URL Slug + :param mentioned_list: @指定用户列表，优先于is_at_all（仅text类型支持） + :param msg_category: 消息类别（如：daily_report），用于区分不同场景的webhook """ if not self.enabled: logger.info("企业微信推送未启用") return try: - post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug) + post_url = self._get_webhook_url(project_name=project_name, url_slug=url_slug, msg_category=msg_category) # 企业微信消息内容最大长度限制 # text类型最大2048字节 # https://developer.work.weixin.qq.com/document/path/91770#%E6%96%87%E6%9C%AC%E7%B1%BB%E5%9E%8B # markdown类型最大4096字节 - # https://developer.work.weixin.qq.com/document/path/91770#markdown%E7%B1%BB%E5%9E%8B + # https://developer.work.weixin.qq.com/document/path/91770#%E6%96%87%E6%9C%AC%E7%B1%BB%E5%9E%8B MAX_CONTENT_BYTES = 4096 if msg_type == 'markdown' else 2048 - # 检查内容长度 - content_length = len(content.encode('utf-8')) + # 对于 markdown 类型，需要计算格式化后的实际长度（包括标题） + if msg_type == 'markdown': + # 模拟格式化后的内容，计算实际字节数 + formatted_content = self.format_markdown_content(content, title) + if mentioned_list: + mention_tags = ' '.join([f'<@{user}>' for user in (mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])]) + formatted_content = f"{formatted_content}\n\n{mention_tags}" + content_length = len(formatted_content.encode('utf-8')) + else: + # text 类型直接检查原始内容长度 + content_length = len(content.encode('utf-8')) + # text 类型如果有 mentioned_list，也需要加上 mention_tags 的长度 + if mentioned_list: + mention_tags = ' '.join([f'<@{user}>' for user in (mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])]) + content_length += len(f"\n\n{mention_tags}".encode('utf-8')) if content_length <= MAX_CONTENT_BYTES: # 内容长度在限制范围内，直接发送 - data = self._build_message(content, title, msg_type, is_at_all) + data = self._build_message(content, title, msg_type, is_at_all, mentioned_list) self._send_message(post_url, data) else: # 内容超过限制，需要分割发送 logger.warning(f"消息内容超过{MAX_CONTENT_BYTES}字节限制，将分割发送。总长度: {content_length}字节") - self._send_message_in_chunks(content, title, post_url, msg_type, is_at_all, MAX_CONTENT_BYTES) + self._send_message_in_chunks(content, title, post_url, msg_type, is_at_all, MAX_CONTENT_BYTES, mentioned_list) except Exception as e: logger.error(f"企业微信消息发送失败! {e}") - def _send_message_in_chunks(self, content, title, post_url, msg_type, is_at_all, max_bytes): + def _send_message_in_chunks(self, content, title, post_url, msg_type, is_at_all, max_bytes, mentioned_list=None): """ 将内容分割成多个部分并分别发送 """ - chunks = self._split_content(content, max_bytes) + # 对于 markdown 类型，需要预留空间给标题和 mention_tags + if msg_type == 'markdown': + # 计算标题的开销（模拟格式："## {title} (第X/Y部分)\n\n"） + # 使用最长的标题来计算（假设最多99个分块） + sample_title = f"## {title} (第99/99部分)\n\n" if title else "" + title_overhead = len(sample_title.encode('utf-8')) + + # 计算 mention_tags 的开销 + mention_overhead = 0 + if mentioned_list: + mention_tags = ' '.join([f'<@{user}>' for user in (mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])]) + mention_overhead = len(f"\n\n{mention_tags}".encode('utf-8')) + + # 实际可用的内容空间 + available_bytes = max_bytes - title_overhead - mention_overhead + else: + available_bytes = max_bytes + # text 类型也需要考虑 mention_tags + if mentioned_list: + mention_tags = ' '.join([f'<@{user}>' for user in (mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])]) + mention_overhead = len(f"\n\n{mention_tags}".encode('utf-8')) + available_bytes -= mention_overhead + + chunks = self._split_content(content, available_bytes) for i, chunk in enumerate(chunks): chunk_title = f"{title} (第{i + 1}/{len(chunks)}部分)" if title else f"消息 (第{i + 1}/{len(chunks)}部分)" - data = self._build_message(chunk, chunk_title, msg_type, is_at_all) + data = self._build_message(chunk, chunk_title, msg_type, is_at_all, mentioned_list) self._send_message(post_url, data, chunk_num=i + 1, total_chunks=len(chunks)) def _split_content(self, content, max_bytes): @@ -169,28 +230,50 @@ def _send_request(self, url, data): logger.error(f"企业微信返回的 JSON 解析失败! url:{url}, error: {e}") return None - def _build_message(self, content, title, msg_type, is_at_all): + def _build_message(self, content, title, msg_type, is_at_all, mentioned_list=None): """ 构造消息 """ if msg_type == 'text': - return self._build_text_message(content, is_at_all) + return self._build_text_message(content, is_at_all, mentioned_list) elif msg_type =='markdown': - return self._build_markdown_message(content, title) + return self._build_markdown_message(content, title, mentioned_list) else: raise ValueError(f"不支持的消息类型: {msg_type}") - def _build_text_message(self, content, is_at_all): + def _build_text_message(self, content, is_at_all, mentioned_list=None): """ 构造纯文本消息 """ + # 如果提供了明确的mentioned_list，使用它；否则根据is_at_all决定 + if mentioned_list is not None: + mentions = mentioned_list if isinstance(mentioned_list, list) else [mentioned_list] + else: + mentions = ["@all"] if is_at_all else [] + + # 如果有mentioned_list，在content末尾添加<@userid>语法 + if mentioned_list: + # 确保mentioned_list是列表，并去重 + users = list(set(mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])) + mention_tags = ' '.join([f'<@{user}>' for user in users]) + content = f"{content}\n\n{mention_tags}" + return { "msgtype": "text", "text": { "content": content, - "mentioned_list": ["@all"] if is_at_all else [] + # 不再传递mentioned_list，避免重复@人 + "mentioned_list": [] } } - def _build_markdown_message(self, content, title): + def _build_markdown_message(self, content, title, mentioned_list=None): """ 构造 Markdown 消息 """ formatted_content = self.format_markdown_content(content, title) + + # 如果有mentioned_list，在content末尾添加<@userid>语法 + if mentioned_list: + # 确保mentioned_list是列表，并去重 + users = list(set(mentioned_list if isinstance(mentioned_list, list) else [mentioned_list])) + mention_tags = ' '.join([f'<@{user}>' for user in users]) + formatted_content = f"{formatted_content}\n\n{mention_tags}" + return { "msgtype": "markdown", "markdown": { diff --git a/biz/utils/reporter.py b/biz/utils/reporter.py index ab9a56e89..0b1737323 100644 --- a/biz/utils/reporter.py +++ b/biz/utils/reporter.py @@ -1,9 +1,15 @@ +from typing import Dict, Optional + from biz.llm.factory import Factory class Reporter: - def __init__(self): - self.client = Factory().getClient() + def __init__(self, config: Optional[Dict[str, str]] = None): + """ + 初始化报告生成器 + :param config: 项目专属配置字典 + """ + self.client = Factory().getClient(config=config) def generate_report(self, data: str) -> str: # 根据data生成报告 diff --git a/biz/utils/test_config_isolation.py b/biz/utils/test_config_isolation.py new file mode 100644 index 000000000..bbf812250 --- /dev/null +++ b/biz/utils/test_config_isolation.py @@ -0,0 +1,125 @@ +""" +测试配置隔离功能 +验证多项目并发环境下配置不会互相污染 +""" +import os +import sys +from pathlib import Path + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +from biz.utils.config_loader import config_loader + + +def test_config_isolation(): + """测试配置隔离：不同项目的配置应该独立，不会互相影响""" + + print("=" * 60) + print("测试场景：模拟两个项目并发请求配置") + print("=" * 60) + + # 模拟项目A获取配置 + print("\n[项目A] 获取配置...") + config_a = config_loader.get_config(project_path="asset/project-a") + print(f"[项目A] 配置数量: {len(config_a)}") + + # 模拟项目B获取配置 + print("\n[项目B] 获取配置...") + config_b = config_loader.get_config(project_path="asset/project-b") + print(f"[项目B] 配置数量: {len(config_b)}") + + # 验证全局环境变量未被修改 + print("\n[验证] 检查全局环境变量...") + original_gitlab_token = os.environ.get('GITLAB_ACCESS_TOKEN', '') + print(f"[验证] 全局 GITLAB_ACCESS_TOKEN 保持不变: {bool(original_gitlab_token)}") + + # 验证配置是独立的字典对象 + print("\n[验证] 配置是否为独立对象...") + print(f"[验证] config_a is config_b: {config_a is config_b}") + print(f"[验证] id(config_a): {id(config_a)}") + print(f"[验证] id(config_b): {id(config_b)}") + + # 测试修改一个配置不影响另一个 + print("\n[测试] 修改config_a不应影响config_b...") + config_a['TEST_KEY'] = 'value_from_a' + has_test_key_in_b = 'TEST_KEY' in config_b + print(f"[测试] config_b中包含TEST_KEY: {has_test_key_in_b}") + print(f"[测试] ✓ 配置隔离成功！" if not has_test_key_in_b else "[测试] ✗ 配置污染！") + + print("\n" + "=" * 60) + print("✓ 配置隔离测试完成") + print("=" * 60) + + +def test_config_priority(): + """测试配置优先级：项目配置 > 默认配置""" + + print("\n" + "=" * 60) + print("测试场景：配置优先级") + print("=" * 60) + + # 获取默认配置 + print("\n[默认配置] 加载conf/.env...") + default_config = config_loader.get_config() + print(f"[默认配置] LLM_PROVIDER: {default_config.get('LLM_PROVIDER', 'NOT_SET')}") + + # 模拟项目有专属配置（如果存在的话） + print("\n[项目配置] 加载项目专属配置...") + project_config = config_loader.get_config(project_path="h5/h5-trade") + print(f"[项目配置] LLM_PROVIDER: {project_config.get('LLM_PROVIDER', 'NOT_SET')}") + + print("\n" + "=" * 60) + print("✓ 配置优先级测试完成") + print("=" * 60) + + +def test_concurrent_simulation(): + """模拟并发场景：快速切换多个项目配置""" + + print("\n" + "=" * 60) + print("测试场景：快速并发切换项目配置") + print("=" * 60) + + projects = [ + "asset/project-a", + "asset/project-b", + "h5/h5-trade", + "backend/api-server" + ] + + configs = {} + + print("\n[并发] 快速获取多个项目配置...") + for project in projects: + config = config_loader.get_config(project_path=project) + configs[project] = config + print(f"[并发] {project}: id={id(config)}, keys={len(config)}") + + # 验证所有配置都是独立对象 + print("\n[验证] 检查所有配置对象是否独立...") + ids = [id(config) for config in configs.values()] + unique_ids = set(ids) + + print(f"[验证] 总配置对象数: {len(ids)}") + print(f"[验证] 唯一对象数: {len(unique_ids)}") + print(f"[验证] ✓ 所有配置都是独立对象！" if len(ids) == len(unique_ids) else "[验证] ✗ 存在配置共享！") + + print("\n" + "=" * 60) + print("✓ 并发测试完成") + print("=" * 60) + + +if __name__ == '__main__': + print("\n" + "🧪" * 30) + print("配置隔离功能测试套件") + print("🧪" * 30) + + test_config_isolation() + test_config_priority() + test_concurrent_simulation() + + print("\n" + "✅" * 30) + print("所有测试完成！") + print("✅" * 30) diff --git a/biz/utils/test_config_loader.py b/biz/utils/test_config_loader.py new file mode 100644 index 000000000..f1a2c6ecc --- /dev/null +++ b/biz/utils/test_config_loader.py @@ -0,0 +1,141 @@ +import os +import unittest +from pathlib import Path +import tempfile +import shutil + +from biz.utils.config_loader import ConfigLoader, config_loader + + +class TestConfigLoader(unittest.TestCase): + """测试配置加载器""" + + def setUp(self): + """设置测试环境""" + # 创建临时测试目录 + self.test_dir = tempfile.mkdtemp() + self.original_conf_dir = ConfigLoader.DEFAULT_CONF_DIR + ConfigLoader.DEFAULT_CONF_DIR = self.test_dir # type: ignore + + # 创建默认配置文件 + self.default_env_content = "TEST_VAR=default_value\nLLM_PROVIDER=openai" + self.default_prompt_content = """code_review_prompt: + system_prompt: "Default system prompt" + user_prompt: "Default user prompt" +""" + + Path(self.test_dir).mkdir(exist_ok=True) + with open(os.path.join(self.test_dir, ".env"), "w") as f: + f.write(self.default_env_content) + with open(os.path.join(self.test_dir, "prompt_templates.yml"), "w") as f: + f.write(self.default_prompt_content) + + def tearDown(self): + """清理测试环境""" + ConfigLoader.DEFAULT_CONF_DIR = self.original_conf_dir # type: ignore + if os.path.exists(self.test_dir): + shutil.rmtree(self.test_dir) + + def test_singleton_pattern(self): + """测试单例模式""" + loader1 = ConfigLoader() + loader2 = ConfigLoader() + self.assertIs(loader1, loader2) + + def test_get_default_env_file_path(self): + """测试获取默认环境变量文件路径""" + loader = ConfigLoader() + env_path = loader.get_env_file_path() + expected_path = os.path.join(self.test_dir, ".env") + self.assertEqual(env_path, expected_path) + + def test_get_app_specific_env_file_path(self): + """测试获取应用专属环境变量文件路径""" + # 创建应用专属配置 + app_name = "test_app" + app_dir = os.path.join(self.test_dir, app_name) + os.makedirs(app_dir) + app_env_content = "TEST_VAR=app_specific_value" + with open(os.path.join(app_dir, ".env"), "w") as f: + f.write(app_env_content) + + loader = ConfigLoader() + loader.set_app_name(app_name) + env_path = loader.get_env_file_path() + expected_path = os.path.join(self.test_dir, app_name, ".env") + self.assertEqual(env_path, expected_path) + + def test_get_env_file_path_fallback_to_default(self): + """测试当应用专属配置不存在时，降级到默认配置""" + app_name = "non_existent_app" + loader = ConfigLoader() + loader.set_app_name(app_name) + env_path = loader.get_env_file_path() + expected_path = os.path.join(self.test_dir, ".env") + self.assertEqual(env_path, expected_path) + + def test_get_default_prompt_template_file_path(self): + """测试获取默认Prompt模板文件路径""" + loader = ConfigLoader() + template_path = loader.get_prompt_template_file_path() + expected_path = os.path.join(self.test_dir, "prompt_templates.yml") + self.assertEqual(template_path, expected_path) + + def test_get_app_specific_prompt_template_file_path(self): + """测试获取应用专属Prompt模板文件路径""" + # 创建应用专属配置 + app_name = "test_app" + app_dir = os.path.join(self.test_dir, app_name) + os.makedirs(app_dir) + app_template_content = """code_review_prompt: + system_prompt: "App specific system prompt" + user_prompt: "App specific user prompt" +""" + with open(os.path.join(app_dir, "prompt_templates.yml"), "w") as f: + f.write(app_template_content) + + loader = ConfigLoader() + loader.set_app_name(app_name) + template_path = loader.get_prompt_template_file_path() + expected_path = os.path.join(self.test_dir, app_name, "prompt_templates.yml") + self.assertEqual(template_path, expected_path) + + def test_load_prompt_template_default(self): + """测试加载默认Prompt模板""" + loader = ConfigLoader() + prompts = loader.load_prompt_template("code_review_prompt") + self.assertIn("system_prompt", prompts) + self.assertIn("user_prompt", prompts) + self.assertEqual(prompts["system_prompt"], "Default system prompt") + self.assertEqual(prompts["user_prompt"], "Default user prompt") + + def test_load_prompt_template_app_specific(self): + """测试加载应用专属Prompt模板""" + # 创建应用专属配置 + app_name = "test_app" + app_dir = os.path.join(self.test_dir, app_name) + os.makedirs(app_dir) + app_template_content = """code_review_prompt: + system_prompt: "App specific system prompt" + user_prompt: "App specific user prompt" +""" + with open(os.path.join(app_dir, "prompt_templates.yml"), "w", encoding="utf-8") as f: + f.write(app_template_content) + + loader = ConfigLoader() + prompts = loader.load_prompt_template("code_review_prompt", app_name) + self.assertEqual(prompts["system_prompt"], "App specific system prompt") + self.assertEqual(prompts["user_prompt"], "App specific user prompt") + + def test_create_app_config_dir(self): + """测试创建应用专属配置目录""" + app_name = "new_app" + app_dir = ConfigLoader.create_app_config_dir(app_name) + self.assertTrue(app_dir.exists()) + self.assertTrue(app_dir.is_dir()) + expected_path = Path(self.test_dir) / app_name + self.assertEqual(app_dir, expected_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/conf/.env.dist b/conf/.env.dist index 2f04f52f3..7a151ec86 100644 --- a/conf/.env.dist +++ b/conf/.env.dist @@ -4,7 +4,7 @@ SERVER_PORT=5001 #Timezone TZ=Asia/Shanghai -#大模型供应商配置,支持 deepseek, openai,zhipuai,qwen 和 ollama +#大模型供应商配置,支持 deepseek, openai,zhipuai,qwen, ollama 和 kimi LLM_PROVIDER=deepseek #DeepSeek settings @@ -31,6 +31,11 @@ QWEN_API_MODEL=qwen-coder-plus OLLAMA_API_BASE_URL=http://host.docker.internal:11434 OLLAMA_API_MODEL=deepseek-r1:latest +#Kimi settings (Moonshot AI) +KIMI_API_KEY=xxxx +KIMI_API_BASE_URL=https://api.moonshot.cn/v1 +KIMI_API_MODEL=kimi-k2-turbo-preview + #支持review的文件类型 SUPPORTED_EXTENSIONS=.c,.cc,.cpp,.cs,.css,.cxx,.go,.h,.hh,.hpp,.hxx,.java,.js,.jsx,.md,.php,.py,.sql,.ts,.tsx,.vue,.yml #每次 Review 的最大 Token 限制（超出部分自动截断） @@ -41,14 +46,22 @@ REVIEW_STYLE=professional #钉钉配置 DINGTALK_ENABLED=0 DINGTALK_WEBHOOK_URL=https://oapi.dingtalk.com/robot/send?access_token=xxx +# 钉钉日报专用webhook（可选，如果不配置则使用DINGTALK_WEBHOOK_URL） +# DINGTALK_WEBHOOK_URL_DAILY_REPORT=https://oapi.dingtalk.com/robot/send?access_token=daily-report-token #企业微信配置 WECOM_ENABLED=0 WECOM_WEBHOOK_URL=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx +# Push事件是否使用text消息类型（支持@人）：1=启用（会@commit者），0=使用markdown（默认） +PUSH_WECOM_USE_TEXT_MSG=1 +# 企业微信日报专用webhook（可选，如果不配置则使用WECOM_WEBHOOK_URL） +# WECOM_WEBHOOK_URL_DAILY_REPORT=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=daily-report-key #飞书配置 FEISHU_ENABLED=0 FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/xxx +# 飞书日报专用webhook（可选，如果不配置则使用FEISHU_WEBHOOK_URL） +# FEISHU_WEBHOOK_URL_DAILY_REPORT=https://open.feishu.cn/open-apis/bot/v2/hook/daily-report-hook #自定义webhook配置，使用场景：通过飞书发送应用消息可以实现Push评审通知到提交人，在自定义webhook里可以实现各种定制通知功能 #参数EXTRA_WEBHOOK_URL接收POST请求，data={ai_codereview_data: {}, webhook_data: {}}，ai_codereview_data为本系统通知的数据，webhook_data为原github、gitlab hook触发的数据 @@ -77,9 +90,22 @@ REPORT_CRONTAB_EXPRESSION=0 18 * * 1-5 # 开启Push Review功能(如果不需要push事件触发Code Review，设置为0) PUSH_REVIEW_ENABLED=1 +# 开启Push事件Commit Message检查(如果设置为1，则仅当commit message匹配指定规则时才触发Review) +PUSH_COMMIT_MESSAGE_CHECK_ENABLED=1 +# Push事件Commit Message检查规则(支持正则表达式)，示例：review 或 \[review\] 或 (review|codereview) +PUSH_COMMIT_MESSAGE_CHECK_PATTERN=review # 开启Merge请求过滤，过滤仅当合并目标分支是受保护分支时才Review(开启此选项请确保仓库已配置受保护分支protected branches) MERGE_REVIEW_ONLY_PROTECTED_BRANCHES_ENABLED=0 +# Review白名单功能（控制哪些命名空间或项目允许进行代码审查） +# 开启白名单功能（如果设置为1，则仅白名单中的命名空间/项目才会触发Review） +REVIEW_WHITELIST_ENABLED=0 +# 白名单配置（支持命名空间或完整项目路径，多个用逗号分隔） +# 示例1：仅配置命名空间 - asset,frontend （允许asset和frontend命名空间下的所有项目） +# 示例2：仅配置项目 - asset/asset-batch-center,frontend/web-app （仅允许指定的项目） +# 示例3：混合配置 - asset,frontend/web-app （允许asset命名空间下所有项目 + frontend/web-app项目） +REVIEW_WHITELIST= + # Dashboard登录用户名和密码 DASHBOARD_USER=admin DASHBOARD_PASSWORD=admin @@ -92,3 +118,15 @@ QUEUE_DRIVER=async # gitlab domain slugged WORKER_QUEUE=git_test_com + +# 数据库配置 +# 数据库类型：sqlite（默认）、mysql +DB_TYPE=sqlite +# SQLite数据库文件路径（仅当DB_TYPE=sqlite时使用） +DB_FILE=data/data.db +# MySQL数据库配置（仅当DB_TYPE=mysql时使用） +# DB_HOST=localhost +# DB_PORT=3306 +# DB_USER=root +# DB_PASSWORD=your_password +# DB_NAME=ai_codereview diff --git a/doc/mysql_schema.sql b/doc/mysql_schema.sql new file mode 100644 index 000000000..71a7ab5e8 --- /dev/null +++ b/doc/mysql_schema.sql @@ -0,0 +1,52 @@ +-- MySQL 数据库建表脚本 +-- 使用说明： +-- 1. 创建数据库：CREATE DATABASE ai_codereview CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +-- 2. 切换到数据库：USE ai_codereview; +-- 3. 执行本脚本创建表结构 + +-- 创建数据库（如果不存在） +CREATE DATABASE IF NOT EXISTS ai_codereview CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +USE ai_codereview; + +-- Merge Request 审核日志表 +CREATE TABLE IF NOT EXISTS mr_review_log ( + id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID', + project_name VARCHAR(255) COMMENT '项目名称', + author VARCHAR(255) COMMENT '作者', + source_branch VARCHAR(255) COMMENT '源分支', + target_branch VARCHAR(255) COMMENT '目标分支', + updated_at BIGINT COMMENT '更新时间戳', + commit_messages TEXT COMMENT '提交信息', + score INT COMMENT 'Review评分', + url VARCHAR(500) COMMENT 'MR链接', + review_result LONGTEXT COMMENT 'Review结果', + additions INT DEFAULT 0 COMMENT '新增代码行数', + deletions INT DEFAULT 0 COMMENT '删除代码行数', + last_commit_id VARCHAR(255) DEFAULT '' COMMENT '最后一次commit ID', + INDEX idx_updated_at (updated_at), + INDEX idx_project_name (project_name), + INDEX idx_author (author) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Merge Request审核日志表'; + +-- Push 审核日志表 +CREATE TABLE IF NOT EXISTS push_review_log ( + id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID', + project_name VARCHAR(255) COMMENT '项目名称', + author VARCHAR(255) COMMENT '作者', + branch VARCHAR(255) COMMENT '分支', + updated_at BIGINT COMMENT '更新时间戳', + commit_messages TEXT COMMENT '提交信息', + score INT COMMENT 'Review评分', + review_result LONGTEXT COMMENT 'Review结果', + additions INT DEFAULT 0 COMMENT '新增代码行数', + deletions INT DEFAULT 0 COMMENT '删除代码行数', + INDEX idx_updated_at (updated_at), + INDEX idx_project_name (project_name), + INDEX idx_author (author) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Push审核日志表'; + +-- 查看表结构 +SHOW TABLES; +DESC mr_review_log; +DESC push_review_log; diff --git a/docker-compose.yml b/docker-compose.yml index 2bba4b999..6b37e9150 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,9 @@ services: volumes: - ./data:/app/data - ./log:/app/log + - ./biz:/app/biz + - ./conf:/app/conf + - ./api.py:/app/api.py env_file: - ./conf/.env restart: unless-stopped \ No newline at end of file diff --git a/test_scheduler.py b/test_scheduler.py new file mode 100644 index 000000000..bf1d4b932 --- /dev/null +++ b/test_scheduler.py @@ -0,0 +1,59 @@ +""" +测试调度器配置的脚本 +""" +import os +from datetime import datetime +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger +from dotenv import load_dotenv + +load_dotenv("conf/.env") + +def test_job(): + print(f"[{datetime.now()}] Test job executed!") + +def test_scheduler(): + """测试调度器配置""" + crontab_expression = os.getenv('REPORT_CRONTAB_EXPRESSION', '0 18 * * 1-5') + print(f"Cron expression: {crontab_expression}") + + cron_parts = crontab_expression.split() + cron_minute, cron_hour, cron_day, cron_month, cron_day_of_week = cron_parts + + print(f"Parsed cron components:") + print(f" - minute: {cron_minute}") + print(f" - hour: {cron_hour}") + print(f" - day: {cron_day}") + print(f" - month: {cron_month}") + print(f" - day_of_week: {cron_day_of_week}") + + scheduler = BackgroundScheduler() + + job = scheduler.add_job( + test_job, + trigger=CronTrigger( + minute=cron_minute, + hour=cron_hour, + day=cron_day, + month=cron_month, + day_of_week=cron_day_of_week + ), + id='test_job' + ) + + scheduler.start() + + print(f"\nScheduler started!") + print(f"Next run time: {job.next_run_time}") + print(f"Current time: {datetime.now()}") + + # 列出所有已调度的任务 + print("\nScheduled jobs:") + for job in scheduler.get_jobs(): + print(f" - {job.id}: next run at {job.next_run_time}") + + scheduler.shutdown() + print("\nScheduler test completed!") + +if __name__ == '__main__': + test_scheduler()