From 986fd5bfc6f2671f9a7dbe7013e44a885d2b0767 Mon Sep 17 00:00:00 2001 From: "Leo.Wang" Date: Tue, 13 Aug 2024 17:36:45 +0800 Subject: [PATCH] Add gitlab support (#7179) Co-authored-by: crazywoola <427733928@qq.com> --- .../builtin/gitlab/_assets/gitlab.svg | 2 + .../tools/provider/builtin/gitlab/gitlab.py | 34 ++++++ .../tools/provider/builtin/gitlab/gitlab.yaml | 38 +++++++ .../builtin/gitlab/tools/gitlab_commits.py | 101 ++++++++++++++++++ .../builtin/gitlab/tools/gitlab_commits.yaml | 56 ++++++++++ 5 files changed, 231 insertions(+) create mode 100644 api/core/tools/provider/builtin/gitlab/_assets/gitlab.svg create mode 100644 api/core/tools/provider/builtin/gitlab/gitlab.py create mode 100644 api/core/tools/provider/builtin/gitlab/gitlab.yaml create mode 100644 api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py create mode 100644 api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml diff --git a/api/core/tools/provider/builtin/gitlab/_assets/gitlab.svg b/api/core/tools/provider/builtin/gitlab/_assets/gitlab.svg new file mode 100644 index 0000000000..07734077d5 --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/_assets/gitlab.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/gitlab/gitlab.py b/api/core/tools/provider/builtin/gitlab/gitlab.py new file mode 100644 index 0000000000..fca34ae15f --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/gitlab.py @@ -0,0 +1,34 @@ +from typing import Any + +import requests + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class GitlabProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + if 'access_tokens' not in credentials or not credentials.get('access_tokens'): + raise ToolProviderCredentialValidationError("Gitlab Access Tokens is required.") + + if 'site_url' not in credentials or not credentials.get('site_url'): + site_url = 'https://gitlab.com' + else: + site_url = credentials.get('site_url') + + try: + headers = { + "Content-Type": "application/vnd.text+json", + "Authorization": f"Bearer {credentials.get('access_tokens')}", + } + + response = requests.get( + url= f"{site_url}/api/v4/user", + headers=headers) + if response.status_code != 200: + raise ToolProviderCredentialValidationError((response.json()).get('message')) + except Exception as e: + raise ToolProviderCredentialValidationError("Gitlab Access Tokens and Api Version is invalid. {}".format(e)) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/gitlab/gitlab.yaml b/api/core/tools/provider/builtin/gitlab/gitlab.yaml new file mode 100644 index 0000000000..b5feea2382 --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/gitlab.yaml @@ -0,0 +1,38 @@ +identity: + author: Leo.Wang + name: gitlab + label: + en_US: Gitlab + zh_Hans: Gitlab + description: + en_US: Gitlab plugin for commit + zh_Hans: 用于获取Gitlab commit的插件 + icon: gitlab.svg +credentials_for_provider: + access_tokens: + type: secret-input + required: true + label: + en_US: Gitlab access token + zh_Hans: Gitlab access token + placeholder: + en_US: Please input your Gitlab access token + zh_Hans: 请输入你的 Gitlab access token + help: + en_US: Get your Gitlab access token from Gitlab + zh_Hans: 从 Gitlab 获取您的 access token + url: https://docs.gitlab.com/16.9/ee/api/oauth2.html + site_url: + type: text-input + required: false + default: 'https://gitlab.com' + label: + en_US: Gitlab site url + zh_Hans: Gitlab site url + placeholder: + en_US: Please input your Gitlab site url + zh_Hans: 请输入你的 Gitlab site url + help: + en_US: Find your Gitlab url + zh_Hans: 找到你的Gitlab url + url: https://gitlab.com/help diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py new file mode 100644 index 0000000000..212bdb03ab --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py @@ -0,0 +1,101 @@ +import json +from datetime import datetime, timedelta +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GitlabCommitsTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + project = tool_parameters.get('project', '') + employee = tool_parameters.get('employee', '') + start_time = tool_parameters.get('start_time', '') + end_time = tool_parameters.get('end_time', '') + + if not project: + return self.create_text_message('Project is required') + + if not start_time: + start_time = (datetime.utcnow() - timedelta(days=1)).isoformat() + if not end_time: + end_time = datetime.utcnow().isoformat() + + access_token = self.runtime.credentials.get('access_tokens') + site_url = self.runtime.credentials.get('site_url') + + if 'access_tokens' not in self.runtime.credentials or not self.runtime.credentials.get('access_tokens'): + return self.create_text_message("Gitlab API Access Tokens is required.") + if 'site_url' not in self.runtime.credentials or not self.runtime.credentials.get('site_url'): + site_url = 'https://gitlab.com' + + # Get commit content + result = self.fetch(user_id, site_url, access_token, project, employee, start_time, end_time) + + return self.create_text_message(json.dumps(result, ensure_ascii=False)) + + def fetch(self,user_id: str, site_url: str, access_token: str, project: str, employee: str = None, start_time: str = '', end_time: str = '') -> list[dict[str, Any]]: + domain = site_url + headers = {"PRIVATE-TOKEN": access_token} + results = [] + + try: + # Get all of projects + url = f"{domain}/api/v4/projects" + response = requests.get(url, headers=headers) + response.raise_for_status() + projects = response.json() + + filtered_projects = [p for p in projects if project == "*" or p['name'] == project] + + for project in filtered_projects: + project_id = project['id'] + project_name = project['name'] + print(f"Project: {project_name}") + + # Get all of proejct commits + commits_url = f"{domain}/api/v4/projects/{project_id}/repository/commits" + params = { + 'since': start_time, + 'until': end_time + } + if employee: + params['author'] = employee + + commits_response = requests.get(commits_url, headers=headers, params=params) + commits_response.raise_for_status() + commits = commits_response.json() + + for commit in commits: + commit_sha = commit['id'] + print(f"\tCommit SHA: {commit_sha}") + + diff_url = f"{domain}/api/v4/projects/{project_id}/repository/commits/{commit_sha}/diff" + diff_response = requests.get(diff_url, headers=headers) + diff_response.raise_for_status() + diffs = diff_response.json() + + for diff in diffs: + # Caculate code lines of changed + added_lines = diff['diff'].count('\n+') + removed_lines = diff['diff'].count('\n-') + total_changes = added_lines + removed_lines + + if total_changes > 1: + final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if line.startswith('+') and not line.startswith('+++')]) + results.append({ + "project": project_name, + "commit_sha": commit_sha, + "diff": final_code + }) + print(f"Commit code:{final_code}") + except requests.RequestException as e: + print(f"Error fetching data from GitLab: {e}") + + return results \ No newline at end of file diff --git a/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml new file mode 100644 index 0000000000..fc4e7eb7bb --- /dev/null +++ b/api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml @@ -0,0 +1,56 @@ +identity: + name: gitlab_commits + author: Leo.Wang + label: + en_US: Gitlab Commits + zh_Hans: Gitlab代码提交内容 +description: + human: + en_US: A tool for query gitlab commits. Input should be a exists username. + zh_Hans: 一个用于查询gitlab代码提交记录的的工具，输入的内容应该是一个已存在的用户名或者项目名。 + llm: A tool for query gitlab commits. Input should be a exists username or project. +parameters: + - name: employee + type: string + required: false + label: + en_US: employee + zh_Hans: 员工用户名 + human_description: + en_US: employee + zh_Hans: 员工用户名 + llm_description: employee for gitlab + form: llm + - name: project + type: string + required: true + label: + en_US: project + zh_Hans: 项目名 + human_description: + en_US: project + zh_Hans: 项目名 + llm_description: project for gitlab + form: llm + - name: start_time + type: string + required: false + label: + en_US: start_time + zh_Hans: 开始时间 + human_description: + en_US: start_time + zh_Hans: 开始时间 + llm_description: start_time for gitlab + form: llm + - name: end_time + type: string + required: false + label: + en_US: end_time + zh_Hans: 结束时间 + human_description: + en_US: end_time + zh_Hans: 结束时间 + llm_description: end_time for gitlab + form: llm