feat: tools/gitlab (#7329)

Co-authored-by: crazywoola <427733928@qq.com>

api/core/tools/provider/builtin/gitlab/gitlab.py

@@ -29,6 +29,6 @@ class GitlabProvider(BuiltinToolProviderController):
                 if response.status_code != 200:
                     raise ToolProviderCredentialValidationError((response.json()).get('message'))
             except Exception as e:
-                raise ToolProviderCredentialValidationError("Gitlab Access Tokens and Api Version is invalid. {}".format(e))
+                raise ToolProviderCredentialValidationError("Gitlab Access Tokens is invalid. {}".format(e))
         except Exception as e:
             raise ToolProviderCredentialValidationError(str(e))

api/core/tools/provider/builtin/gitlab/gitlab.yaml

@@ -2,37 +2,37 @@ identity:
   author: Leo.Wang
   name: gitlab
   label:
-    en_US: Gitlab
-    zh_Hans: Gitlab
+    en_US: GitLab
+    zh_Hans: GitLab
   description:
-    en_US: Gitlab plugin for commit
-    zh_Hans: 用于获取Gitlab commit的插件
+    en_US: GitLab plugin, API v4 only.
+    zh_Hans: 用于获取GitLab内容的插件，目前仅支持 API v4。
   icon: gitlab.svg
 credentials_for_provider:
   access_tokens:
     type: secret-input
     required: true
     label:
-      en_US: Gitlab access token
-      zh_Hans: Gitlab access token
+      en_US: GitLab access token
+      zh_Hans: GitLab access token
     placeholder:
-      en_US: Please input your Gitlab access token
-      zh_Hans: 请输入你的 Gitlab access token
+      en_US: Please input your GitLab access token
+      zh_Hans: 请输入你的 GitLab access token
     help:
-      en_US: Get your Gitlab access token from Gitlab
-      zh_Hans: 从 Gitlab 获取您的 access token
+      en_US: Get your GitLab access token from GitLab
+      zh_Hans: 从 GitLab 获取您的 access token
     url: https://docs.gitlab.com/16.9/ee/api/oauth2.html
   site_url:
     type: text-input
     required: false
     default: 'https://gitlab.com'
     label:
-      en_US: Gitlab site url
-      zh_Hans: Gitlab site url
+      en_US: GitLab site url
+      zh_Hans: GitLab site url
     placeholder:
-      en_US: Please input your Gitlab site url
-      zh_Hans: 请输入你的 Gitlab site url
+      en_US: Please input your GitLab site url
+      zh_Hans: 请输入你的 GitLab site url
     help:
-      en_US: Find your Gitlab url
-      zh_Hans: 找到你的Gitlab url
+      en_US: Find your GitLab url
+      zh_Hans: 找到你的 GitLab url
     url: https://gitlab.com/help

api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.py

@@ -18,6 +18,7 @@ class GitlabCommitsTool(BuiltinTool):
         employee = tool_parameters.get('employee', '')
         start_time = tool_parameters.get('start_time', '')
         end_time = tool_parameters.get('end_time', '')
+        change_type = tool_parameters.get('change_type', 'all')
 
         if not project:
             return self.create_text_message('Project is required')
@@ -36,11 +37,11 @@ class GitlabCommitsTool(BuiltinTool):
             site_url = 'https://gitlab.com'
         
         # Get commit content
-        result = self.fetch(user_id, site_url, access_token, project, employee, start_time, end_time)
+        result = self.fetch(user_id, site_url, access_token, project, employee, start_time, end_time, change_type)
 
-        return self.create_text_message(json.dumps(result, ensure_ascii=False))
+        return [self.create_json_message(item) for item in result]
     
-    def fetch(self,user_id: str, site_url: str, access_token: str, project: str, employee: str = None, start_time: str = '', end_time: str = '') -> list[dict[str, Any]]:
+    def fetch(self,user_id: str, site_url: str, access_token: str, project: str, employee: str = None, start_time: str = '', end_time: str = '', change_type: str = '') -> list[dict[str, Any]]:
         domain = site_url
         headers = {"PRIVATE-TOKEN": access_token}
         results = []
@@ -74,7 +75,7 @@ class GitlabCommitsTool(BuiltinTool):
 
                 for commit in commits:
                     commit_sha = commit['id']
-                    print(f"\tCommit SHA: {commit_sha}")
+                    author_name = commit['author_name']
 
                     diff_url = f"{domain}/api/v4/projects/{project_id}/repository/commits/{commit_sha}/diff"
                     diff_response = requests.get(diff_url, headers=headers)
@@ -87,14 +88,23 @@ class GitlabCommitsTool(BuiltinTool):
                         removed_lines = diff['diff'].count('\n-')
                         total_changes = added_lines + removed_lines
 
-                        if total_changes > 1:
+                        if change_type == "new":
+                            if added_lines > 1:
                                 final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if line.startswith('+') and not line.startswith('+++')])
                                 results.append({
-                                "project": project_name,
                                     "commit_sha": commit_sha,
+                                    "author_name": author_name,
                                     "diff": final_code
                                 })
-                            print(f"Commit code:{final_code}")
+                        else:
+                            if total_changes > 1:
+                                final_code = ''.join([line[1:] for line in diff['diff'].split('\n') if (line.startswith('+') or line.startswith('-')) and not line.startswith('+++') and not line.startswith('---')])
+                                final_code_escaped = json.dumps(final_code)[1:-1]  # Escape the final code
+                                results.append({
+                                    "commit_sha": commit_sha,
+                                    "author_name": author_name,
+                                    "diff": final_code_escaped
+                                })
         except requests.RequestException as e:
             print(f"Error fetching data from GitLab: {e}")
         

api/core/tools/provider/builtin/gitlab/tools/gitlab_commits.yaml

@@ -2,24 +2,24 @@ identity:
   name: gitlab_commits
   author: Leo.Wang
   label:
-    en_US: Gitlab Commits
-    zh_Hans: Gitlab代码提交内容
+    en_US: GitLab Commits
+    zh_Hans: GitLab 提交内容查询
 description:
   human:
-    en_US: A tool for query gitlab commits. Input should be a exists username.
-    zh_Hans: 一个用于查询gitlab代码提交记录的的工具，输入的内容应该是一个已存在的用户名或者项目名。
-  llm: A tool for query gitlab commits. Input should be a exists username or project.
+    en_US: A tool for query GitLab commits, Input should be a exists username or projec.
+    zh_Hans: 一个用于查询 GitLab 代码提交内容的工具，输入的内容应该是一个已存在的用户名或者项目名。
+  llm: A tool for query GitLab commits, Input should be a exists username or project.
 parameters:
-  - name: employee
+  - name: username
     type: string
     required: false
     label:
-      en_US: employee
+      en_US: username
       zh_Hans: 员工用户名
     human_description:
-      en_US: employee
+      en_US: username
       zh_Hans: 员工用户名
-    llm_description: employee for gitlab
+    llm_description: User name for GitLab
     form: llm
   - name: project
     type: string
@@ -30,7 +30,7 @@ parameters:
     human_description:
       en_US: project
       zh_Hans: 项目名
-    llm_description: project for gitlab
+    llm_description: project for GitLab
     form: llm
   - name: start_time
     type: string
@@ -41,7 +41,7 @@ parameters:
     human_description:
       en_US: start_time
       zh_Hans: 开始时间
-    llm_description: start_time for gitlab
+    llm_description: Start time for GitLab
     form: llm
   - name: end_time
     type: string
@@ -52,5 +52,26 @@ parameters:
     human_description:
       en_US: end_time
       zh_Hans: 结束时间
-    llm_description: end_time for gitlab
+    llm_description: End time for GitLab
+    form: llm
+  - name: change_type
+    type: select
+    required: false
+    options:
+      - value: all
+        label:
+          en_US: all
+          zh_Hans: 所有
+      - value: new
+        label:
+          en_US: new
+          zh_Hans: 新增
+    default: all
+    label:
+      en_US: change_type
+      zh_Hans: 变更类型
+    human_description:
+      en_US: change_type
+      zh_Hans: 变更类型
+    llm_description: Content change type for GitLab
     form: llm

api/core/tools/provider/builtin/gitlab/tools/gitlab_files.py

@@ -0,0 +1,95 @@
+from typing import Any, Union
+
+import requests
+
+from core.tools.entities.tool_entities import ToolInvokeMessage
+from core.tools.tool.builtin_tool import BuiltinTool
+
+
+class GitlabFilesTool(BuiltinTool):
+    def _invoke(self, 
+                user_id: str,
+                tool_parameters: dict[str, Any]
+        ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]:
+        
+        project = tool_parameters.get('project', '')
+        branch = tool_parameters.get('branch', '')
+        path = tool_parameters.get('path', '')
+
+
+        if not project:
+            return self.create_text_message('Project is required')
+        if not branch:
+            return self.create_text_message('Branch is required')
+
+        if not path:
+            return self.create_text_message('Path is required')
+
+        access_token = self.runtime.credentials.get('access_tokens')
+        site_url = self.runtime.credentials.get('site_url')
+
+        if 'access_tokens' not in self.runtime.credentials or not self.runtime.credentials.get('access_tokens'):
+            return self.create_text_message("Gitlab API Access Tokens is required.")
+        if 'site_url' not in self.runtime.credentials or not self.runtime.credentials.get('site_url'):
+            site_url = 'https://gitlab.com'
+    
+        # Get project ID from project name
+        project_id = self.get_project_id(site_url, access_token, project)
+        if not project_id:
+            return self.create_text_message(f"Project '{project}' not found.")
+
+        # Get commit content
+        result = self.fetch(user_id, project_id, site_url, access_token, branch, path)
+
+        return [self.create_json_message(item) for item in result]
+    
+    def extract_project_name_and_path(self, path: str) -> tuple[str, str]:
+        parts = path.split('/', 1)
+        if len(parts) < 2:
+            return None, None
+        return parts[0], parts[1]
+
+    def get_project_id(self, site_url: str, access_token: str, project_name: str) -> Union[str, None]:
+        headers = {"PRIVATE-TOKEN": access_token}
+        try:
+            url = f"{site_url}/api/v4/projects?search={project_name}"
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            projects = response.json()
+            for project in projects:
+                if project['name'] == project_name:
+                    return project['id']
+        except requests.RequestException as e:
+            print(f"Error fetching project ID from GitLab: {e}")
+        return None
+    
+    def fetch(self,user_id: str, project_id: str, site_url: str, access_token: str, branch: str, path: str = None) -> list[dict[str, Any]]:
+        domain = site_url
+        headers = {"PRIVATE-TOKEN": access_token}
+        results = []
+
+        try:
+            # List files and directories in the given path
+            url = f"{domain}/api/v4/projects/{project_id}/repository/tree?path={path}&ref={branch}"
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            items = response.json()
+
+            for item in items:
+                item_path = item['path']
+                if item['type'] == 'tree':  # It's a directory
+                    results.extend(self.fetch(project_id, site_url, access_token, branch, item_path))
+                else:  # It's a file
+                    file_url = f"{domain}/api/v4/projects/{project_id}/repository/files/{item_path}/raw?ref={branch}"
+                    file_response = requests.get(file_url, headers=headers)
+                    file_response.raise_for_status()
+                    file_content = file_response.text
+                    results.append({
+                        "path": item_path,
+                        "branch": branch,
+                        "content": file_content
+                    })
+        except requests.RequestException as e:
+            print(f"Error fetching data from GitLab: {e}")
+        
+        return results

api/core/tools/provider/builtin/gitlab/tools/gitlab_files.yaml

@@ -0,0 +1,45 @@
+identity:
+  name: gitlab_files
+  author: Leo.Wang
+  label:
+    en_US: GitLab Files
+    zh_Hans: GitLab 文件获取
+description:
+  human:
+    en_US: A tool for query GitLab files, Input should be branch and a exists file or directory path.
+    zh_Hans: 一个用于查询 GitLab 文件的工具，输入的内容应该是分支和一个已存在文件或者文件夹路径。
+  llm: A tool for query GitLab files, Input should be a exists file or directory path.
+parameters:
+  - name: project
+    type: string
+    required: true
+    label:
+      en_US: project
+      zh_Hans: 项目
+    human_description:
+      en_US: project
+      zh_Hans: 项目
+    llm_description: Project for GitLab
+    form: llm
+  - name: branch
+    type: string
+    required: true
+    label:
+      en_US: branch
+      zh_Hans: 分支
+    human_description:
+      en_US: branch
+      zh_Hans: 分支
+    llm_description: Branch for GitLab
+    form: llm
+  - name: path
+    type: string
+    required: true
+    label:
+      en_US: path
+      zh_Hans: 文件路径
+    human_description:
+      en_US: path
+      zh_Hans: 文件路径
+    llm_description: File path for GitLab
+    form: llm