support huawei cloud obs storage (#7980) (#7981)

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -73,6 +73,12 @@ TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
 
+# Huawei OBS Storage Configuration
+HUAWEI_OBS_BUCKET_NAME=your-bucket-name
+HUAWEI_OBS_SECRET_KEY=your-secret-key
+HUAWEI_OBS_ACCESS_KEY=your-access-key
+HUAWEI_OBS_SERVER=your-server-url
+
 # OCI Storage configuration
 OCI_ENDPOINT=your-endpoint
 OCI_BUCKET_NAME=your-bucket-name

api/configs/middleware/__init__.py

@@ -9,6 +9,7 @@ from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorag
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from configs.middleware.storage.oci_storage_config import OCIStorageConfig
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
@@ -184,6 +185,7 @@ class MiddlewareConfig(
     AzureBlobStorageConfig,
     GoogleCloudStorageConfig,
     TencentCloudCOSStorageConfig,
+    HuaweiCloudOBSStorageConfig,
     S3StorageConfig,
     OCIStorageConfig,
     # configs of vdb and vdb providers

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class HuaweiCloudOBSStorageConfig(BaseModel):
+    """
+    Huawei Cloud OBS storage configs
+    """
+
+    HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
+        description="Huawei Cloud OBS bucket name",
+        default=None,
+    )
+
+    HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
+        description="Huawei Cloud OBS Access key",
+        default=None,
+    )
+
+    HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
+        description="Huawei Cloud OBS Secret key",
+        default=None,
+    )
+
+    HUAWEI_OBS_SERVER: Optional[str] = Field(
+        description="Huawei Cloud OBS server URL",
+        default=None,
+    )

api/extensions/ext_storage.py

@@ -6,6 +6,7 @@ from flask import Flask
 from extensions.storage.aliyun_storage import AliyunStorage
 from extensions.storage.azure_storage import AzureStorage
 from extensions.storage.google_storage import GoogleStorage
+from extensions.storage.huawei_storage import HuaweiStorage
 from extensions.storage.local_storage import LocalStorage
 from extensions.storage.oci_storage import OCIStorage
 from extensions.storage.s3_storage import S3Storage
@@ -30,6 +31,8 @@ class Storage:
             self.storage_runner = TencentStorage(app=app)
         elif storage_type == "oci-storage":
             self.storage_runner = OCIStorage(app=app)
+        elif storage_type == "huawei-obs":
+            self.storage_runner = HuaweiStorage(app=app)
         else:
             self.storage_runner = LocalStorage(app=app)
 

api/extensions/storage/huawei_storage.py

@@ -0,0 +1,53 @@
+from collections.abc import Generator
+
+from flask import Flask
+from obs import ObsClient
+
+from extensions.storage.base_storage import BaseStorage
+
+
+class HuaweiStorage(BaseStorage):
+    """Implementation for huawei obs storage."""
+
+    def __init__(self, app: Flask):
+        super().__init__(app)
+        app_config = self.app.config
+        self.bucket_name = app_config.get("HUAWEI_OBS_BUCKET_NAME")
+        self.client = ObsClient(
+            access_key_id=app_config.get("HUAWEI_OBS_ACCESS_KEY"),
+            secret_access_key=app_config.get("HUAWEI_OBS_SECRET_KEY"),
+            server=app_config.get("HUAWEI_OBS_SERVER"),
+        )
+
+    def save(self, filename, data):
+        self.client.putObject(bucketName=self.bucket_name, objectKey=filename, content=data)
+
+    def load_once(self, filename: str) -> bytes:
+        data = self.client.getObject(bucketName=self.bucket_name, objectKey=filename)["body"].response.read()
+        return data
+
+    def load_stream(self, filename: str) -> Generator:
+        def generate(filename: str = filename) -> Generator:
+            response = self.client.getObject(bucketName=self.bucket_name, objectKey=filename)["body"].response
+            yield from response.read(4096)
+
+        return generate()
+
+    def download(self, filename, target_filepath):
+        self.client.getObject(bucketName=self.bucket_name, objectKey=filename, downloadPath=target_filepath)
+
+    def exists(self, filename):
+        res = self._get_meta(filename)
+        if res is None:
+            return False
+        return True
+
+    def delete(self, filename):
+        self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)
+
+    def _get_meta(self, filename):
+        res = self.client.getObjectMetadata(bucketName=self.bucket_name, objectKey=filename)
+        if res.status < 300:
+            return res
+        else:
+            return None

api/pyproject.toml

@@ -119,6 +119,7 @@ celery = "~5.3.6"
 chardet = "~5.1.0"
 cohere = "~5.2.4"
 cos-python-sdk-v5 = "1.9.30"
+esdk-obs-python = "3.24.6.1"
 dashscope = { version = "~1.17.0", extras = ["tokenizer"] }
 flask = "~3.0.1"
 flask-compress = "~1.14"

docker/.env.example

@@ -242,7 +242,7 @@ CONSOLE_CORS_ALLOW_ORIGINS=*
 # ------------------------------
 
 # The type of storage to use for storing user files.
-# Supported values are `local` and `s3` and `azure-blob` and `google-storage` and `tencent-cos`,
+# Supported values are `local` and `s3` and `azure-blob` and `google-storage` and `tencent-cos` and `huawei-obs`
 # Default: `local`
 STORAGE_TYPE=local
 
@@ -300,6 +300,16 @@ TENCENT_COS_REGION=your-region
 # The scheme of the Tencent COS service.
 TENCENT_COS_SCHEME=your-scheme
 
+# Huawei OBS Configuration
+# The name of the Huawei OBS bucket to use for storing files.
+HUAWEI_OBS_BUCKET_NAME=your-bucket-name
+# The secret key to use for authenticating with the Huawei OBS service.
+HUAWEI_OBS_SECRET_KEY=your-secret-key
+# The access key to use for authenticating with the Huawei OBS service.
+HUAWEI_OBS_ACCESS_KEY=your-access-key
+# The server url of the HUAWEI OBS service.
+HUAWEI_OBS_SERVER=your-server-url
+
 # ------------------------------
 # Vector Database Configuration
 # ------------------------------

docker/docker-compose.yaml

@@ -72,6 +72,10 @@ x-shared-env: &shared-api-worker-env
   TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-}
   TENCENT_COS_REGION: ${TENCENT_COS_REGION:-}
   TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-}
+  HUAWEI_OBS_BUCKET_NAME: ${HUAWEI_OBS_BUCKET_NAME:-}
+  HUAWEI_OBS_SECRET_KEY: ${HUAWEI_OBS_SECRET_KEY:-}
+  HUAWEI_OBS_ACCESS_KEY: ${HUAWEI_OBS_ACCESS_KEY:-}
+  HUAWEI_OBS_SERVER: ${HUAWEI_OBS_SERVER:-}
   OCI_ENDPOINT: ${OCI_ENDPOINT:-}
   OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-}
   OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-}