//! 腾讯云 COS 存储驱动。 //! //! 基础对象读写复用 S3 兼容驱动;COS/CI 数据处理使用 COS 原生 query //! 签名,因为 CI 处理参数必须参与签名,不能追加在普通 S3 presigned URL 后面。 pub(crate) mod cors; mod native_media_metadata; mod native_thumbnail; mod signing; #[cfg(test)] mod tests; use std::{sync::Arc, time::Duration}; use url::Url; use super::s3::{S3Driver, S3DriverConfig, S3DriverOptions, S3StaticCredentials}; use super::s3_compatible::S3CompatibleDriver; use super::s3_config::{S3ConfigError, normalize_s3_endpoint_and_bucket}; use crate::config::OUTBOUND_HTTP_USER_AGENT; use aster_drive_storage::error::{StorageErrorKind, storage_driver_error}; use aster_drive_storage::object_key; use aster_drive_storage::traits::driver::{DirectDownloadOptions, DirectDownloadRequest}; use aster_drive_storage::traits::extensions::{ DirectDownloadStorageDriver, PresignedUploadStorageDriver, }; use aster_drive_storage::{ MapStorageErr, MultipartStorageCapabilities, MultipartUploadMode, Result, }; pub(super) const COS_NATIVE_PROCESSING_PROVIDER: &str = "tencent_cos_ci"; pub(super) const MAX_COS_THUMBNAIL_TTL: Duration = Duration::from_secs(5 * 60); fn non_empty_xml_text(text: Option<&str>) -> Option { let trimmed = text?.trim(); (!trimmed.is_empty()).then(|| trimmed.to_string()) } pub struct TencentCosDriver { storage: S3CompatibleDriver, client: reqwest::Client, endpoint: String, bucket: String, access_key: String, secret_key: String, base_path: String, use_cname: bool, download_base_url: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct TencentCosDriverConfig { pub endpoint: String, pub bucket: String, pub base_path: String, pub use_cname: bool, pub download_base_url: Option, pub connect_timeout: Duration, pub read_timeout: Duration, pub operation_timeout: Duration, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct TencentCosStaticCredentials { pub access_key: String, pub secret_key: String, } impl TencentCosDriver { pub fn validate_connection_config(config: &TencentCosDriverConfig) -> Result<()> { let normalized = normalize_s3_endpoint_and_bucket(&config.endpoint, &config.bucket) .map_err(Self::rewrap_s3_config_error)?; if normalized.endpoint.trim().is_empty() { return Err(storage_driver_error( StorageErrorKind::Misconfigured, "COS endpoint is required", )); } let endpoint = Url::parse(&normalized.endpoint) .map_storage_err_ctx(StorageErrorKind::Misconfigured, "parse COS endpoint")?; let host = endpoint.host_str().ok_or_else(|| { storage_driver_error(StorageErrorKind::Misconfigured, "COS endpoint missing host") })?; if !endpoint.username().is_empty() || endpoint.password().is_some() || endpoint.query().is_some() || endpoint.fragment().is_some() || config.endpoint.contains('\\') || config.endpoint.chars().any(char::is_control) { return Err(storage_driver_error( StorageErrorKind::Misconfigured, "COS endpoint must not contain credentials, query, fragment, backslashes, or control characters", )); } if config.use_cname && endpoint.path() != "/" && !endpoint.path().is_empty() { return Err(storage_driver_error( StorageErrorKind::Misconfigured, "COS CNAME endpoint must not contain a path; use base_path for object key prefixes", )); } if !config.use_cname && !host.ends_with(".myqcloud.com") { return Err(storage_driver_error( StorageErrorKind::Misconfigured, "COS endpoint must use a Tencent COS myqcloud.com host unless CNAME mode is enabled", )); } if let Some(download_base_url) = config.download_base_url.as_deref() { let url = Url::parse(download_base_url).map_storage_err_ctx( StorageErrorKind::Misconfigured, "parse COS download base URL", )?; if url.scheme() != "https" || url.host_str().is_none() || !url.username().is_empty() || url.password().is_some() || url.port().is_some() || url.query().is_some() || url.fragment().is_some() { return Err(storage_driver_error( StorageErrorKind::Misconfigured, "COS download base URL must be an HTTPS URL with a hostname and no credentials, non-default port, query, or fragment", )); } } Ok(()) } pub fn validate_config( config: &TencentCosDriverConfig, credentials: &TencentCosStaticCredentials, ) -> Result<()> { Self::validate_connection_config(config)?; let normalized = normalize_s3_endpoint_and_bucket(&config.endpoint, &config.bucket) .map_err(Self::rewrap_s3_config_error)?; S3Driver::validate_config( &S3DriverConfig { endpoint: signing::cos_s3_endpoint( &normalized.endpoint, &normalized.bucket, config.use_cname, )?, bucket: normalized.bucket, base_path: config.base_path.clone(), region: "auto".to_string(), path_style: config.use_cname, connect_timeout: config.connect_timeout, read_timeout: config.read_timeout, operation_timeout: config.operation_timeout, }, &S3StaticCredentials { access_key: credentials.access_key.clone(), secret_key: credentials.secret_key.clone(), }, )?; Ok(()) } pub fn new( config: TencentCosDriverConfig, credentials: TencentCosStaticCredentials, ) -> Result { Self::validate_config(&config, &credentials)?; let normalized = normalize_s3_endpoint_and_bucket(&config.endpoint, &config.bucket) .map_err(Self::rewrap_s3_config_error)?; let s3_driver = S3Driver::new( S3DriverConfig { endpoint: signing::cos_s3_endpoint( &normalized.endpoint, &normalized.bucket, config.use_cname, )?, bucket: normalized.bucket.clone(), base_path: config.base_path.clone(), region: "auto".to_string(), path_style: config.use_cname, connect_timeout: config.connect_timeout, read_timeout: config.read_timeout, operation_timeout: config.operation_timeout, }, S3StaticCredentials { access_key: credentials.access_key.clone(), secret_key: credentials.secret_key.clone(), }, if config.use_cname { S3DriverOptions::path_style() } else { S3DriverOptions::virtual_hosted_style() }, |builder| { signing::configure_cos_auth( builder, config.use_cname.then(|| normalized.bucket.clone()), ) }, )?; let storage = S3CompatibleDriver::from_s3_driver_with_multipart_capabilities( Arc::new(s3_driver), MultipartStorageCapabilities { // Tencent COS Upload Part official contract: up to 10,000 // parts, each 1 MiB..5 GiB; the final part may be smaller. min_part_size: 1024 * 1024, max_part_size: Some(5 * 1024 * 1024 * 1024), max_parts: 10_000, max_object_size: None, upload_mode: MultipartUploadMode::NativeStreaming, }, ); let client = cos_ci_http_client(&config)?; Ok(Self { storage, client, endpoint: normalized.endpoint, bucket: normalized.bucket, access_key: credentials.access_key, secret_key: credentials.secret_key, base_path: config.base_path, use_cname: config.use_cname, download_base_url: config.download_base_url, }) } pub fn s3_driver(&self) -> std::sync::Arc { self.storage.s3_driver() } fn rewrap_s3_config_error(error: S3ConfigError) -> aster_drive_storage::StorageError { let message = match error { S3ConfigError::MissingBucket => { "bucket is required for S3-compatible storage".to_string() } S3ConfigError::InvalidEndpoint(message) => message, }; storage_driver_error(StorageErrorKind::Misconfigured, message) } fn full_key(&self, path: &str) -> String { object_key::join_key_prefix(&self.base_path, path) } } fn cos_ci_http_client(config: &TencentCosDriverConfig) -> Result { reqwest::Client::builder() .connect_timeout(config.connect_timeout) .read_timeout(config.read_timeout) .timeout(config.operation_timeout) .redirect(reqwest::redirect::Policy::none()) .user_agent(OUTBOUND_HTTP_USER_AGENT) .build() .map_storage_err_ctx(StorageErrorKind::Misconfigured, "build COS CI HTTP client") } #[async_trait::async_trait] impl DirectDownloadStorageDriver for TencentCosDriver { async fn resolve_download_url( &self, path: &str, expires: Duration, options: DirectDownloadOptions, ) -> Result> { if self.download_base_url.is_some() { let expires = super::s3::presigned::clamp_presign_ttl(expires, "COS resolve_download_url"); let url = self.signed_custom_download_url(path, expires, &options)?; return Ok(Some(DirectDownloadRequest::temporary_url(url, expires))); } self.storage .resolve_download_url(path, expires, options) .await } } #[async_trait::async_trait] impl PresignedUploadStorageDriver for TencentCosDriver { async fn presigned_put_request( &self, path: &str, expires: Duration, ) -> Result> { self.storage .s3_driver() .presigned_put_request(path, expires) .await } fn presigned_single_put_requires_etag(&self) -> bool { // Single-object completion verifies the uploaded object's metadata and // size server-side. ETag remains required for presigned multipart parts. false } } super::s3_compatible::delegate_s3_compatible_storage_driver!( TencentCosDriver, storage, native_thumbnail, native_media_metadata ); super::s3_compatible::delegate_s3_compatible_multipart_driver!(TencentCosDriver, storage);