diff --git a/application/single_app/config.py b/application/single_app/config.py index 6cd524ec4..a3f37abd0 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.066" +VERSION = "0.250.070" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_file_sync.py b/application/single_app/functions_file_sync.py index 103f81ef6..036bfa618 100644 --- a/application/single_app/functions_file_sync.py +++ b/application/single_app/functions_file_sync.py @@ -11,11 +11,12 @@ import uuid from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Tuple -from urllib.parse import quote, unquote, urlparse +from urllib.parse import parse_qsl, quote, unquote, urlparse from azure.core.exceptions import ResourceNotFoundError as AzureResourceNotFoundError from azure.identity import ClientSecretCredential, DefaultAzureCredential from azure.cosmos.exceptions import CosmosResourceNotFoundError +from azure.storage.blob import BlobServiceClient, ContainerClient from flask import current_app, has_app_context from msal import ConfidentialClientApplication @@ -78,12 +79,14 @@ FILE_SYNC_SCOPES = {FILE_SYNC_SCOPE_PERSONAL, FILE_SYNC_SCOPE_GROUP, FILE_SYNC_SCOPE_PUBLIC} FILE_SYNC_SOURCE_TYPE_SMB = "smb" FILE_SYNC_SOURCE_TYPE_AZURE_FILES = "azure_files" +FILE_SYNC_SOURCE_TYPE_AZURE_BLOB = "azure_blob" FILE_SYNC_SOURCE_TYPE_ONEDRIVE = "onedrive" FILE_SYNC_SOURCE_TYPE_SHAREPOINT_ON_PREM = "sharepoint_on_prem" FILE_SYNC_SOURCE_TYPE_GOOGLE_WORKSPACE = "google_workspace" FILE_SYNC_KNOWN_SOURCE_TYPES = { FILE_SYNC_SOURCE_TYPE_SMB, FILE_SYNC_SOURCE_TYPE_AZURE_FILES, + FILE_SYNC_SOURCE_TYPE_AZURE_BLOB, FILE_SYNC_SOURCE_TYPE_ONEDRIVE, FILE_SYNC_SOURCE_TYPE_SHAREPOINT_ON_PREM, FILE_SYNC_SOURCE_TYPE_GOOGLE_WORKSPACE, @@ -91,21 +94,57 @@ FILE_SYNC_IMPLEMENTED_SOURCE_TYPES = { FILE_SYNC_SOURCE_TYPE_SMB, FILE_SYNC_SOURCE_TYPE_AZURE_FILES, + FILE_SYNC_SOURCE_TYPE_AZURE_BLOB, FILE_SYNC_SOURCE_TYPE_ONEDRIVE, } FILE_SYNC_ADMIN_VISIBLE_SOURCE_TYPES = { FILE_SYNC_SOURCE_TYPE_SMB, FILE_SYNC_SOURCE_TYPE_AZURE_FILES, + FILE_SYNC_SOURCE_TYPE_AZURE_BLOB, } FILE_SYNC_SOURCE_TYPE_LABELS = { FILE_SYNC_SOURCE_TYPE_SMB: "SMB", FILE_SYNC_SOURCE_TYPE_AZURE_FILES: "Azure Files", + FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: "Azure Blob Storage", FILE_SYNC_SOURCE_TYPE_ONEDRIVE: "OneDrive", FILE_SYNC_SOURCE_TYPE_SHAREPOINT_ON_PREM: "On-prem SharePoint", FILE_SYNC_SOURCE_TYPE_GOOGLE_WORKSPACE: "Google Workspace", } +AZURE_STORAGE_ENDPOINT_SUFFIXES = ( + "core.windows.net", + "core.usgovcloudapi.net", + "core.chinacloudapi.cn", + "core.cloudapi.de", +) +AZURE_BLOB_SAS_PERMISSION_LABELS = { + "r": "Read", + "l": "List", + "a": "Add", + "c": "Create", + "w": "Write", + "d": "Delete", + "x": "Delete version", + "t": "Tags", + "m": "Move", + "e": "Execute", + "o": "Ownership", + "p": "Permissions", + "i": "Immutability policy", + "y": "Permanent delete", +} FILE_SYNC_MANAGER_ROLES = ("Owner", "Admin", "DocumentManager") FILE_SYNC_PERSONAL_APP_ROLE = "PersonalFileSyncUser" +FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE = "File Sync run failed. Contact an administrator if the problem continues." +FILE_SYNC_PUBLIC_ITEM_ERROR_MESSAGE = "File Sync could not process this item. Contact an administrator if the problem continues." + + +class FileSyncPublicValidationError(ValueError): + """A reviewed validation message that is safe to return to File Sync clients.""" + + def __init__(self, public_message: str): + self.public_message = str(public_message or "File Sync validation failed.") + super().__init__(self.public_message) + FILE_SYNC_DEFAULTS = { "enable_file_sync": False, @@ -137,6 +176,7 @@ FILE_SYNC_IDENTITY_AUTH_TYPES_BY_SOURCE = { FILE_SYNC_SOURCE_TYPE_SMB: {"username_password", "anonymous"}, FILE_SYNC_SOURCE_TYPE_AZURE_FILES: {"managed_identity", "client_secret", "connection_string"}, + FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: {"managed_identity", "client_secret", "connection_string"}, FILE_SYNC_SOURCE_TYPE_ONEDRIVE: {"client_secret"}, } FILE_SYNC_IDENTITY_AUTH_TYPES = set().union(*FILE_SYNC_IDENTITY_AUTH_TYPES_BY_SOURCE.values()) @@ -254,7 +294,7 @@ def _default_auth_type_for_source_type(source_type: str) -> str: normalized_source_type = _normalize_source_type(source_type) if normalized_source_type == FILE_SYNC_SOURCE_TYPE_ONEDRIVE: return "global_identity" - if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: + if normalized_source_type in {FILE_SYNC_SOURCE_TYPE_AZURE_FILES, FILE_SYNC_SOURCE_TYPE_AZURE_BLOB}: return "managed_identity" return "username_password" @@ -553,6 +593,7 @@ def list_file_sync_sources(scope_type: str, scope_id: str) -> List[Dict[str, Any def sanitize_file_sync_source(source: Dict[str, Any]) -> Dict[str, Any]: sanitized_source = dict(source or {}) auth = dict(sanitized_source.get("auth") or {}) + credential_metadata = sanitized_source.get("credential_metadata") identity_id = str(sanitized_source.get("identity_id") or "").strip() if identity_id: try: @@ -564,9 +605,21 @@ def sanitize_file_sync_source(source: Dict[str, Any]) -> Dict[str, Any]: ) sanitized_source["identity_name"] = identity.get("name", "") auth = dict(identity.get("auth") or {}) + if sanitized_source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + resolved_identity_auth = get_workspace_identity_auth( + sanitized_source.get("scope_type"), + _source_scope_id(sanitized_source), + identity_id, + ) + credential_metadata = _azure_blob_credential_metadata( + resolved_identity_auth, + sanitized_source.get("connection") or {}, + tolerate_validation_errors=True, + ) except Exception: sanitized_source["identity_name"] = "Unavailable identity" auth = {"auth_type": "identity_missing"} + credential_metadata = {} password_stored = bool(auth.get("password") or auth.get("password_secret_name")) secret_stored = bool(auth.get("secret") or auth.get("secret_secret_name")) sanitized_source["identity_id"] = identity_id @@ -580,6 +633,7 @@ def sanitize_file_sync_source(source: Dict[str, Any]) -> Dict[str, Any]: "password": ui_trigger_word if password_stored else "", "secret": ui_trigger_word if secret_stored else "", } + sanitized_source["credential_metadata"] = _sanitize_azure_blob_credential_metadata(credential_metadata) sanitized_source.pop("auth", None) return sanitized_source @@ -587,7 +641,7 @@ def sanitize_file_sync_source(source: Dict[str, Any]) -> Dict[str, Any]: def sanitize_file_sync_run(run: Dict[str, Any]) -> Dict[str, Any]: sanitized_run = dict(run or {}) if sanitized_run.get("error_message"): - sanitized_run["error_message"] = str(sanitized_run["error_message"])[:1000] + sanitized_run["error_message"] = FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE return sanitized_run @@ -713,6 +767,522 @@ def _normalize_azure_files_connection( } +def _azure_blob_endpoint_suffix_for_hostname(hostname: Any) -> Tuple[str, str]: + normalized_hostname = str(hostname or "").strip().lower() + for endpoint_suffix in AZURE_STORAGE_ENDPOINT_SUFFIXES: + blob_hostname_suffix = f".blob.{endpoint_suffix}" + if not normalized_hostname.endswith(blob_hostname_suffix): + continue + account_name = normalized_hostname[:-len(blob_hostname_suffix)] + if 3 <= len(account_name) <= 24 and account_name.isascii() and account_name.isalnum(): + return account_name, endpoint_suffix + raise ValueError("Azure Blob Storage endpoint must use a supported Azure Blob service hostname") + + +def _normalize_azure_blob_url(value: Any) -> Tuple[str, List[str]]: + raw_url = _normalize_text(value, max_length=2048) + if not raw_url: + return "", [] + if "://" not in raw_url: + if re.match(r"^[a-z0-9]{3,24}$", raw_url): + raw_url = f"https://{raw_url}.blob.core.windows.net" + else: + raw_url = f"https://{raw_url}" + + parsed_url = urlparse(raw_url) + try: + parsed_port = parsed_url.port + except ValueError as error: + raise ValueError("Azure Blob Storage endpoint is invalid") from error + if ( + parsed_url.scheme != "https" + or not parsed_url.hostname + or parsed_url.username is not None + or parsed_url.password is not None + or parsed_port is not None + or parsed_url.query + or parsed_url.fragment + or parsed_url.params + ): + raise ValueError("Azure Blob Storage sources require an HTTPS blob service URL or storage account name") + + account_name, endpoint_suffix = _azure_blob_endpoint_suffix_for_hostname(parsed_url.hostname) + path_parts = [unquote(path_part) for path_part in parsed_url.path.split("/") if path_part] + return f"https://{account_name}.blob.{endpoint_suffix}", path_parts + + +def _parse_azure_blob_sas_parameters(sas_token: Any) -> Dict[str, str]: + normalized_token = str(sas_token or "").strip().lstrip("?") + if not normalized_token: + return {} + parameters = {} + for raw_key, raw_value in parse_qsl(normalized_token, keep_blank_values=True): + key = str(raw_key or "").strip().lower() + if not key or key in parameters: + raise ValueError("Azure Blob Storage SAS token is invalid") + parameters[key] = str(raw_value or "").strip() + return parameters + + +def _parse_azure_blob_sas_datetime(value: Any) -> Optional[datetime]: + raw_value = str(value or "").strip() + if not raw_value: + return None + normalized_value = raw_value[:-1] + "+00:00" if raw_value.endswith("Z") else raw_value + try: + parsed_value = datetime.fromisoformat(normalized_value) + except ValueError as error: + raise ValueError("Azure Blob Storage SAS date is invalid") from error + if parsed_value.tzinfo is None: + parsed_value = parsed_value.replace(tzinfo=timezone.utc) + return parsed_value.astimezone(timezone.utc) + + +def _azure_blob_permission_labels(permissions: str) -> List[str]: + return [AZURE_BLOB_SAS_PERMISSION_LABELS.get(permission, permission) for permission in permissions] + + +def _azure_blob_sas_metadata(parameters: Dict[str, str]) -> Dict[str, Any]: + resource = str(parameters.get("sr") or "").lower() + services = str(parameters.get("ss") or "").lower() + resource_types = str(parameters.get("srt") or "").lower() + permissions = str(parameters.get("sp") or "").lower() + stored_access_policy = bool(parameters.get("si")) + if services or resource_types: + sas_scope = "account" + elif resource == "c": + sas_scope = "container" + elif resource == "b": + sas_scope = "blob" + else: + sas_scope = "unknown" + + starts_at = _parse_azure_blob_sas_datetime(parameters.get("st")) + expires_at = _parse_azure_blob_sas_datetime(parameters.get("se")) + warnings = [] + if sas_scope == "account": + warnings.append( + "Account SAS grants more access than this single-container source needs. Prefer a container SAS with Read and List only." + ) + if sas_scope in {"container", "account"}: + extra_permissions = "".join(permission for permission in permissions if permission not in {"r", "l"}) + if extra_permissions: + warnings.append( + f"{('Container' if sas_scope == 'container' else 'Account')} SAS includes permissions File Sync does not need: " + f"{', '.join(_azure_blob_permission_labels(extra_permissions))}. Read and List are sufficient." + ) + if stored_access_policy: + warnings.append( + "Permissions and expiry are controlled by a stored access policy and cannot be fully validated from this SAS token. Ensure the policy grants Read and List." + ) + + return { + "credential_type": "sas", + "sas_scope": sas_scope, + "permissions": permissions, + "starts_at": starts_at.isoformat() if starts_at else "", + "expires_at": expires_at.isoformat() if expires_at else "", + "https_only": str(parameters.get("spr") or "").lower() == "https", + "stored_access_policy": stored_access_policy, + "signed_version": str(parameters.get("sv") or ""), + "ip_range": str(parameters.get("sip") or ""), + "services": services, + "resource_types": resource_types, + "warnings": warnings, + } + + +def _parse_azure_blob_connection_string(connection_string: Any) -> Dict[str, Any]: + raw_connection_string = str(connection_string or "").strip() + if not raw_connection_string: + raise ValueError("Azure Blob Storage connection string is required") + + fields = {} + for segment in raw_connection_string.split(";"): + if not segment.strip(): + continue + if "=" not in segment: + raise ValueError("Azure Blob Storage connection string is invalid") + raw_key, raw_value = segment.split("=", 1) + key = raw_key.strip().lower() + if not key or key in fields: + raise ValueError("Azure Blob Storage connection string is invalid") + fields[key] = raw_value.strip() + + if fields.get("usedevelopmentstorage", "").lower() == "true": + raise ValueError("Azure Blob Storage development endpoints are not allowed") + + blob_endpoint = fields.get("blobendpoint", "") + if blob_endpoint: + account_url, endpoint_path_parts = _normalize_azure_blob_url(blob_endpoint) + else: + if fields.get("defaultendpointsprotocol", "").lower() != "https": + raise ValueError("Azure Blob Storage connection strings require HTTPS endpoints") + account_name = fields.get("accountname", "").lower() + endpoint_suffix = fields.get("endpointsuffix", "").lower() + if not (3 <= len(account_name) <= 24 and account_name.isascii() and account_name.isalnum()): + raise ValueError("Azure Blob Storage connection string account name is invalid") + if endpoint_suffix not in AZURE_STORAGE_ENDPOINT_SUFFIXES: + raise ValueError("Azure Blob Storage connection string endpoint is not allowed") + account_url, endpoint_path_parts = _normalize_azure_blob_url( + f"https://{account_name}.blob.{endpoint_suffix}" + ) + + if len(endpoint_path_parts) > 1: + endpoint_container_name = endpoint_path_parts[0] + elif endpoint_path_parts: + endpoint_container_name = endpoint_path_parts[0] + else: + endpoint_container_name = "" + + sas_token = fields.get("sharedaccesssignature", "").lstrip("?") + if sas_token: + sas_parameters = _parse_azure_blob_sas_parameters(sas_token) + credential_metadata = _azure_blob_sas_metadata(sas_parameters) + elif fields.get("accountkey"): + sas_parameters = {} + credential_metadata = { + "credential_type": "account_key", + "sas_scope": "account", + "permissions": "", + "starts_at": "", + "expires_at": "", + "https_only": True, + "stored_access_policy": False, + "signed_version": "", + "ip_range": "", + "services": "", + "resource_types": "", + "warnings": [ + "Storage account keys grant more access than this single-container source needs. Prefer a container SAS with Read and List only." + ], + } + else: + raise ValueError("Azure Blob Storage connection string requires a SAS token or account key") + + return { + "account_url": account_url, + "endpoint_path_parts": endpoint_path_parts, + "endpoint_container_name": endpoint_container_name, + "sas_token": sas_token, + "sas_parameters": sas_parameters, + "sas_metadata": credential_metadata, + } + + +def _parse_azure_blob_sas_url(sas_url: Any) -> Dict[str, Any]: + raw_sas_url = str(sas_url or "").strip() + parsed_url = urlparse(raw_sas_url) + try: + parsed_port = parsed_url.port + except ValueError as error: + raise ValueError("Azure Blob Storage SAS URL is invalid") from error + if ( + parsed_url.scheme != "https" + or not parsed_url.hostname + or parsed_url.username is not None + or parsed_url.password is not None + or parsed_port is not None + or not parsed_url.query + or parsed_url.fragment + or parsed_url.params + ): + raise ValueError("Azure Blob Storage SAS URL must use HTTPS and include a SAS token") + + account_name, endpoint_suffix = _azure_blob_endpoint_suffix_for_hostname(parsed_url.hostname) + account_url = f"https://{account_name}.blob.{endpoint_suffix}" + endpoint_path_parts = [unquote(path_part) for path_part in parsed_url.path.split("/") if path_part] + sas_token = parsed_url.query + sas_parameters = _parse_azure_blob_sas_parameters(sas_token) + return { + "account_url": account_url, + "endpoint_path_parts": endpoint_path_parts, + "endpoint_container_name": endpoint_path_parts[0] if endpoint_path_parts else "", + "sas_token": sas_token, + "sas_parameters": sas_parameters, + "sas_metadata": _azure_blob_sas_metadata(sas_parameters), + } + + +def _parse_azure_blob_credential( + credential_value: Any, + account_url: Any = "", + container_name: Any = "", +) -> Dict[str, Any]: + raw_credential = str(credential_value or "").strip() + if not raw_credential: + raise ValueError("Azure Blob Storage credential is required") + + if raw_credential.lower().startswith("https://"): + return _parse_azure_blob_sas_url(raw_credential) + + normalized_lower = raw_credential.lower() + connection_string_keys = ( + "accountname=", + "blobendpoint=", + "defaultendpointsprotocol=", + "usedevelopmentstorage=", + ) + if ";" in raw_credential or normalized_lower.startswith(connection_string_keys): + return _parse_azure_blob_connection_string(raw_credential) + + safe_account_url, account_path_parts = _normalize_azure_blob_url(account_url) + if not safe_account_url or account_path_parts: + raise ValueError("Azure Blob Storage account URL is required when using a standalone SAS token") + normalized_container_name = _normalize_azure_container_name(container_name) + sas_token = raw_credential.lstrip("?") + sas_parameters = _parse_azure_blob_sas_parameters(sas_token) + return { + "account_url": safe_account_url, + "endpoint_path_parts": [normalized_container_name], + "endpoint_container_name": normalized_container_name, + "sas_token": sas_token, + "sas_parameters": sas_parameters, + "sas_metadata": _azure_blob_sas_metadata(sas_parameters), + } + + +def _validate_azure_blob_connection_string(connection_string: Any) -> Dict[str, Any]: + return _parse_azure_blob_credential(connection_string) + + +def _validate_azure_blob_sas_for_container( + parsed_connection: Dict[str, Any], + container_name: str, + account_url: str = "", +) -> None: + metadata = parsed_connection.get("sas_metadata") or {} + normalized_container_name = _normalize_azure_container_name(container_name) + if account_url: + normalized_account_url, account_path_parts = _normalize_azure_blob_url(account_url) + if account_path_parts or normalized_account_url != parsed_connection.get("account_url"): + raise FileSyncPublicValidationError( + "The Blob credential account does not match the configured Blob service URL." + ) + if metadata.get("credential_type") != "sas": + if parsed_connection.get("endpoint_path_parts"): + raise FileSyncPublicValidationError( + "A storage account key connection string must use a Blob service endpoint without a container path." + ) + return + + sas_parameters = parsed_connection.get("sas_parameters") or {} + if not sas_parameters.get("sv") or not sas_parameters.get("sig"): + raise FileSyncPublicValidationError( + "The Blob SAS is incomplete. Generate a new SAS URL or token that includes its signed version and signature." + ) + has_account_scope = bool(sas_parameters.get("ss") or sas_parameters.get("srt")) + if has_account_scope and sas_parameters.get("sr"): + raise FileSyncPublicValidationError( + "The Blob SAS contains conflicting account and resource scope fields. Generate a new container or account SAS." + ) + + endpoint_container_name = str(parsed_connection.get("endpoint_container_name") or "").lower() + if endpoint_container_name and endpoint_container_name != normalized_container_name: + raise FileSyncPublicValidationError( + "The container in the Blob SAS URL does not match the selected container." + ) + + sas_scope = metadata.get("sas_scope") + if sas_scope == "blob": + raise FileSyncPublicValidationError( + "A blob/object SAS cannot list a container. Use a container SAS with Read and List permissions." + ) + if sas_scope not in {"container", "account"}: + raise FileSyncPublicValidationError( + "The SAS scope is not supported for container sync. Use a container SAS with Read and List permissions." + ) + if len(parsed_connection.get("endpoint_path_parts") or []) > 1: + raise FileSyncPublicValidationError( + "The Blob SAS URL may identify one container only, not an individual blob or nested path." + ) + if sas_scope == "account": + if "b" not in metadata.get("services", ""): + raise FileSyncPublicValidationError("The account SAS must include the Blob service.") + resource_types = metadata.get("resource_types", "") + if "c" not in resource_types or "o" not in resource_types: + raise FileSyncPublicValidationError( + "The account SAS must include Container and Object resource types." + ) + + if not metadata.get("https_only"): + raise FileSyncPublicValidationError("The Blob SAS must set Allowed protocols to HTTPS only.") + + stored_access_policy = bool(metadata.get("stored_access_policy")) + permissions = str(metadata.get("permissions") or "") + if not stored_access_policy and not {"r", "l"}.issubset(set(permissions)): + raise FileSyncPublicValidationError( + "The Blob SAS must include both Read and List permissions. Regenerate the container SAS with Read and List selected." + ) + + expires_at = _parse_azure_blob_sas_datetime(metadata.get("expires_at")) + if not stored_access_policy and not expires_at: + raise FileSyncPublicValidationError("The Blob SAS must include an expiry time.") + if expires_at and expires_at <= _now(): + raise FileSyncPublicValidationError( + "The Blob SAS has expired. Generate and save a new SAS credential." + ) + starts_at = _parse_azure_blob_sas_datetime(metadata.get("starts_at")) + if starts_at and starts_at > _now() + timedelta(minutes=5): + raise FileSyncPublicValidationError( + "The Blob SAS is not valid yet. Check its start time or allow for clock skew." + ) + + +def _sanitize_azure_blob_credential_metadata(metadata: Any) -> Dict[str, Any]: + if not isinstance(metadata, dict): + return {} + sanitized_metadata = { + "credential_type": _normalize_text(metadata.get("credential_type"), 30), + "sas_scope": _normalize_text(metadata.get("sas_scope"), 30), + "permissions": _normalize_text(metadata.get("permissions"), 30), + "starts_at": _normalize_text(metadata.get("starts_at"), 80), + "expires_at": _normalize_text(metadata.get("expires_at"), 80), + "https_only": bool(metadata.get("https_only")), + "stored_access_policy": bool(metadata.get("stored_access_policy")), + "signed_version": _normalize_text(metadata.get("signed_version"), 30), + "ip_range": _normalize_text(metadata.get("ip_range"), 100), + "services": _normalize_text(metadata.get("services"), 30), + "resource_types": _normalize_text(metadata.get("resource_types"), 30), + "warnings": [ + _normalize_text(warning, 500) + for warning in (metadata.get("warnings") or []) + if str(warning or "").strip() + ][:10], + } + return sanitized_metadata + + +def _azure_blob_credential_metadata( + auth: Dict[str, Any], + connection: Dict[str, Any], + tolerate_validation_errors: bool = False, +) -> Dict[str, Any]: + auth = auth or {} + auth_type = _normalize_text(auth.get("auth_type"), 50).lower() or "managed_identity" + if auth_type == "connection_string": + credential_value = _resolved_auth_secret(auth) + parsed_connection = _parse_azure_blob_credential( + credential_value, + account_url=connection.get("account_url"), + container_name=connection.get("container_name"), + ) + validation_warning = "" + try: + _validate_azure_blob_sas_for_container( + parsed_connection, + connection.get("container_name"), + account_url=connection.get("account_url"), + ) + except FileSyncPublicValidationError as error: + if not tolerate_validation_errors: + raise + validation_warning = error.public_message + metadata = _sanitize_azure_blob_credential_metadata(parsed_connection.get("sas_metadata")) + if validation_warning and validation_warning not in metadata["warnings"]: + metadata["warnings"].append(validation_warning) + return metadata + return _sanitize_azure_blob_credential_metadata({ + "credential_type": auth_type, + "sas_scope": "", + "permissions": "", + "starts_at": "", + "expires_at": "", + "https_only": True, + "stored_access_policy": False, + "signed_version": "", + "ip_range": "", + "services": "", + "resource_types": "", + "warnings": [], + }) + + +def _prevalidate_new_azure_blob_credential( + raw_credentials: Dict[str, Any], + existing_auth: Dict[str, Any], + connection: Dict[str, Any], +) -> Dict[str, Any]: + raw_credentials = raw_credentials or {} + existing_auth = existing_auth or {} + auth_type = _normalize_text( + raw_credentials.get("auth_type", existing_auth.get("auth_type", "managed_identity")), + 50, + ).lower() + if auth_type != "connection_string": + return {} + credential_value = _get_file_sync_secret_value( + raw_credentials, + "connection_string", + "secret", + "key", + ) + if credential_value in [None, "", ui_trigger_word]: + return {} + parsed_connection = _parse_azure_blob_credential( + credential_value, + account_url=connection.get("account_url"), + container_name=connection.get("container_name"), + ) + _validate_azure_blob_sas_for_container( + parsed_connection, + connection.get("container_name"), + account_url=connection.get("account_url"), + ) + return _sanitize_azure_blob_credential_metadata(parsed_connection.get("sas_metadata")) + + +def _normalize_azure_container_name(value: Any) -> str: + container_name = _normalize_text(value, 63).lower() + if container_name == "$root": + return container_name + if ( + not re.match(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])$", container_name) + or "--" in container_name + ): + raise ValueError("Azure Blob Storage container name must be 3-63 lowercase letters, numbers, or hyphens") + return container_name + + +def _normalize_azure_blob_connection( + connection: Dict[str, Any], + existing_connection: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + connection = connection or {} + existing_connection = existing_connection or {} + raw_url = ( + connection.get("account_url") + or connection.get("blob_service_url") + or connection.get("account_name") + or existing_connection.get("account_url") + or existing_connection.get("blob_service_url") + or existing_connection.get("account_name") + or "" + ) + account_url, url_path_parts = _normalize_azure_blob_url(raw_url) + if not account_url: + raise ValueError("Azure Blob Storage sources require a blob service URL or storage account name") + + container_name = _normalize_text( + connection.get("container_name", existing_connection.get("container_name", "")), + 63, + ) + if not container_name and url_path_parts: + container_name = url_path_parts[0] + container_name = _normalize_azure_container_name(container_name) + + raw_blob_prefix = connection.get("blob_prefix", existing_connection.get("blob_prefix", "")) + if not raw_blob_prefix and len(url_path_parts) > 1: + raw_blob_prefix = "/".join(url_path_parts[1:]) + blob_prefix = _normalize_selected_path(raw_blob_prefix) + return { + "account_url": account_url, + "container_name": container_name, + "blob_prefix": blob_prefix, + "selected_paths": _normalize_selected_paths(connection.get("selected_paths", existing_connection.get("selected_paths", []))), + } + + def _normalize_onedrive_connection( connection: Dict[str, Any], existing_connection: Optional[Dict[str, Any]] = None, @@ -728,6 +1298,8 @@ def _normalize_connection_payload(source_type: str, connection: Dict[str, Any], normalized_source_type = _normalize_source_type(source_type) if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _normalize_azure_files_connection(connection, existing_connection) + if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + return _normalize_azure_blob_connection(connection, existing_connection) if normalized_source_type == FILE_SYNC_SOURCE_TYPE_ONEDRIVE: return _normalize_onedrive_connection(connection, existing_connection) return { @@ -736,6 +1308,54 @@ def _normalize_connection_payload(source_type: str, connection: Dict[str, Any], } +def _hydrate_azure_blob_connection_from_credential( + connection: Dict[str, Any], + existing_connection: Dict[str, Any], + raw_credentials: Dict[str, Any], +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + hydrated_connection = dict(connection or {}) + hydrated_credentials = dict(raw_credentials or {}) + account_url_input = str(hydrated_connection.get("account_url") or "").strip() + auth_type = _normalize_text(hydrated_credentials.get("auth_type"), 50).lower() + credential_field_names = ( + ("connection_string", "secret", "key") + if auth_type == "connection_string" + else ("connection_string", "key") + ) + existing_credential_value = _get_file_sync_secret_value( + hydrated_credentials, + *credential_field_names, + ) + if account_url_input.lower().startswith("https://") and "?" in account_url_input and existing_credential_value in [None, "", ui_trigger_word]: + hydrated_credentials["connection_string"] = account_url_input + hydrated_credentials["secret"] = account_url_input + hydrated_credentials["auth_type"] = "connection_string" + auth_type = "connection_string" + if auth_type and auth_type != "connection_string": + return hydrated_connection, hydrated_credentials + credential_value = _get_file_sync_secret_value( + hydrated_credentials, + *credential_field_names, + ) + if credential_value in [None, "", ui_trigger_word]: + return hydrated_connection, hydrated_credentials + if not auth_type: + hydrated_credentials["auth_type"] = "connection_string" + + fallback_account_url = hydrated_connection.get("account_url") or existing_connection.get("account_url") or "" + fallback_container_name = hydrated_connection.get("container_name") or existing_connection.get("container_name") or "" + parsed_credential = _parse_azure_blob_credential( + credential_value, + account_url=fallback_account_url, + container_name=fallback_container_name, + ) + if "?" in account_url_input or not hydrated_connection.get("account_url"): + hydrated_connection["account_url"] = parsed_credential.get("account_url", "") + if not hydrated_connection.get("container_name") and parsed_credential.get("endpoint_container_name"): + hydrated_connection["container_name"] = parsed_credential["endpoint_container_name"] + return hydrated_connection, hydrated_credentials + + def _normalize_patterns(value: Any) -> List[str]: return parse_file_sync_list(value) @@ -812,6 +1432,17 @@ def _prepare_auth_payload( if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _prepare_azure_files_auth_payload(scope_type, scope_id, source_id, raw_credentials, existing_auth, auth_type) + if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + prepared_auth = _prepare_azure_files_auth_payload( + scope_type, + scope_id, + source_id, + raw_credentials, + existing_auth, + auth_type, + source_label="Azure Blob Storage", + ) + return prepared_auth username = _normalize_text(raw_credentials.get("username", existing_auth.get("username", "")), 255) domain = _normalize_text(raw_credentials.get("domain", existing_auth.get("domain", "")), 255) @@ -872,6 +1503,7 @@ def _prepare_azure_files_auth_payload( raw_credentials: Dict[str, Any], existing_auth: Dict[str, Any], auth_type: str, + source_label: str = "Azure Files", ) -> Dict[str, Any]: prepared_auth = {"auth_type": auth_type} if auth_type == "managed_identity": @@ -886,7 +1518,7 @@ def _prepare_azure_files_auth_payload( if auth_type == "client_secret": client_id = _normalize_text(raw_credentials.get("client_id", raw_credentials.get("identity", existing_auth.get("identity", ""))), 255) if not client_id: - raise ValueError("Azure Files service principal identities require a client ID") + raise ValueError(f"{source_label} service principal identities require a client ID") prepared_auth["identity"] = client_id tenant_id = _normalize_text(raw_credentials.get("tenant_id", existing_auth.get("tenant_id", "")), 255) if tenant_id: @@ -898,7 +1530,7 @@ def _prepare_azure_files_auth_payload( elif existing_auth.get("secret"): prepared_auth["secret"] = existing_auth["secret"] else: - raise ValueError("Azure Files service principal identities require a client secret") + raise ValueError(f"{source_label} service principal identities require a client secret") return prepared_auth return _store_prepared_secret(scope_type, scope_id, source_id, prepared_auth, "secret", str(secret_value)) @@ -909,7 +1541,12 @@ def _prepare_azure_files_auth_payload( elif existing_auth.get("secret"): prepared_auth["secret"] = existing_auth["secret"] else: - raise ValueError("Azure Files connection string identities require a connection string") + credential_description = ( + "a connection string, SAS URL, or SAS token" + if source_label == "Azure Blob Storage" + else "a connection string" + ) + raise ValueError(f"{source_label} credential authentication requires {credential_description}") return prepared_auth return _store_prepared_secret(scope_type, scope_id, source_id, prepared_auth, "secret", str(secret_value)) @@ -945,11 +1582,18 @@ def _normalize_source_payload( connection = payload.get("connection") or {} existing_connection = existing_source.get("connection") or {} + raw_credentials = payload.get("credentials") or payload.get("auth") or {} + identity_id = _normalize_text(payload.get("identity_id", existing_source.get("identity_id", "")), 255) + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB and not identity_id: + connection, raw_credentials = _hydrate_azure_blob_connection_from_credential( + connection, + existing_connection, + raw_credentials, + ) filters = payload.get("filters") or {} existing_filters = existing_source.get("filters") or {} schedule = payload.get("schedule") or {} existing_schedule = existing_source.get("schedule") or {} - identity_id = _normalize_text(payload.get("identity_id", existing_source.get("identity_id", "")), 255) raw_delete_policy = _normalize_text( payload.get("remote_delete_policy", existing_source.get("remote_delete_policy", config["file_sync_default_remote_delete_policy"])), 50, @@ -962,12 +1606,23 @@ def _normalize_source_payload( if folder_tag_mode not in FILE_SYNC_FOLDER_TAG_MODES: folder_tag_mode = "parent" + normalized_connection = _normalize_connection_payload(source_type, connection, existing_connection) + prevalidated_credential_metadata = {} + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB and not identity_id: + prevalidated_credential_metadata = _prevalidate_new_azure_blob_credential( + raw_credentials, + existing_source.get("auth") or {}, + normalized_connection, + ) + default_source_name = existing_source.get("name", f"{_source_type_label(source_type)} File Sync Source") + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB and normalized_connection.get("container_name"): + default_source_name = existing_source.get("name") or f"{normalized_connection['container_name']} Blob Sync" normalized_source = { - "name": _normalize_text(payload.get("name", existing_source.get("name", f"{_source_type_label(source_type)} File Sync Source")), 120), + "name": _normalize_text(payload.get("name") or default_source_name, 120), "source_type": source_type, "enabled": _as_bool(payload.get("enabled", existing_source.get("enabled", True))), "recursive": _as_bool(payload.get("recursive", existing_source.get("recursive", True))) and config["file_sync_allow_recursive_sources"], - "connection": _normalize_connection_payload(source_type, connection, existing_connection), + "connection": normalized_connection, "filters": { "include_patterns": _normalize_patterns(filters.get("include_patterns", existing_filters.get("include_patterns", []))), "exclude_patterns": _normalize_patterns(filters.get("exclude_patterns", existing_filters.get("exclude_patterns", []))), @@ -982,15 +1637,22 @@ def _normalize_source_payload( if identity_id: _get_file_sync_identity(scope_type, scope_id, identity_id, source_type) normalized_source["auth"] = {} + resolved_auth = get_workspace_identity_auth(scope_type, scope_id, identity_id) else: normalized_source["auth"] = _prepare_auth_payload( scope_type=scope_type, scope_id=scope_id, source_id=source_id, source_type=source_type, - raw_credentials=payload.get("credentials") or payload.get("auth") or {}, + raw_credentials=raw_credentials, existing_auth=existing_source.get("auth") or {}, ) + resolved_auth = normalized_source["auth"] + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + normalized_source["credential_metadata"] = ( + prevalidated_credential_metadata + or _azure_blob_credential_metadata(resolved_auth, normalized_source["connection"]) + ) return normalized_source @@ -1056,6 +1718,13 @@ def _prepare_connection_test_auth( if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _prepare_connection_test_azure_files_auth(raw_credentials, existing_auth, auth_type) + if normalized_source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + return _prepare_connection_test_azure_files_auth( + raw_credentials, + existing_auth, + auth_type, + source_label="Azure Blob Storage", + ) prepared_auth = { "auth_type": auth_type, @@ -1082,6 +1751,7 @@ def _prepare_connection_test_azure_files_auth( raw_credentials: Dict[str, Any], existing_auth: Dict[str, Any], auth_type: str, + source_label: str = "Azure Files", ) -> Dict[str, Any]: prepared_auth = {"auth_type": auth_type} if auth_type == "managed_identity": @@ -1096,7 +1766,7 @@ def _prepare_connection_test_azure_files_auth( if auth_type == "client_secret": client_id = _normalize_text(raw_credentials.get("client_id", raw_credentials.get("identity", existing_auth.get("identity", ""))), 255) if not client_id: - raise ValueError("Azure Files service principal identities require a client ID") + raise ValueError(f"{source_label} service principal identities require a client ID") prepared_auth["identity"] = client_id tenant_id = _normalize_text(raw_credentials.get("tenant_id", existing_auth.get("tenant_id", "")), 255) if tenant_id: @@ -1108,7 +1778,7 @@ def _prepare_connection_test_azure_files_auth( elif existing_auth.get("secret"): prepared_auth["secret"] = existing_auth["secret"] else: - raise ValueError("Azure Files service principal identities require a client secret") + raise ValueError(f"{source_label} service principal identities require a client secret") else: prepared_auth["secret"] = str(secret_value) return prepared_auth @@ -1120,7 +1790,12 @@ def _prepare_connection_test_azure_files_auth( elif existing_auth.get("secret"): prepared_auth["secret"] = existing_auth["secret"] else: - raise ValueError("Azure Files connection string identities require a connection string") + credential_description = ( + "a connection string, SAS URL, or SAS token" + if source_label == "Azure Blob Storage" + else "a connection string" + ) + raise ValueError(f"{source_label} credential authentication requires {credential_description}") else: prepared_auth["secret"] = str(secret_value) return prepared_auth @@ -1144,9 +1819,16 @@ def _build_connection_test_source( connection = payload.get("connection") or {} existing_connection = existing_source.get("connection") or {} + raw_credentials = payload.get("credentials") or payload.get("auth") or {} + identity_id = _normalize_text(payload.get("identity_id", existing_source.get("identity_id", "")), 255) + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB and not identity_id: + connection, raw_credentials = _hydrate_azure_blob_connection_from_credential( + connection, + existing_connection, + raw_credentials, + ) config = get_file_sync_config() test_source_id = source_id or "connection-test" - identity_id = _normalize_text(payload.get("identity_id", existing_source.get("identity_id", "")), 255) auth = {} if identity_id: _get_file_sync_identity(scope_type, scope_id, identity_id, source_type) @@ -1154,10 +1836,10 @@ def _build_connection_test_source( else: auth = _prepare_connection_test_auth( source_type, - payload.get("credentials") or payload.get("auth") or {}, + raw_credentials, existing_source.get("auth") or {}, ) - return { + test_source = { "id": test_source_id, "source_id": test_source_id, "scope_type": _validate_scope(scope_type), @@ -1169,6 +1851,12 @@ def _build_connection_test_source( "connection": _normalize_connection_payload(source_type, connection, existing_connection), "auth": auth, } + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + test_source["credential_metadata"] = _azure_blob_credential_metadata( + auth, + test_source["connection"], + ) + return test_source def test_file_sync_source_connection( @@ -1183,6 +1871,8 @@ def test_file_sync_source_connection( return _test_onedrive_connection(source) if source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _test_azure_files_connection(source) + if source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + return _test_azure_blob_connection(source) try: smbclient = _register_smb_session(source) @@ -1227,6 +1917,8 @@ def browse_file_sync_source_path( entries = _browse_onedrive_path(source, browse_path) elif source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: entries = _browse_azure_files_path(source, browse_path) + elif source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + entries = _browse_azure_blob_path(source, browse_path) else: entries = _browse_smb_path(source, browse_path) return { @@ -1268,6 +1960,109 @@ def _test_azure_files_connection(source: Dict[str, Any]) -> Dict[str, Any]: raise ValueError("Azure Files connection test failed. Verify the file endpoint, share, and identity permissions.") from error +def _test_azure_blob_connection(source: Dict[str, Any]) -> Dict[str, Any]: + try: + container_client = _get_azure_blob_container_client(source) + blob_prefix = source.get("connection", {}).get("blob_prefix", "") + browse_prefix = f"{blob_prefix.rstrip('/')}/" if blob_prefix else "" + entries_checked = 0 + files_seen = 0 + folders_seen = 0 + read_verified = False + for entry in container_client.walk_blobs(name_starts_with=browse_prefix, delimiter="/"): + entries_checked += 1 + if _azure_blob_item_is_folder(entry): + folders_seen += 1 + else: + files_seen += 1 + if not read_verified: + container_client.get_blob_client(_azure_blob_item_name(entry)).get_blob_properties() + read_verified = True + if entries_checked >= 25: + break + if not read_verified: + for blob in container_client.list_blobs(name_starts_with=blob_prefix or None): + blob_name = _azure_blob_item_name(blob) + if not blob_name or _azure_blob_item_is_folder(blob): + continue + container_client.get_blob_client(blob_name).get_blob_properties() + read_verified = True + files_seen += 1 + break + credential_metadata = _sanitize_azure_blob_credential_metadata(source.get("credential_metadata")) + warnings = list(credential_metadata.get("warnings") or []) + if not read_verified: + warnings.append( + "List access was verified, but Read access could not be tested because no blob was found under the configured source root." + ) + credential_metadata["warnings"] = warnings + return { + "success": True, + "source_type": source["source_type"], + "recursive": source.get("recursive", True), + "entries_checked": entries_checked, + "files_seen": files_seen, + "folders_seen": folders_seen, + "read_verified": read_verified, + "credential_metadata": credential_metadata, + } + except FileSyncPublicValidationError: + raise + except Exception as error: + diagnostics = _azure_blob_error_diagnostics(error, source) + log_event( + "[FileSync] Azure Blob connection test failed.", + level=logging.WARNING, + extra=diagnostics, + exceptionTraceback=True, + ) + error_code = diagnostics.get("error_code", "") + if error_code == "authorizationpermissionmismatch": + raise FileSyncPublicValidationError( + "Azure rejected the Blob operation. A container SAS must include both Read and List permissions." + ) from error + if error_code == "authorizationfailure": + raise FileSyncPublicValidationError( + "Azure denied the Blob request. Confirm the SAS matches the account and container, is within its start/expiry window, and is allowed by the storage account network policy." + ) from error + if error_code in {"authenticationfailed", "invalidauthenticationinfo"}: + raise FileSyncPublicValidationError( + "Azure could not authenticate the Blob credential. Check the account, container, signature, start time, and expiry." + ) from error + if error_code in {"authorizationipmismatch", "ipsourcenotallowed", "ipauthorizationfailure"}: + raise FileSyncPublicValidationError( + "Azure blocked the Blob request because of an IP or network restriction. Include the app's outbound addresses or adjust the storage network policy." + ) from error + if error_code in {"containernotfound", "resourcenotfound"} or isinstance(error, AzureResourceNotFoundError): + raise FileSyncPublicValidationError( + "Azure could not find the selected Blob container, or the credential cannot access it." + ) from error + raise ValueError("Azure Blob Storage connection test failed. Verify the blob endpoint, container, prefix, and identity permissions.") from error + + +def _azure_blob_error_diagnostics(error: Exception, source: Dict[str, Any]) -> Dict[str, Any]: + response = getattr(error, "response", None) + headers = getattr(response, "headers", {}) or {} + credential_metadata = _sanitize_azure_blob_credential_metadata(source.get("credential_metadata")) + return { + "exception_type": type(error).__name__, + "error_code": _normalize_azure_storage_error_code(getattr(error, "error_code", "")), + "status_code": getattr(error, "status_code", None) or getattr(response, "status_code", None), + "request_id": _normalize_text(headers.get("x-ms-request-id", ""), 100), + "source_id": source.get("id"), + "auth_kind": credential_metadata.get("credential_type", ""), + "scope_kind": credential_metadata.get("sas_scope", ""), + "permissions": credential_metadata.get("permissions", ""), + } + + +def _normalize_azure_storage_error_code(value: Any) -> str: + raw_value = str(getattr(value, "value", value) or "").strip() + if "." in raw_value: + raw_value = raw_value.rsplit(".", 1)[-1] + return re.sub(r"[^a-z0-9]", "", raw_value.lower())[:100] + + def delete_file_sync_source(scope_type: str, scope_id: str, source_id: str, deleted_by: str, delete_associated_files: bool = False) -> Dict[str, Any]: source = get_authorized_sync_source(scope_type, source_id, deleted_by, scope_id=scope_id) delete_result = { @@ -1607,19 +2402,33 @@ def _process_file_sync_source( _log_file_sync_activity(source, triggered_by, "run_completed", {"run_id": run["id"], "counts": counts}) return run except Exception as error: - error_message = str(error) + detailed_error_message = str(error) run = _update_run( run, { "status": "failed", "counts": counts, "completed_at": _now_iso(), - "error_message": error_message, + "error_message": FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE, }, ) _update_source_after_run(source, run) - _log_file_sync_activity(source, triggered_by, "run_failed", {"run_id": run["id"], "error": error_message}) - log_event(f"[FileSync] Run failed for source {source.get('id')}: {error_message}", level=logging.ERROR, exceptionTraceback=True) + _log_file_sync_activity( + source, + triggered_by, + "run_failed", + {"run_id": run["id"], "error": FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE}, + ) + log_event( + "[FileSync] Run failed.", + level=logging.ERROR, + extra={ + "source_id": source.get("id"), + "run_id": run.get("id"), + "error": detailed_error_message, + }, + exceptionTraceback=True, + ) return run @@ -1713,6 +2522,8 @@ def _list_remote_files(source: Dict[str, Any], config: Dict[str, Any]) -> List[D return _list_onedrive_files(source, config) if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _list_azure_files(source, config) + if source_type == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + return _list_azure_blobs(source, config) return _list_smb_files(source, config) @@ -1721,6 +2532,8 @@ def _stage_remote_file(source: Dict[str, Any], remote_file: Dict[str, Any]) -> T return _stage_onedrive_file(source, remote_file) if source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_FILES: return _stage_azure_files_file(source, remote_file) + if source.get("source_type") == FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: + return _stage_azure_blob_file(source, remote_file) return _stage_smb_file(source, remote_file["remote_path"], remote_file["file_name"]) @@ -2052,6 +2865,92 @@ def _get_azure_files_share_client(source: Dict[str, Any]): return _get_azure_files_service_client(source).get_share_client(share_name) +def _get_azure_blob_service_client(source: Dict[str, Any]): + connection = source.get("connection") or {} + auth = _get_identity_auth_for_source(source) or source.get("auth") or {} + auth_type = _normalize_text(auth.get("auth_type"), 50).lower() or "managed_identity" + if auth_type == "connection_string": + credential_value = _resolved_auth_secret(auth) + if not credential_value: + raise ValueError("Azure Blob Storage credential authentication requires a connection string, SAS URL, or SAS token") + container_name = source.get("connection", {}).get("container_name") or "" + parsed_connection = _parse_azure_blob_credential( + credential_value, + account_url=connection.get("account_url"), + container_name=container_name, + ) + _validate_azure_blob_sas_for_container( + parsed_connection, + container_name, + account_url=connection.get("account_url"), + ) + if parsed_connection.get("sas_token"): + safe_sas_account_url, safe_sas_path_parts = _normalize_azure_blob_url( + parsed_connection["account_url"] + ) + if safe_sas_path_parts: + raise ValueError("Azure Blob Storage SAS account URL must be a service URL") + return BlobServiceClient( + account_url=safe_sas_account_url, + credential=parsed_connection["sas_token"], + ) + return BlobServiceClient.from_connection_string(credential_value) + + safe_account_url, account_path_parts = _normalize_azure_blob_url(connection.get("account_url")) + if not safe_account_url: + raise ValueError("Azure Blob Storage source is missing an account URL") + if account_path_parts: + raise ValueError("Azure Blob Storage source account URL must be a service URL") + if auth_type == "client_secret": + client_id = auth.get("identity") or "" + client_secret = _resolved_auth_secret(auth) + tenant_id = auth.get("tenant_id") or TENANT_ID + if not tenant_id or not client_id or not client_secret: + raise ValueError("Azure Blob Storage service principal authentication requires tenant ID, client ID, and client secret") + credential = ClientSecretCredential( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + ) + elif auth_type == "managed_identity": + credential = DefaultAzureCredential(managed_identity_client_id=auth.get("managed_identity_client_id") or None) + else: + raise ValueError("Azure Blob Storage sources require managed identity, service principal, or connection string authentication") + return BlobServiceClient(account_url=safe_account_url, credential=credential) + + +def _get_azure_blob_container_client(source: Dict[str, Any]): + container_name = source.get("connection", {}).get("container_name") or "" + if not container_name: + raise ValueError("Azure Blob Storage source is missing a container name") + auth = _get_identity_auth_for_source(source) or source.get("auth") or {} + auth_type = _normalize_text(auth.get("auth_type"), 50).lower() or "managed_identity" + if auth_type == "connection_string": + credential_value = _resolved_auth_secret(auth) + parsed_connection = _parse_azure_blob_credential( + credential_value, + account_url=source.get("connection", {}).get("account_url"), + container_name=container_name, + ) + _validate_azure_blob_sas_for_container( + parsed_connection, + container_name, + account_url=source.get("connection", {}).get("account_url"), + ) + if parsed_connection.get("sas_token"): + safe_sas_account_url, safe_sas_path_parts = _normalize_azure_blob_url( + parsed_connection["account_url"] + ) + if safe_sas_path_parts: + raise ValueError("Azure Blob Storage SAS account URL must be a service URL") + return ContainerClient( + account_url=safe_sas_account_url, + container_name=container_name, + credential=parsed_connection["sas_token"], + ) + return _get_azure_blob_service_client(source).get_container_client(container_name) + + def _resolved_auth_secret(auth: Dict[str, Any]) -> str: if auth.get("secret_secret_name"): return retrieve_secret_from_key_vault_by_full_name(auth["secret_secret_name"]) @@ -2093,6 +2992,138 @@ def _azure_files_item_change_token(item: Any) -> Optional[str]: return getattr(item, "etag", None) or getattr(item, "ETag", None) +def _azure_blob_item_value(item: Any, field_name: str, default_value: Any = None) -> Any: + if hasattr(item, "get"): + return item.get(field_name, default_value) + return getattr(item, field_name, default_value) + + +def _azure_blob_item_name(item: Any) -> str: + return str(_azure_blob_item_value(item, "name", "") or "") + + +def _azure_blob_item_is_folder(item: Any) -> bool: + metadata = _azure_blob_item_value(item, "metadata", {}) or {} + metadata_is_folder = isinstance(metadata, dict) and str(metadata.get("hdi_isfolder", "")).lower() == "true" + resource_type = str(_azure_blob_item_value(item, "resource_type", "") or "").lower() + return ( + type(item).__name__ == "BlobPrefix" + or _azure_blob_item_name(item).endswith("/") + or metadata_is_folder + or resource_type == "directory" + ) + + +def _azure_blob_item_size(item: Any) -> int: + try: + return int(_azure_blob_item_value(item, "size", 0) or 0) + except (TypeError, ValueError): + return 0 + + +def _azure_blob_item_modified_at(item: Any) -> Optional[str]: + return _format_smb_modified_at(_azure_blob_item_value(item, "last_modified")) + + +def _azure_blob_item_change_token(item: Any) -> Optional[str]: + change_token = _azure_blob_item_value(item, "etag") + return str(change_token) if change_token is not None else None + + +def _join_azure_blob_path(parent_path: str, child_path: str) -> str: + parent = str(parent_path or "").strip("/") + child = str(child_path or "").strip("/") + if not parent: + return child + if not child: + return parent + return f"{parent}/{child}" + + +def _relative_azure_blob_path(blob_prefix: str, blob_name: str) -> str: + normalized_prefix = str(blob_prefix or "").strip("/") + normalized_blob_name = str(blob_name or "").strip("/") + if normalized_prefix and normalized_blob_name.lower().startswith(normalized_prefix.lower()): + return normalized_blob_name[len(normalized_prefix):].lstrip("/") + return normalized_blob_name + + +def _build_azure_blob_url(account_url: str, container_name: str, blob_name: str) -> str: + encoded_container = quote(container_name, safe="") + encoded_blob_name = "/".join(quote(part, safe="") for part in str(blob_name or "").split("/")) + return f"{account_url.rstrip('/')}/{encoded_container}/{encoded_blob_name}" + + +def _browse_azure_blob_path(source: Dict[str, Any], browse_path: str) -> List[Dict[str, Any]]: + container_client = _get_azure_blob_container_client(source) + blob_prefix = source.get("connection", {}).get("blob_prefix", "") + full_path = _join_azure_blob_path(blob_prefix, browse_path) + browse_prefix = f"{full_path.rstrip('/')}/" if full_path else "" + entries = [] + for entry in container_client.walk_blobs(name_starts_with=browse_prefix, delimiter="/"): + entry_name = _azure_blob_item_name(entry) + if not entry_name: + continue + relative_path = _relative_azure_blob_path(blob_prefix, entry_name).rstrip("/") + display_name = relative_path.split("/")[-1] + if not display_name: + continue + entries.append( + { + "name": display_name, + "path": relative_path, + "type": "folder" if _azure_blob_item_is_folder(entry) else "file", + "size": _azure_blob_item_size(entry), + "modified_at": _azure_blob_item_modified_at(entry), + } + ) + if len(entries) >= 100: + break + return entries + + +def _list_azure_blobs(source: Dict[str, Any], config: Dict[str, Any]) -> List[Dict[str, Any]]: + container_client = _get_azure_blob_container_client(source) + connection = source.get("connection") or {} + account_url = connection.get("account_url", "") + container_name = connection.get("container_name", "") + blob_prefix = connection.get("blob_prefix", "") + selected_paths = connection.get("selected_paths") or [""] + recursive_enabled = bool(source.get("recursive", True) and config.get("file_sync_allow_recursive_sources", True)) + max_items = config["file_sync_max_files_per_run"] * 2 + remote_files = [] + seen_blob_names = set() + + for selected_path in selected_paths: + selection_prefix = _join_azure_blob_path(blob_prefix, selected_path) + for blob in container_client.list_blobs(name_starts_with=selection_prefix or None): + blob_name = _azure_blob_item_name(blob) + if not blob_name or _azure_blob_item_is_folder(blob) or blob_name in seen_blob_names: + continue + if selected_path and blob_name != selection_prefix and not blob_name.startswith(f"{selection_prefix.rstrip('/')}/"): + continue + + selection_remainder = blob_name[len(selection_prefix):].lstrip("/") if selection_prefix else blob_name + if not recursive_enabled and "/" in selection_remainder: + continue + + seen_blob_names.add(blob_name) + remote_files.append( + { + "remote_path": _build_azure_blob_url(account_url, container_name, blob_name), + "relative_path": _relative_azure_blob_path(blob_prefix, blob_name), + "file_name": blob_name.rstrip("/").split("/")[-1], + "size": _azure_blob_item_size(blob), + "modified_at": _azure_blob_item_modified_at(blob), + "remote_change_token": _azure_blob_item_change_token(blob), + "azure_blob_name": blob_name, + } + ) + if len(remote_files) >= max_items: + return remote_files + return remote_files + + def _join_azure_file_path(parent_path: str, child_name: str) -> str: parent = str(parent_path or "").strip("/") child = str(child_name or "").strip("/") @@ -2354,6 +3385,23 @@ def _stage_azure_files_file(source: Dict[str, Any], remote_file: Dict[str, Any]) return temp_file.name, sha256_hash.hexdigest() +def _stage_azure_blob_file(source: Dict[str, Any], remote_file: Dict[str, Any]) -> Tuple[str, str]: + container_client = _get_azure_blob_container_client(source) + blob_name = remote_file.get("azure_blob_name") or remote_file.get("relative_path") or remote_file.get("file_name") + blob_client = container_client.get_blob_client(blob_name) + suffix = os.path.splitext(remote_file.get("file_name") or "")[1] or ".bin" + temp_dir = "/sc-temp-files" if os.path.exists("/sc-temp-files") else None + sha256_hash = hashlib.sha256() + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix, dir=temp_dir) as temp_file: + downloader = blob_client.download_blob() + for chunk in downloader.chunks(): + if not chunk: + continue + temp_file.write(chunk) + sha256_hash.update(chunk) + return temp_file.name, sha256_hash.hexdigest() + + def _derive_tags_for_remote_file(source: Dict[str, Any], remote_file: Dict[str, Any]) -> List[str]: filters = source.get("filters") or {} tags = list(filters.get("fixed_tags") or []) @@ -2600,7 +3648,7 @@ def _upsert_failed_item( "remote_change_token": remote_file.get("remote_change_token"), "remote_web_url": remote_file.get("web_url"), "status": "failed", - "error_message": str(error)[:1000], + "error_message": FILE_SYNC_PUBLIC_ITEM_ERROR_MESSAGE, "last_sync_run_id": run_id, "last_sync_action": "failed", "last_seen_at": now_iso, @@ -2625,8 +3673,18 @@ def _handle_remote_deletes(source: Dict[str, Any], existing_items: Dict[str, Dic counts["deleted"] = counts.get("deleted", 0) + 1 except Exception as delete_error: item["status"] = "delete_failed" - item["error_message"] = str(delete_error)[:1000] + item["error_message"] = FILE_SYNC_PUBLIC_ITEM_ERROR_MESSAGE counts["failed"] = counts.get("failed", 0) + 1 + log_event( + "[FileSync] Remote document deletion failed.", + level=logging.ERROR, + extra={ + "source_id": source.get("id"), + "document_id": item.get("document_id"), + "error": str(delete_error), + }, + exceptionTraceback=True, + ) else: item["status"] = "remote_missing" item["updated_at"] = now_iso diff --git a/application/single_app/functions_workspace_identities.py b/application/single_app/functions_workspace_identities.py index 0217dc007..237c1a9fb 100644 --- a/application/single_app/functions_workspace_identities.py +++ b/application/single_app/functions_workspace_identities.py @@ -52,7 +52,7 @@ "general": "action", } WORKSPACE_IDENTITY_USAGE_SOURCE_TYPES = { - "file_sync": ["smb", "azure_files", "onedrive", "google_drive", "google_shared_drive"], + "file_sync": ["smb", "azure_files", "azure_blob", "onedrive", "google_drive", "google_shared_drive"], "action": ["action"], } WORKSPACE_IDENTITY_USAGE_AUTH_TYPES = { diff --git a/application/single_app/route_backend_file_sync.py b/application/single_app/route_backend_file_sync.py index baba504f8..bf059c11d 100644 --- a/application/single_app/route_backend_file_sync.py +++ b/application/single_app/route_backend_file_sync.py @@ -1,7 +1,10 @@ # route_backend_file_sync.py +import logging + from flask import jsonify, request +from functions_appinsights import log_event from functions_authentication import admin_required, enabled_required, get_current_user_id, get_current_user_info, login_required, user_required from functions_file_sync import ( FILE_SYNC_MANAGER_ROLES, @@ -9,6 +12,7 @@ FILE_SYNC_SCOPE_PERSONAL, FILE_SYNC_SCOPE_PUBLIC, FILE_SYNC_SOURCE_TYPE_SMB, + FileSyncPublicValidationError, assert_public_workspace_role, browse_file_sync_source_path, create_file_sync_source, @@ -116,13 +120,27 @@ def _serialize_public_workspace_target(workspace): } def _map_exception(error): + expected_error = isinstance(error, (PermissionError, LookupError, ValueError)) + log_event( + "[FileSync] Request failed.", + level=logging.WARNING if expected_error else logging.ERROR, + extra={ + "endpoint": request.endpoint or "", + "method": request.method, + "exception_type": type(error).__name__, + "error": str(error), + }, + exceptionTraceback=not expected_error, + ) + if isinstance(error, FileSyncPublicValidationError): + return _error(error.public_message, 400) if isinstance(error, PermissionError): - return _error(str(error), 403) + return _error("You do not have permission to perform this File Sync operation.", 403) if isinstance(error, LookupError): - return _error(str(error), 404) + return _error("The requested File Sync resource was not found.", 404) if isinstance(error, ValueError): - return _error(str(error), 400) - return _error(str(error), 500) + return _error("The File Sync request could not be completed. Verify the source configuration and try again.", 400) + return _error("An unexpected error occurred while processing the File Sync request.", 500) def _list_sources(scope_type, scope_id): sources = [sanitize_file_sync_source(source) for source in list_file_sync_sources(scope_type, scope_id)] diff --git a/application/single_app/static/js/public/public_workspace.js b/application/single_app/static/js/public/public_workspace.js index 1fe52cc01..e09ff21c7 100644 --- a/application/single_app/static/js/public/public_workspace.js +++ b/application/single_app/static/js/public/public_workspace.js @@ -1075,6 +1075,7 @@ function getPublicDocumentSyncTypeConfig(doc) { const sourceTypeMap = { smb: { label: 'SMB', className: 'bg-primary text-white', title: 'Managed by File Sync from an SMB source.' }, azure_files: { label: 'Azure Files', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Files.' }, + azure_blob: { label: 'Azure Blob', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Blob Storage.' }, m365sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sharepoint: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, diff --git a/application/single_app/static/js/workspace/workspace-file-sync.js b/application/single_app/static/js/workspace/workspace-file-sync.js index a0ebab633..aeb062ccc 100644 --- a/application/single_app/static/js/workspace/workspace-file-sync.js +++ b/application/single_app/static/js/workspace/workspace-file-sync.js @@ -38,6 +38,12 @@ function initializeFileSyncRoot(root) { description: 'Azure Storage file share endpoint.', enabled: true, }, + { + value: 'azure_blob', + label: 'Azure Blob Storage', + description: 'Blobs from an Azure Storage container.', + enabled: true, + }, { value: 'onedrive', label: 'OneDrive', @@ -112,6 +118,7 @@ function initializeFileSyncRoot(root) { const sourceTypeAuthTypes = { smb: ['username_password', 'anonymous'], azure_files: ['managed_identity', 'client_secret', 'connection_string'], + azure_blob: ['managed_identity', 'client_secret', 'connection_string'], onedrive: ['global_identity'], }; const authTypeLabels = { @@ -124,6 +131,12 @@ function initializeFileSyncRoot(root) { }; const getSourceTypeAuthTypes = (sourceTypeValue) => sourceTypeAuthTypes[sourceTypeValue || 'smb'] || ['username_password', 'anonymous']; const formatAuthType = (authType) => authTypeLabels[authType] || String(authType || '').replace(/_/g, ' '); + const formatSourceAuthType = (sourceTypeValue, authType) => { + if (sourceTypeValue === 'azure_blob' && authType === 'connection_string') { + return 'Blob connection string or SAS'; + } + return formatAuthType(authType); + }; const identitySupportsFileSync = (identity, sourceTypeValue) => { const usageContexts = Array.isArray(identity.usage_contexts) && identity.usage_contexts.length > 0 ? identity.usage_contexts @@ -860,6 +873,8 @@ function initializeFileSyncRoot(root) { accountUrl: source.connection?.account_url || source.connection?.share_url || '', shareName: source.connection?.share_name || '', directoryPath: source.connection?.directory_path || '', + containerName: source.connection?.container_name || '', + blobPrefix: source.connection?.blob_prefix || '', selectedPaths: source.connection?.selected_paths || [], identityId: source.identity_id || '', authType: source.credentials?.auth_type || 'username_password', @@ -989,6 +1004,9 @@ function initializeFileSyncRoot(root) { const accountUrlField = buildLabeledInput('file-sync-account-url', 'File service URL', 'url', values.accountUrl); const shareNameField = buildLabeledInput('file-sync-share-name', 'Share name', 'text', values.shareName); const directoryPathField = buildLabeledInput('file-sync-directory-path', 'Directory path', 'text', values.directoryPath); + const blobAccountUrlField = buildLabeledInput('file-sync-blob-account-url', 'Blob service URL or account name', 'text', values.accountUrl); + const containerNameField = buildLabeledInput('file-sync-container-name', 'Container name', 'text', values.containerName); + const blobPrefixField = buildLabeledInput('file-sync-blob-prefix', 'Blob prefix', 'text', values.blobPrefix); const enabledField = buildCheckbox('file-sync-enabled', 'Enabled', values.enabled); const recursiveField = buildCheckbox( 'file-sync-recursive', @@ -1000,6 +1018,8 @@ function initializeFileSyncRoot(root) { formGrid.appendChild(nameField.wrapper); if (selectedSourceType === 'azure_files') { appendChildren(formGrid, [accountUrlField.wrapper, shareNameField.wrapper, directoryPathField.wrapper]); + } else if (selectedSourceType === 'azure_blob') { + appendChildren(formGrid, [blobAccountUrlField.wrapper, containerNameField.wrapper, blobPrefixField.wrapper]); } else if (selectedSourceType === 'onedrive') { const oneDriveSummary = createElement('div', { className: 'col-12 text-muted small', text: 'OneDrive sync uses an admin-managed global connector identity with Microsoft Graph application permissions.' }); formGrid.appendChild(oneDriveSummary); @@ -1028,13 +1048,19 @@ function initializeFileSyncRoot(root) { className: 'alert alert-info py-2 mb-0', text: 'OneDrive uses the admin-managed global File Sync connector identity. Personal users choose what to sync, not tenant credentials.', }); + const azureBlobCredentialNotice = createElement('div', { + className: 'alert alert-info py-2 mb-3', + text: 'Each source syncs one container. A container SAS needs Read and List only. Extra permissions are accepted but are not needed and will be flagged. Account SAS and storage account keys work but grant broader access. To sync multiple containers, create one source per container. Managed identity is recommended. Secret credentials use Azure Key Vault when enabled; otherwise they use the existing File Sync source or identity credential storage.', + }); + const azureBlobCredentialAssessment = createElement('div', { className: 'd-none' }); + renderCredentialMetadata(azureBlobCredentialAssessment, source?.credential_metadata, false); const localCredentialsWrapper = createElement('div', { className: 'row g-3' }); const authTypeWrapper = createElement('div', { className: 'col-md-6' }); const authTypeLabel = createElement('label', { className: 'form-label', text: 'Authentication', attributes: { for: 'file-sync-auth-type' } }); const authTypeSelect = createElement('select', { className: 'form-select', attributes: { id: 'file-sync-auth-type' } }); getSourceTypeAuthTypes(selectedSourceType).forEach((value) => { - const option = createElement('option', { text: formatAuthType(value), attributes: { value } }); + const option = createElement('option', { text: formatSourceAuthType(selectedSourceType, value), attributes: { value } }); option.selected = values.authType === value; authTypeSelect.appendChild(option); }); @@ -1043,8 +1069,65 @@ function initializeFileSyncRoot(root) { const domainField = buildLabeledInput('file-sync-domain', 'Domain', 'text', values.domain); const clientIdField = buildLabeledInput('file-sync-client-id', 'Client ID', 'text', values.clientId); const passwordField = buildLabeledInput('file-sync-password', source?.credentials?.password_stored || source?.credentials?.secret_stored ? 'Secret (stored)' : 'Secret', 'password', values.password || values.secret); + const blobCredentialHelp = createElement('div', { + className: 'form-text d-none', + text: 'Paste a storage connection string, full container SAS URL, or standalone SAS token. A standalone token uses the Blob service URL and container entered above.', + }); + passwordField.wrapper.appendChild(blobCredentialHelp); appendChildren(localCredentialsWrapper, [authTypeWrapper, usernameField.wrapper, domainField.wrapper, clientIdField.wrapper, passwordField.wrapper]); + let autoSourceNameEnabled = !source; + let lastDerivedSourceName = ''; + const deriveSourceNameFromContainer = () => { + if (selectedSourceType !== 'azure_blob' || !autoSourceNameEnabled) { + return; + } + const containerName = containerNameField.input.value.trim(); + if (!containerName) { + return; + } + lastDerivedSourceName = `${containerName} Blob Sync`; + nameField.input.value = lastDerivedSourceName; + }; + const hydrateBlobFieldsFromSasUrl = (credentialValue) => { + const normalizedValue = String(credentialValue || '').trim(); + if (selectedSourceType !== 'azure_blob' || !normalizedValue.toLowerCase().startsWith('https://')) { + return; + } + try { + const parsedUrl = new URL(normalizedValue); + if (!parsedUrl.search || !parsedUrl.hostname.toLowerCase().includes('.blob.')) { + return; + } + if (authTypeSelect.value !== 'connection_string') { + authTypeSelect.value = 'connection_string'; + updateCredentialVisibility(); + } + const pathParts = parsedUrl.pathname.split('/').filter(Boolean).map((pathPart) => decodeURIComponent(pathPart)); + blobAccountUrlField.input.value = `${parsedUrl.protocol}//${parsedUrl.host}`; + if (pathParts.length > 0) { + containerNameField.input.value = pathParts[0]; + deriveSourceNameFromContainer(); + } + } catch (error) { + // Server validation provides the reviewed error message for malformed SAS URLs. + } + }; + nameField.input.addEventListener('input', () => { + if (nameField.input.value.trim() !== lastDerivedSourceName) { + autoSourceNameEnabled = false; + } + }); + containerNameField.input.addEventListener('input', deriveSourceNameFromContainer); + passwordField.input.addEventListener('input', () => hydrateBlobFieldsFromSasUrl(passwordField.input.value)); + blobAccountUrlField.input.addEventListener('input', () => { + const accountValue = blobAccountUrlField.input.value.trim(); + if (accountValue.toLowerCase().startsWith('https://') && accountValue.includes('?')) { + passwordField.input.value = accountValue; + hydrateBlobFieldsFromSasUrl(accountValue); + } + }); + const updateCredentialVisibility = () => { const usingIdentity = Boolean(identitySelect.value); localCredentialsWrapper.classList.toggle('d-none', usingIdentity); @@ -1056,6 +1139,7 @@ function initializeFileSyncRoot(root) { domainField.wrapper.classList.toggle('d-none', !usesUsernamePassword); clientIdField.wrapper.classList.toggle('d-none', !usesClientSecret); passwordField.wrapper.classList.toggle('d-none', ['anonymous', 'managed_identity', 'global_identity'].includes(authType)); + blobCredentialHelp.classList.toggle('d-none', !(selectedSourceType === 'azure_blob' && authType === 'connection_string' && !usingIdentity)); const passwordLabel = passwordField.wrapper.querySelector('label'); if (passwordLabel) { if (usesUsernamePassword) { @@ -1063,7 +1147,13 @@ function initializeFileSyncRoot(root) { } else if (usesClientSecret) { passwordLabel.textContent = source?.credentials?.secret_stored ? 'Client secret (stored)' : 'Client secret'; } else if (authType === 'connection_string') { - passwordLabel.textContent = source?.credentials?.secret_stored ? 'Connection string (stored)' : 'Connection string'; + if (selectedSourceType === 'azure_blob') { + passwordLabel.textContent = source?.credentials?.secret_stored + ? 'Blob connection string or SAS (stored)' + : 'Blob connection string, SAS URL, or SAS token'; + } else { + passwordLabel.textContent = source?.credentials?.secret_stored ? 'Connection string (stored)' : 'Connection string'; + } } else { passwordLabel.textContent = source?.credentials?.secret_stored ? 'Secret (stored)' : 'Secret'; } @@ -1119,10 +1209,17 @@ function initializeFileSyncRoot(root) { intervalField.wrapper.classList.toggle('d-none', !scheduleField.input.checked); }); + let identityAndAuthenticationChildren = [identityWrapper, localCredentialsWrapper]; + if (selectedSourceType === 'onedrive') { + identityAndAuthenticationChildren = [globalConnectorNotice]; + } else if (selectedSourceType === 'azure_blob') { + identityAndAuthenticationChildren = [azureBlobCredentialNotice, azureBlobCredentialAssessment, identityWrapper, localCredentialsWrapper]; + } + appendChildren(content, [ typeSummary, createConfigCard('General', [formGrid, generalSwitches]), - createConfigCard('Identity and Authentication', selectedSourceType === 'onedrive' ? [globalConnectorNotice] : [identityWrapper, localCredentialsWrapper]), + createConfigCard('Identity and Authentication', identityAndAuthenticationChildren), createConfigCard('Selection, Subfolders, and Filters', [recursiveField.wrapper, selectedPathControl.wrapper, patternControl.wrapper, extensionControl.wrapper, folderGrid]), createConfigCard('Tags', [tagsField.wrapper]), createConfigCard('Sync Schedule', [scheduleField.wrapper, intervalField.wrapper]), @@ -1153,22 +1250,40 @@ function initializeFileSyncRoot(root) { credentials.client_secret = ''; credentials.connection_string = ''; } + if (identitySelect.value) { + credentials.password = ''; + credentials.secret = ''; + credentials.client_secret = ''; + credentials.connection_string = ''; + credentials.identity = ''; + credentials.client_id = ''; + } const selectedPaths = selectedPathControl.getValues(); - const connection = selectedSourceType === 'azure_files' - ? { + let connection; + if (selectedSourceType === 'azure_files') { + connection = { account_url: accountUrlField.input.value.trim(), share_name: shareNameField.input.value.trim(), directory_path: directoryPathField.input.value.trim(), selected_paths: selectedPaths, - } - : selectedSourceType === 'onedrive' - ? { - selected_paths: selectedPaths, - } - : { - unc_path: uncField.input.value.trim(), - selected_paths: selectedPaths, - }; + }; + } else if (selectedSourceType === 'azure_blob') { + connection = { + account_url: blobAccountUrlField.input.value.trim(), + container_name: containerNameField.input.value.trim(), + blob_prefix: blobPrefixField.input.value.trim(), + selected_paths: selectedPaths, + }; + } else if (selectedSourceType === 'onedrive') { + connection = { + selected_paths: selectedPaths, + }; + } else { + connection = { + unc_path: uncField.input.value.trim(), + selected_paths: selectedPaths, + }; + } return { name: nameField.input.value.trim(), source_type: selectedSourceType, @@ -1217,6 +1332,9 @@ function initializeFileSyncRoot(root) { body: JSON.stringify(buildPayload()), }); const connection = payload.connection || {}; + if (selectedSourceType === 'azure_blob') { + renderCredentialMetadata(azureBlobCredentialAssessment, connection.credential_metadata, false); + } showModalStatus(`Connection OK. Checked ${connection.entries_checked || 0} top-level item(s).`, 'success'); } catch (error) { showModalStatus(error.message, 'danger'); @@ -1520,6 +1638,108 @@ function initializeFileSyncRoot(root) { return date.toLocaleString(); }; + const sasPermissionLabels = { + r: 'Read', + l: 'List', + a: 'Add', + c: 'Create', + w: 'Write', + d: 'Delete', + x: 'Delete version', + t: 'Tags', + m: 'Move', + e: 'Execute', + o: 'Ownership', + p: 'Permissions', + i: 'Immutability policy', + y: 'Permanent delete', + }; + + const getCredentialMetadataLines = (metadata = {}) => { + if (!metadata || !metadata.credential_type) { + return []; + } + const lines = []; + const credentialLabels = { + account_key: 'Storage account key', + client_secret: 'Service principal', + connection_string: 'Connection string', + managed_identity: 'Managed identity', + }; + const scopeLabels = { + account: 'Account SAS', + blob: 'Blob SAS', + container: 'Container SAS', + unknown: 'SAS', + }; + const credentialLabel = metadata.credential_type === 'sas' + ? (scopeLabels[metadata.sas_scope] || 'SAS') + : (credentialLabels[metadata.credential_type] || formatAuthType(metadata.credential_type)); + lines.push({ text: credentialLabel, type: 'summary' }); + + const permissions = String(metadata.permissions || '') + .split('') + .filter((permission, index, allPermissions) => permission && allPermissions.indexOf(permission) === index) + .map((permission) => sasPermissionLabels[permission] || permission); + if (permissions.length > 0) { + lines.push({ text: `Permissions: ${permissions.join(', ')}`, type: 'detail' }); + } + if (metadata.starts_at) { + lines.push({ text: `Valid from ${formatDate(metadata.starts_at)}`, type: 'detail' }); + } + if (metadata.ip_range) { + lines.push({ text: `IP restriction: ${metadata.ip_range}`, type: 'warning' }); + } + + if (metadata.expires_at) { + const expiresAt = new Date(metadata.expires_at); + if (!Number.isNaN(expiresAt.getTime())) { + const remainingMilliseconds = expiresAt.getTime() - Date.now(); + const hoursRemaining = Math.ceil(remainingMilliseconds / 3600000); + const daysRemaining = Math.ceil(remainingMilliseconds / 86400000); + let remainingText = 'expired'; + if (remainingMilliseconds >= 0 && hoursRemaining <= 48) { + remainingText = `${hoursRemaining} hour${hoursRemaining === 1 ? '' : 's'} remaining`; + } else if (remainingMilliseconds >= 0) { + remainingText = `${daysRemaining} day${daysRemaining === 1 ? '' : 's'} remaining`; + } + lines.push({ + text: `Expires ${formatDate(metadata.expires_at)} (${remainingText})`, + type: remainingMilliseconds < 0 || daysRemaining <= 30 ? 'warning' : 'detail', + }); + } + } else if (metadata.stored_access_policy) { + lines.push({ text: 'Expiry is controlled by a stored access policy.', type: 'warning' }); + } else if (metadata.credential_type === 'account_key') { + lines.push({ text: 'This credential does not expire automatically.', type: 'warning' }); + } + + (metadata.warnings || []).forEach((warning) => { + lines.push({ text: String(warning || ''), type: 'warning' }); + }); + return lines.filter((line) => line.text); + }; + + const renderCredentialMetadata = (container, metadata, compact = false) => { + if (!container) { + return; + } + container.replaceChildren(); + const lines = getCredentialMetadataLines(metadata); + container.className = lines.length === 0 + ? 'd-none' + : (compact ? 'small mt-1' : 'alert alert-light border py-2 mb-3'); + lines.forEach((line) => { + const lineElement = createElement('div', { + className: line.type === 'warning' + ? 'text-warning-emphasis' + : (line.type === 'summary' ? 'fw-semibold' : 'text-muted'), + text: line.text, + }); + container.appendChild(lineElement); + }); + }; + const getSourceConnectionLabel = (source = {}) => { const connection = source.connection || {}; const selectedPaths = Array.isArray(connection.selected_paths) ? connection.selected_paths : []; @@ -1533,6 +1753,12 @@ function initializeFileSyncRoot(root) { .join('/'); return `${baseLabel}${selectedPathLabel}`; } + if ((source.source_type || 'smb') === 'azure_blob') { + const baseLabel = [connection.account_url, connection.container_name, connection.blob_prefix] + .filter(Boolean) + .join('/'); + return `${baseLabel}${selectedPathLabel}`; + } return `${connection.unc_path || ''}${selectedPathLabel}`; }; @@ -1646,7 +1872,9 @@ function initializeFileSyncRoot(root) { const nameText = createElement('div', { className: 'fw-semibold', text: source.name || 'File Sync Source' }); const pathText = createElement('div', { className: 'small text-muted', text: getSourceConnectionLabel(source) }); const recursionText = createElement('div', { className: 'small text-muted', text: source.recursive === false ? 'Top folder only' : 'Includes subfolders' }); - appendChildren(nameCell, [nameText, pathText, recursionText]); + const credentialMetadata = createElement('div', { className: 'd-none' }); + renderCredentialMetadata(credentialMetadata, source.credential_metadata, true); + appendChildren(nameCell, [nameText, pathText, recursionText, credentialMetadata]); const statusText = source.enabled ? 'Enabled' : 'Disabled'; const typeCell = createElement('td'); diff --git a/application/single_app/static/js/workspace/workspace-identities.js b/application/single_app/static/js/workspace/workspace-identities.js index 5d47ff735..96dfd0420 100644 --- a/application/single_app/static/js/workspace/workspace-identities.js +++ b/application/single_app/static/js/workspace/workspace-identities.js @@ -18,9 +18,9 @@ function initializeWorkspaceIdentityRoot(root) { const capabilityConfigs = { file_sync: { label: 'File Sync', - help: 'Use this identity for File Sync sources, including SMB, Azure Files, and admin-approved cloud drive connectors.', + help: 'Use this identity for File Sync sources, including SMB, Azure Files, Azure Blob Storage, and admin-approved cloud drive connectors.', provider: 'smb', - sourceTypes: ['smb', 'azure_files', 'onedrive', 'google_drive', 'google_shared_drive'], + sourceTypes: ['smb', 'azure_files', 'azure_blob', 'onedrive', 'google_drive', 'google_shared_drive'], usageContexts: ['file_sync'], authTypes: ['username_password', 'anonymous', 'managed_identity', 'client_secret', 'connection_string'], }, diff --git a/application/single_app/static/js/workspace/workspace-utils.js b/application/single_app/static/js/workspace/workspace-utils.js index b820ed437..150d6ec95 100644 --- a/application/single_app/static/js/workspace/workspace-utils.js +++ b/application/single_app/static/js/workspace/workspace-utils.js @@ -33,6 +33,7 @@ function getDocumentSyncTypeConfig(doc) { const sourceTypeMap = { smb: { label: 'SMB', className: 'bg-primary text-white', title: 'Managed by File Sync from an SMB source.' }, azure_files: { label: 'Azure Files', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Files.' }, + azure_blob: { label: 'Azure Blob', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Blob Storage.' }, m365sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sharepoint: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, diff --git a/application/single_app/support_menu_config.py b/application/single_app/support_menu_config.py index d84934bbc..8c7cd1688 100644 --- a/application/single_app/support_menu_config.py +++ b/application/single_app/support_menu_config.py @@ -186,12 +186,12 @@ def _latest_feature_card(feature_id, title, icon, summary, details, why, guidanc ), _latest_feature_card( 'release_250_file_sync', - 'File Sync for SMB and Azure Files', + 'File Sync for Storage Sources', 'bi-arrow-repeat', - 'File Sync can bring SMB share and Azure Files content into workspaces, with reusable identities and workflow triggers for automated refreshes.', + 'File Sync can bring SMB, Azure Files, and Azure Blob Storage content into workspaces, with reusable identities and workflow triggers for automated refreshes.', 'Users can configure sync sources where enabled, use identities for credentials, review synced-document badges, and connect sync sources to workflows that run before or after file changes. Additional sync providers are planned for future releases.', 'This matters because workspace documents can stay closer to authoritative file shares instead of depending on repeated manual uploads.', - ['Use Workspace > Sync to configure SMB or Azure Files sources when admins enable File Sync.', 'Use Workspace > Identities to reuse credentials for sync sources and actions.', 'Use workflows with File Sync triggers when analysis should run after synced content changes.'], + ['Use Workspace > Sync to configure SMB, Azure Files, or Azure Blob Storage sources when admins enable File Sync.', 'Use Workspace > Identities to reuse credentials for sync sources and actions.', 'Use workflows with File Sync triggers when analysis should run after synced content changes.'], actions=[{'label': 'Open Workspace Sync', 'description': 'Review sync sources and run history.', 'href': '/workspace?feature_action=file_sync', 'icon': 'bi-arrow-repeat', 'requires_settings': ['enable_user_workspace']}, {'label': 'Open Workspace Identities', 'description': 'Review reusable identities for sync and actions.', 'href': '/workspace#identities-tab', 'icon': 'bi-person-badge', 'requires_settings': ['enable_user_workspace']}], image_label='File Sync', ), @@ -525,10 +525,10 @@ def _latest_feature_card(feature_id, title, icon, summary, details, why, guidanc 'admin_release_250_file_sync', 'File Sync Administration', 'bi-arrow-repeat', - 'Admins can enable File Sync, choose SMB and Azure Files source types, configure scope gates, limits, connector identities, and workflow integration.', - 'File Sync administration controls which workspaces can sync files, which source types are available, whether app roles are required, and how identities are used for SMB and Azure Files credentials.', + 'Admins can enable File Sync, choose SMB, Azure Files, and Azure Blob Storage source types, configure scope gates, limits, connector identities, and workflow integration.', + 'File Sync administration controls which workspaces can sync files, which source types are available, whether app roles are required, and how identities are used for storage credentials.', 'This matters because synced ingestion needs tenant-level rollout controls before users connect shared file sources.', - ['Screenshot idea: capture File Sync source-type availability and workspace scope controls.', 'Show SMB and Azure Files controls while noting more providers are planned.', 'Call out workflow triggers that can run when File Sync detects changes.'], + ['Screenshot idea: capture File Sync source-type availability and workspace scope controls.', 'Show SMB, Azure Files, and Azure Blob Storage controls while noting more providers are planned.', 'Call out workflow triggers that can run when File Sync detects changes.'], actions=[ {'label': 'Open File Sync', 'description': 'Review File Sync administration.', 'href': '#file-sync', 'admin_tab': '#file-sync', 'icon': 'bi-arrow-repeat'}, {'label': 'Open Global Identities', 'description': 'Review connector identities used by sync sources.', 'href': '#global-workspace-identities-root', 'admin_tab': '#workspace-identities', 'admin_section': 'global-workspace-identities-root', 'icon': 'bi-person-badge'}, diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index 8af4d70d8..fc7af65e8 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -7312,6 +7312,13 @@
Visible Source Types
Available now.
+
+
+ + +
Available now.
+
+
diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index c75f12b87..e033155f4 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -3798,6 +3798,7 @@ const sourceTypeMap = { smb: { label: 'SMB', className: 'bg-primary text-white', title: 'Managed by File Sync from an SMB source.' }, azure_files: { label: 'Azure Files', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Files.' }, + azure_blob: { label: 'Azure Blob', className: 'bg-info text-dark', title: 'Managed by File Sync from Azure Blob Storage.' }, m365sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sp: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, m365_sharepoint: { label: 'M365SP', className: 'bg-info text-dark', title: 'Managed by File Sync from Microsoft 365 SharePoint.' }, diff --git a/docs/explanation/features/AZURE_BLOB_STORAGE_FILE_SYNC.md b/docs/explanation/features/AZURE_BLOB_STORAGE_FILE_SYNC.md new file mode 100644 index 000000000..c3909171a --- /dev/null +++ b/docs/explanation/features/AZURE_BLOB_STORAGE_FILE_SYNC.md @@ -0,0 +1,111 @@ +# Azure Blob Storage File Sync + +Implemented in version: **0.250.067** +Security hardening in version: **0.250.068** +Container SAS support and credential visibility in version: **0.250.069** +Non-Key-Vault compatibility and List/Read connection validation in version: **0.250.070** + +Related issue: [#1027](https://github.com/microsoft/simplechat/issues/1027) + +## Overview + +Azure Blob Storage is available as an admin-controlled File Sync source for personal, group, and public workspaces. Authorized workspace managers can connect a storage account and container, optionally limit synchronization to a blob-name prefix or selected virtual paths, and use the existing File Sync schedules, filters, tags, delete policies, run history, and workflow triggers. + +## Dependencies + +- File Sync and the target workspace scope must be enabled in Admin Settings. +- Redis Cache must be configured before File Sync runs are effective. +- The existing `azure-storage-blob==12.24.1` dependency provides Blob service access. +- Managed identity requires an Azure Storage data-plane role such as **Storage Blob Data Reader** on the target account or container. +- Client-secret and Blob credential authentication use Azure Key Vault when secret storage is enabled. When it is disabled, they use the existing File Sync source or reusable-identity credential persistence and remain redacted from browser responses. +- The application version was updated in `application/single_app/config.py` to `0.250.070` for non-Key-Vault compatibility and List/Read-only connection validation. + +## Technical Specifications + +### Architecture + +- The source type is stored as `source_type: "azure_blob"` in the existing personal, group, or public File Sync source container. +- Connection data contains `account_url`, `container_name`, optional `blob_prefix`, and optional `selected_paths`. +- Storage account names are expanded to `https://.blob.core.windows.net`. Full service and container URLs are accepted only when their host is a valid Azure Blob endpoint in the supported public, US Government, China, or Germany cloud suffixes. +- Blob service URLs and connection-string endpoints are validated again immediately before SDK client construction. Loopback, link-local, arbitrary custom, development-storage, userinfo, nonstandard-port, query-string, and fragment endpoints are rejected. +- Blob names are presented as virtual folders in the existing source browser. Directory-marker blobs are ignored during synchronization. +- Blob ETags, last-modified timestamps, and content lengths are translated into the shared remote-file contract for change detection. +- Downloads are streamed in chunks into the existing temporary-file ingestion path, preserving file limits, supported-format checks, document processing, tags, and source attribution. +- Existing File Sync ownership, role, scope-assignment, admin-management, scheduling, and remote-delete checks apply without new routes or authorization paths. + +### Authentication + +- **Managed identity** is the recommended authentication mode. +- **Service principal** authentication uses tenant ID, client ID, and a client secret. Key Vault is used when configured. +- **Blob credential** authentication accepts a storage connection string, full container SAS URL, or standalone SAS token. Key Vault is used when configured. +- The Blob credential field accepts three Azure-provided formats: a storage connection string, a full container SAS URL, or a standalone SAS token. A standalone token uses the Blob service URL and container entered in the source form. +- **Container SAS** is the recommended secret-based option for a source that syncs one container. It must grant **Read** (`r`) and **List** (`l`) and require HTTPS. Write, Create, Add, Delete, tag, move, ownership, and policy permissions are not required; they are accepted but shown as least-privilege warnings. +- **Account SAS** and storage account keys are accepted for compatibility but are identified as broader than a single-container source needs. Account SAS must include the Blob service, Container and Object resource types, and Read and List permissions. +- **Blob/object SAS** is rejected because File Sync must list the selected container before reading blobs. +- Reusable workspace identities can declare Azure Blob Storage support for personal, group, or public scopes. +- Unsaved connection tests use credentials in memory. Saving stores secret material through Key Vault when enabled or through existing File Sync credential persistence when Key Vault is disabled. +- Detailed SDK and network failures are written through sanitized server logging. API responses, run history, activity records, and failed item records use fixed public messages instead of returning exception text. + +Each File Sync source targets one container. To synchronize multiple containers, create one source per container. Account-wide container discovery is not performed by this feature. + +SAS scope, non-secret permission letters, start time, expiry, HTTPS status, optional IP range, and least-privilege warnings are stored as non-secret metadata. The source list shows the credential scope, named permissions, exact expiry, and days remaining without exposing the SAS token. + +### Admin Configuration + +1. Open **Admin Settings > File Sync**. +2. Enable File Sync globally and enable the required personal, group, or public scopes. +3. Under **Visible Source Types**, turn on **Azure Blob Storage**. +4. Configure workspace assignments or admin-only source management when required. +5. Grant the selected identity least-privilege Blob data access to the target account or container. + +Azure Blob Storage is opt-in. Existing installations retain the SMB and Azure Files default source-type visibility until an admin enables Blob Storage and saves settings. + +## Usage Instructions + +1. Open the **Sync** tab in an enabled personal, group, or public workspace. +2. Select **Add Source**, then choose **Azure Blob Storage**. +3. Enter the Blob service URL or storage account name, container name, and optional blob prefix. + - Alternatively, paste the full container SAS URL into either **Blob service URL or account name** or **Blob connection string, SAS URL, or SAS token**. The form derives the non-secret Blob service URL, container, and default source name, switches to Blob credential authentication, and stores the SAS through the configured File Sync secret-storage path. +4. Select managed identity or a compatible reusable identity. Source-local service-principal and Blob credentials work with or without Key Vault; Key Vault is preferred when available. + - For a container SAS connection string, use `BlobEndpoint=https://.blob.core.windows.net/` plus `SharedAccessSignature=`. + - A full SAS URL uses `https://.blob.core.windows.net/?`. A standalone token begins with `?` or the first SAS parameter and requires the account/container fields above. + - Select **Read** and **List**, set **Allowed protocols** to HTTPS only, and choose an expiry that covers the intended schedule and rotation window. +5. Use **Browse** or **Add Path** to narrow the source, then configure recursion, filters, tags, schedule, and remote-delete behavior. +6. Test the connection before saving, then run the source manually or on its schedule. + +## File Structure + +- `application/single_app/functions_file_sync.py`: source registration, validation, authentication, connection test, browse, list, metadata, and streamed download adapter. +- `application/single_app/functions_workspace_identities.py`: reusable identity source-type compatibility. +- `application/single_app/templates/admin_settings.html`: admin source-type visibility switch. +- `application/single_app/static/js/workspace/workspace-file-sync.js`: shared source workflow for all workspace scopes. +- `application/single_app/static/js/workspace/workspace-utils.js`: personal workspace source badge. +- `application/single_app/templates/group_workspaces.html`: group workspace source badge. +- `application/single_app/static/js/public/public_workspace.js`: public workspace source badge. +- `functional_tests/test_file_sync_azure_blob_storage.py`: connector behavior and all-scope wiring coverage. +- `ui_tests/test_admin_file_sync_settings_ui.py`: admin switch coverage. +- `ui_tests/test_workspace_file_sync_ui.py`: source workflow coverage. + +## Testing and Validation + +The functional test executes account and URL normalization, container validation, optional Key Vault and non-Key-Vault secret persistence, virtual-path browsing, recursive and non-recursive listing, ETag metadata mapping, streamed downloads, and SHA-256 generation with fake Blob SDK clients. It also verifies the shared source workflow is connected to personal, group, and public workspace roots. + +Security coverage rejects non-Azure and internal endpoints in direct URLs and connection strings, verifies HTTPS-only Azure cloud suffixes, and confirms backend exception details never cross API, run-history, activity, or item-response boundaries. + +Container SAS coverage validates endpoint/container matching, required Read and List permissions, extra-permission warnings, account-SAS breadth warnings, object-SAS rejection, expiry and start times, stored access policies, non-secret metadata serialization, and direct container-client construction. + +Credential-format coverage verifies storage connection strings, full SAS URLs, and standalone SAS tokens. A SAS URL pasted into the account or credential field is promoted into the configured File Sync secret-storage path; only its canonical account and container remain in source connection metadata. + +Neighboring Azure Files and OneDrive regression suites confirm that adding Azure Blob Storage does not remove existing source types or compatible identity behavior. Playwright coverage validates the admin switch and Blob-specific source fields. + +## Known Limitations + +- Blob snapshots, versions, deleted blobs, and soft-deleted records are not synchronized. +- Blob names are shown as virtual folders; Azure Blob Storage does not provide physical directories unless hierarchical namespace features are enabled. +- Without Key Vault, secret material remains in the existing File Sync source or identity credential record. Enable Key Vault or use managed identity when stronger secret isolation is required. +- Custom domains, Azurite/development storage, Azure Stack endpoints, and direct private-link hostnames are not accepted. Use the standard Azure Blob service hostname; Azure Private Endpoint DNS can resolve that hostname privately. +- SAS tokens using a stored access policy may omit explicit permissions or expiry. File Sync reports that these values are policy-controlled, validates List and Read through the connection test when blobs are available, and cannot display the policy's expiry without querying Azure control data. +- A connection test can verify List immediately. Read is verified against the first available blob; an empty source reports that Read could not be exercised. +- SAS IP restrictions must include the App Service outbound address that performs sync. An administrator should account for all active outbound addresses and deployment slots. +- Avoid setting a SAS start time exactly to the current time because clock skew can delay usability. Omit it or set it several minutes in the past. +- SAS credentials do not rotate automatically. Replace the source or reusable-identity secret before expiry; the source list shows remaining days to support this process. \ No newline at end of file diff --git a/docs/explanation/fixes/AZURE_BLOB_CONTAINER_SAS_SUPPORT_FIX.md b/docs/explanation/fixes/AZURE_BLOB_CONTAINER_SAS_SUPPORT_FIX.md new file mode 100644 index 000000000..ec63ec283 --- /dev/null +++ b/docs/explanation/fixes/AZURE_BLOB_CONTAINER_SAS_SUPPORT_FIX.md @@ -0,0 +1,66 @@ +# Azure Blob Container SAS Support Fix + +Fixed in version: **0.250.070** + +Related issue: [#1027](https://github.com/microsoft/simplechat/issues/1027) + +Related pull request: [#1088](https://github.com/microsoft/simplechat/pull/1088) + +## Issue Description + +Azure Blob File Sync accepted service/account connection strings but rejected container SAS credentials when Azure supplied either a container path in `BlobEndpoint` or a full container SAS URL. A standalone SAS token was also not recognized. Users could not see the SAS scope, required permissions, expiry, or whether a broader credential granted unnecessary access. + +## Root Cause Analysis + +The SSRF-hardened connection-string validator required `BlobEndpoint` to be a service URL with no path. A container SAS legitimately uses a canonical Azure Blob endpoint followed by one container segment. The connector also treated connection strings as opaque secrets, so it could not safely surface non-secret SAS metadata or validate least privilege before SDK calls. + +## Technical Details + +### Files Modified + +- `application/single_app/functions_file_sync.py` +- `application/single_app/static/js/workspace/workspace-file-sync.js` +- `application/single_app/config.py` +- `functional_tests/test_file_sync_azure_blob_storage.py` +- `ui_tests/test_workspace_file_sync_ui.py` +- Azure Blob File Sync feature documentation and release notes + +### Code Changes + +- Parses `BlobEndpoint` and `SharedAccessSignature` separately while preserving the SAS token only in File Sync secret credential data. Key Vault is used when enabled; existing source/identity persistence is used otherwise. +- Accepts storage connection strings, full container SAS URLs, and standalone SAS tokens. +- Derives the canonical service URL, selected container, and default source name when a full SAS URL is pasted into either Blob field; the form selects Blob credential authentication, the token is promoted to secret storage, and it is not persisted in source connection fields. +- Allows exactly one canonical container path and requires it to match the source's selected container. +- Constructs `ContainerClient` directly for SAS credentials, preventing container SAS from being treated as an account-level service client. +- Requires Read (`r`) and List (`l`) for explicit container SAS permissions. +- Requires account SAS to include Blob service plus Container and Object resource types. +- Rejects blob/object SAS because it cannot enumerate a container. +- Requires HTTPS-only SAS protocol and rejects expired or not-yet-valid tokens outside a small clock-skew tolerance. +- Accepts extra permissions but emits named least-privilege warnings. Account SAS and account keys are identified as broader than required. +- Stores and returns only non-secret metadata: credential type, scope, permission letters, validity window, expiry, HTTPS state, IP range, resource scope, and warnings. +- Shows credential scope, named permissions, exact expiry, days remaining, and warnings in the source workflow and source list. +- Supports both secret-storage modes: Key Vault when enabled, or existing File Sync source/identity credential persistence when disabled. +- Tests only the operations File Sync needs during connection validation: container listing and reading one available blob. It does not call Get Container Properties. +- Logs only non-secret Azure diagnostics for failed tests: exception type, Azure error code, HTTP status, request ID, credential scope, and permission letters. The SAS token, signed URL, and raw SDK message are excluded. +- Returns reviewed guidance for Azure permission mismatch, authentication/signature failure, SAS IP restrictions, and missing containers while leaving unknown SDK failures generic. + +## Validation + +- Container SAS with matching endpoint, HTTPS, Read, and List is accepted. +- Full container SAS URLs and standalone SAS tokens are accepted. A token with Read but without List receives explicit regeneration guidance. +- Endpoint/container mismatch, object SAS, missing Read/List, HTTP-capable SAS, expired SAS, and future-not-valid SAS are rejected. +- Extra Write/Delete permissions are accepted with a warning. +- Account SAS remains accepted with breadth and extra-permission warnings. +- SAS tokens and signatures are not included in credential metadata or browser responses. +- Connection tests validate List and, when a blob exists, Read access. +- Saving works when Key Vault is disabled; source and identity serializers continue to redact the credential from browser responses. +- Azure permission failures identify the missing Read/List requirement; authentication failures point to account/container/signature/start/expiry checks; IP failures point to App Service outbound and storage network policy configuration. + +## Operational Guidance + +- Each File Sync source synchronizes one container. Use one source per container. +- Prefer managed identity. When SAS is required, prefer a container SAS with only Read and List. +- Set expiry far enough ahead for scheduled runs and rotate the saved credential before expiry. +- Stored access policies hide effective permission and expiry details from the token; confirm the policy grants Read and List. +- Ensure SAS IP restrictions include all App Service outbound addresses that may execute sync. +- Avoid a start time equal to the current time because clock skew can make a new SAS temporarily unusable. \ No newline at end of file diff --git a/docs/explanation/fixes/AZURE_BLOB_FILE_SYNC_SSRF_AND_ERROR_DISCLOSURE_FIX.md b/docs/explanation/fixes/AZURE_BLOB_FILE_SYNC_SSRF_AND_ERROR_DISCLOSURE_FIX.md new file mode 100644 index 000000000..e4ab7a8fb --- /dev/null +++ b/docs/explanation/fixes/AZURE_BLOB_FILE_SYNC_SSRF_AND_ERROR_DISCLOSURE_FIX.md @@ -0,0 +1,51 @@ +# Azure Blob File Sync SSRF and Error Disclosure Fix + +Fixed in version: **0.250.068** + +Related issue: [#1027](https://github.com/microsoft/simplechat/issues/1027) + +Related pull request: [#1088](https://github.com/microsoft/simplechat/pull/1088) + +## Issue Description + +Azure Blob File Sync accepted any syntactically valid HTTPS account URL before creating a `BlobServiceClient`. A workspace manager could therefore provide an arbitrary host and cause the server to attempt a request outside Azure Blob Storage. File Sync routes and persisted run/item failures also exposed raw exception text to browser responses and history surfaces, which could reveal endpoints, request details, or secret-bearing SDK messages. + +## Root Cause Analysis + +The initial URL normalizer checked only the HTTPS scheme and presence of a network location. It did not require an Azure-owned Blob service hostname, reject URL credentials or nonstandard ports, or validate endpoints embedded in connection strings. The route exception mapper returned `str(error)` directly, and failed run/item records persisted the same raw exception text for later serialization. + +## Technical Details + +### Files Modified + +- `application/single_app/functions_file_sync.py` +- `application/single_app/route_backend_file_sync.py` +- `application/single_app/config.py` +- `functional_tests/test_file_sync_azure_blob_storage.py` +- Related versioned tests and Azure Blob File Sync documentation + +### Code Changes + +- Added an allowlist for Azure public, US Government, China, and Germany Storage endpoint suffixes. +- Required a 3-24 character lowercase alphanumeric storage account label and reconstructed a canonical HTTPS Blob service URL after validation. +- Rejected arbitrary hosts, IP literals, loopback/link-local targets, userinfo, explicit ports, query strings, fragments, development storage, HTTP connection strings, and unsupported endpoint suffixes. +- Revalidated stored account URLs and connection strings immediately before `BlobServiceClient` construction. +- Replaced route exception responses with fixed messages for authorization, lookup, validation, and unexpected failures. +- Replaced persisted and serialized run/item exception details with generic public messages. +- Preserved detailed exception type and text only in `log_event`, which applies the repository's structured log sanitization before writing server diagnostics. + +## Testing Approach + +`functional_tests/test_file_sync_azure_blob_storage.py` now validates accepted Azure cloud endpoints and rejects internal, arbitrary, credential-bearing, port-bearing, query-bearing, and fragment-bearing URLs. It covers safe and unsafe connection strings, checks route source for raw exception serialization, verifies run-history sanitization, and confirms raw item errors are not persisted. + +Neighboring Azure Files and OneDrive File Sync regression tests remain part of validation to ensure the source registry and shared identity behavior are unchanged. + +## Impact Analysis + +Existing sources using standard Azure Blob service hostnames continue to work. Azure Private Endpoint configurations should continue using the standard account Blob hostname with private DNS resolution. Custom domains, Azurite/development storage, Azure Stack endpoints, direct private-link hostnames, and non-Azure Blob-compatible endpoints are intentionally rejected. + +## Validation + +- Before: arbitrary HTTPS hosts could reach Blob SDK client construction, and raw backend exceptions could be returned or persisted for client-visible history. +- After: only canonical allowlisted Azure Blob endpoints reach the SDK, while client-visible failures are generic and detailed sanitized diagnostics stay server-side. +- The application version was updated to `0.250.068` for this security fix. \ No newline at end of file diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 56c79cbb7..5cc2e5fea 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,35 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.070)** + +#### Bug Fixes + +* **Azure Blob Container SAS Support and Credential Guidance** + * Added support for storage connection strings, full container SAS URLs, and standalone SAS tokens. Pasted SAS URLs derive the canonical account, selected container, and default source name without persisting the token in connection metadata. + * Validates required Read and List permissions, HTTPS-only protocol, account-SAS Blob resource scope, start time, and expiry. Extra permissions and broader account credentials remain usable but produce least-privilege warnings. + * Shows non-secret SAS scope, named permissions, exact expiry, days remaining, stored-policy status, IP restrictions, and warnings in connection tests and source rows. + * Supports saving Blob credentials with or without Azure Key Vault; Key Vault is used when enabled and existing File Sync credential persistence is used otherwise. + * (Ref: microsoft/simplechat#1027, `functions_file_sync.py`, `workspace-file-sync.js`, `AZURE_BLOB_CONTAINER_SAS_SUPPORT_FIX.md`) + +### **(v0.250.068)** + +#### Bug Fixes + +* **Azure Blob File Sync Endpoint and Error Hardening** + * Restricted Azure Blob File Sync URLs and connection strings to validated HTTPS Azure Blob endpoints, blocking arbitrary, internal, development-storage, and credential-bearing endpoint forms before SDK requests are created. + * Replaced raw File Sync route, run-history, activity, and item exception text with fixed client-safe messages while retaining detailed sanitized diagnostics in server logs. + * (Ref: microsoft/simplechat#1027, PR #1088 security review, `functions_file_sync.py`, `route_backend_file_sync.py`) + +### **(v0.250.067)** + +#### New Features + +* **Azure Blob Storage File Sync** + * Added Azure Blob Storage as an admin-controlled File Sync source for personal, group, and public workspaces, with account, container, prefix, selected-path, filter, tag, schedule, and remote-delete controls. + * Added managed identity, Key Vault-backed service principal and connection string authentication, connection testing, virtual-folder browsing, ETag change detection, and streamed ingestion through the existing document pipeline. + * (Ref: microsoft/simplechat#1027, `functions_file_sync.py`, `workspace-file-sync.js`, `AZURE_BLOB_STORAGE_FILE_SYNC.md`) + ### **(v0.250.066)** #### Bug Fixes diff --git a/functional_tests/test_file_sync_azure_blob_storage.py b/functional_tests/test_file_sync_azure_blob_storage.py new file mode 100644 index 000000000..de9442c08 --- /dev/null +++ b/functional_tests/test_file_sync_azure_blob_storage.py @@ -0,0 +1,1126 @@ +#!/usr/bin/env python3 +# test_file_sync_azure_blob_storage.py +""" +Functional test for Azure Blob Storage File Sync. +Version: 0.250.070 +Implemented in: 0.250.067 +Security hardening in: 0.250.068 +Container SAS support in: 0.250.069 +Non-Key-Vault and List/Read validation fix in: 0.250.070 + +This test ensures Azure Blob Storage is wired into the shared File Sync +pipeline for every supported workspace scope without requiring live Azure +Storage or Cosmos DB access. +""" + +import ast +import hashlib +import json +import logging +import os +import re +import sys +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qsl, quote, unquote, urlparse + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def read_text(relative_path): + """Read a repository file as UTF-8 text.""" + return (REPO_ROOT / relative_path).read_text(encoding="utf-8") + + +def parse_app(relative_path): + """Parse an application module with AST.""" + return ast.parse(read_text(f"application/single_app/{relative_path}")) + + +def function_names(parsed): + """Return function names from a parsed module.""" + return {node.name for node in ast.walk(parsed) if isinstance(node, ast.FunctionDef)} + + +def load_functions(relative_path, names, additional_globals=None): + """Load selected top-level functions without importing application config.""" + parsed = parse_app(relative_path) + selected_nodes = [ + node + for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name in names + ] + module = ast.Module(body=selected_nodes, type_ignores=[]) + ast.fix_missing_locations(module) + namespace = { + "Any": Any, + "Dict": Dict, + "FileSyncPublicValidationError": ValueError, + "List": List, + "Optional": Optional, + "Tuple": Tuple, + "datetime": datetime, + "hashlib": hashlib, + "json": json, + "os": os, + "parse_qsl": parse_qsl, + "quote": quote, + "re": re, + "tempfile": tempfile, + "timedelta": timedelta, + "timezone": timezone, + "unquote": unquote, + "urlparse": urlparse, + } + namespace.update(additional_globals or {}) + exec(compile(module, relative_path, "exec"), namespace) + return namespace + + +def test_version_and_dependency_pin(): + """Validate the app version and existing Azure Blob SDK dependency pin.""" + config_text = read_text("application/single_app/config.py") + requirements_text = read_text("application/single_app/requirements.txt") + + assert 'VERSION = "0.250.070"' in config_text + assert "azure-storage-blob==12.24.1" in requirements_text + + +def test_file_sync_backend_azure_blob_wiring(): + """Validate Azure Blob connector helpers and shared dispatch exist.""" + file_sync_text = read_text("application/single_app/functions_file_sync.py") + names = function_names(parse_app("functions_file_sync.py")) + + expected_functions = { + "_normalize_azure_blob_connection", + "_test_azure_blob_connection", + "_get_azure_blob_service_client", + "_get_azure_blob_container_client", + "_browse_azure_blob_path", + "_list_azure_blobs", + "_stage_azure_blob_file", + "_list_remote_files", + "_stage_remote_file", + } + assert expected_functions.issubset(names) + assert 'FILE_SYNC_SOURCE_TYPE_AZURE_BLOB = "azure_blob"' in file_sync_text + assert 'FILE_SYNC_SOURCE_TYPE_AZURE_BLOB: {"managed_identity", "client_secret", "connection_string"}' in file_sync_text + assert "BlobServiceClient.from_connection_string" in file_sync_text + assert "DefaultAzureCredential(managed_identity_client_id=" in file_sync_text + assert "ClientSecretCredential(" in file_sync_text + assert '"remote_change_token":' in file_sync_text + assert '"azure_blob_name":' in file_sync_text + assert "downloader.chunks()" in file_sync_text + assert "Azure Blob Storage secret credentials must be stored in Azure Key Vault" not in file_sync_text + + +def test_azure_blob_connection_contract(): + """Validate storage account, container, and prefix fields are represented.""" + file_sync_text = read_text("application/single_app/functions_file_sync.py") + + for marker in [ + '"account_url": account_url', + '"container_name": container_name', + '"blob_prefix": blob_prefix', + '"selected_paths": _normalize_selected_paths', + "list_blobs(name_starts_with=", + "walk_blobs(name_starts_with=", + ]: + assert marker in file_sync_text + connection_test_block = file_sync_text.split("def _test_azure_blob_connection", 1)[1].split( + "def _azure_blob_error_diagnostics", 1 + )[0] + assert "get_container_properties" not in connection_test_block + + +def test_azure_blob_connection_normalization_behavior(): + """Validate account names, URLs, containers, prefixes, and selected paths.""" + names = { + "parse_file_sync_list", + "_normalize_text", + "_normalize_selected_path", + "_normalize_selected_paths", + "_azure_blob_endpoint_suffix_for_hostname", + "_normalize_azure_blob_url", + "_normalize_azure_container_name", + "_normalize_azure_blob_connection", + } + functions = load_functions( + "functions_file_sync.py", + names, + {"AZURE_STORAGE_ENDPOINT_SUFFIXES": ("core.windows.net",)}, + ) + normalize_connection = functions["_normalize_azure_blob_connection"] + + normalized = normalize_connection({ + "account_name": "contosodata", + "container_name": "documents", + "blob_prefix": "incoming/reports/", + "selected_paths": ["2025", "2025", "2026/q1"], + }) + assert normalized == { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + "blob_prefix": "incoming/reports", + "selected_paths": ["2025", "2026/q1"], + } + + url_normalized = normalize_connection({ + "account_url": "https://contosodata.blob.core.windows.net/archive/exports", + }) + assert url_normalized["container_name"] == "archive" + assert url_normalized["blob_prefix"] == "exports" + + root_normalized = normalize_connection({ + "account_name": "contosodata", + "container_name": "$root", + }) + assert root_normalized["container_name"] == "$root" + + try: + normalize_connection({"account_name": "contosodata", "container_name": "Invalid_Name"}) + raise AssertionError("Invalid container names must be rejected") + except ValueError as exc: + assert "container name" in str(exc) + + +def test_azure_blob_endpoints_block_server_side_request_forgery(): + """Validate Blob URLs and connection strings stay on Azure-owned endpoints.""" + endpoint_suffixes = ( + "core.windows.net", + "core.usgovcloudapi.net", + "core.chinacloudapi.cn", + "core.cloudapi.de", + ) + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_azure_blob_endpoint_suffix_for_hostname", + "_normalize_azure_blob_url", + "_parse_azure_blob_sas_parameters", + "_parse_azure_blob_sas_datetime", + "_azure_blob_permission_labels", + "_azure_blob_sas_metadata", + "_parse_azure_blob_connection_string", + "_parse_azure_blob_sas_url", + "_parse_azure_blob_credential", + "_validate_azure_blob_connection_string", + }, + {"AZURE_STORAGE_ENDPOINT_SUFFIXES": endpoint_suffixes}, + ) + normalize_url = functions["_normalize_azure_blob_url"] + validate_connection_string = functions["_validate_azure_blob_connection_string"] + + allowed_urls = [ + "https://contosodata.blob.core.windows.net", + "https://contosodata.blob.core.usgovcloudapi.net", + "https://contosodata.blob.core.chinacloudapi.cn", + "https://contosodata.blob.core.cloudapi.de", + ] + for allowed_url in allowed_urls: + normalized_url, path_parts = normalize_url(allowed_url) + assert normalized_url == allowed_url + assert path_parts == [] + + blocked_urls = [ + "https://127.0.0.1", + "https://169.254.169.254/latest/meta-data", + "https://localhost", + "https://internal.example.com", + "https://contosodata.blob.core.windows.net.evil.example", + "https://contoso-data.blob.core.windows.net", + "https://user:password@contosodata.blob.core.windows.net", + "https://contosodata.blob.core.windows.net:444", + "https://contosodata.blob.core.windows.net?sig=secret", + "https://contosodata.blob.core.windows.net#fragment", + ] + for blocked_url in blocked_urls: + try: + normalize_url(blocked_url) + raise AssertionError(f"Unsafe Blob URL was accepted: {blocked_url}") + except ValueError as exc: + assert "Azure Blob Storage" in str(exc) + + validate_connection_string( + "DefaultEndpointsProtocol=https;AccountName=contosodata;" + "AccountKey=ZmFrZQ==;EndpointSuffix=core.windows.net" + ) + validate_connection_string( + "BlobEndpoint=https://contosodata.blob.core.windows.net;" + "SharedAccessSignature=sv=fake&sig=secret" + ) + blocked_connection_strings = [ + "UseDevelopmentStorage=true", + "DefaultEndpointsProtocol=http;AccountName=contosodata;AccountKey=ZmFrZQ==;EndpointSuffix=core.windows.net", + "DefaultEndpointsProtocol=https;AccountName=contosodata;AccountKey=ZmFrZQ==;EndpointSuffix=example.com", + "BlobEndpoint=https://127.0.0.1;SharedAccessSignature=sv=fake&sig=secret", + "BlobEndpoint=https://internal.example.com;SharedAccessSignature=sv=fake&sig=secret", + ] + for connection_string in blocked_connection_strings: + try: + validate_connection_string(connection_string) + raise AssertionError("Unsafe Blob connection string was accepted") + except ValueError as exc: + assert "Azure Blob Storage" in str(exc) + + +def test_azure_blob_service_client_revalidates_endpoints_before_sdk_use(): + """Validate stored endpoint values cannot bypass checks at the SDK boundary.""" + endpoint_suffixes = ("core.windows.net",) + + class FakeBlobServiceClient: + constructor_calls = [] + connection_string_calls = [] + + def __init__(self, account_url, credential): + self.constructor_calls.append((account_url, credential)) + + @classmethod + def from_connection_string(cls, connection_string): + cls.connection_string_calls.append(connection_string) + return cls("from-connection-string", None) + + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_azure_blob_endpoint_suffix_for_hostname", + "_normalize_azure_blob_url", + "_normalize_azure_container_name", + "_now", + "_parse_azure_blob_sas_parameters", + "_parse_azure_blob_sas_datetime", + "_azure_blob_permission_labels", + "_azure_blob_sas_metadata", + "_parse_azure_blob_connection_string", + "_parse_azure_blob_sas_url", + "_parse_azure_blob_credential", + "_validate_azure_blob_connection_string", + "_validate_azure_blob_sas_for_container", + "_get_azure_blob_service_client", + }, + { + "AZURE_STORAGE_ENDPOINT_SUFFIXES": endpoint_suffixes, + "BlobServiceClient": FakeBlobServiceClient, + "DefaultAzureCredential": lambda managed_identity_client_id=None: { + "managed_identity_client_id": managed_identity_client_id, + }, + "ClientSecretCredential": lambda **kwargs: kwargs, + "TENANT_ID": "tenant", + "_get_identity_auth_for_source": lambda source: None, + "_resolved_auth_secret": lambda auth: str(auth.get("secret") or ""), + }, + ) + get_service_client = functions["_get_azure_blob_service_client"] + + safe_source = { + "connection": {"account_url": "https://contosodata.blob.core.windows.net"}, + "auth": {"auth_type": "managed_identity"}, + } + get_service_client(safe_source) + assert FakeBlobServiceClient.constructor_calls == [ + ("https://contosodata.blob.core.windows.net", {"managed_identity_client_id": None}) + ] + + unsafe_sources = [ + { + "connection": {"account_url": "https://169.254.169.254/latest/meta-data"}, + "auth": {"auth_type": "managed_identity"}, + }, + { + "connection": {"account_url": "https://internal.example.com"}, + "auth": {"auth_type": "managed_identity"}, + }, + { + "connection": {}, + "auth": { + "auth_type": "connection_string", + "secret": "BlobEndpoint=https://127.0.0.1;SharedAccessSignature=sv=fake&sig=secret", + }, + }, + ] + for unsafe_source in unsafe_sources: + try: + get_service_client(unsafe_source) + raise AssertionError("Unsafe endpoint reached BlobServiceClient") + except ValueError as exc: + assert "Azure Blob Storage" in str(exc) + + assert len(FakeBlobServiceClient.constructor_calls) == 1 + assert FakeBlobServiceClient.connection_string_calls == [] + + +def test_azure_blob_container_sas_scope_permissions_and_expiry(): + """Validate container SAS connection strings and safe metadata extraction.""" + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_azure_blob_endpoint_suffix_for_hostname", + "_normalize_azure_blob_url", + "_normalize_azure_container_name", + "_now", + "_parse_azure_blob_sas_parameters", + "_parse_azure_blob_sas_datetime", + "_azure_blob_permission_labels", + "_azure_blob_sas_metadata", + "_parse_azure_blob_connection_string", + "_parse_azure_blob_sas_url", + "_parse_azure_blob_credential", + "_validate_azure_blob_connection_string", + "_validate_azure_blob_sas_for_container", + }, + { + "AZURE_STORAGE_ENDPOINT_SUFFIXES": ("core.windows.net",), + "AZURE_BLOB_SAS_PERMISSION_LABELS": { + "r": "Read", + "l": "List", + "w": "Write", + "d": "Delete", + }, + }, + ) + parse_connection_string = functions["_parse_azure_blob_connection_string"] + parse_credential = functions["_parse_azure_blob_credential"] + validate_for_container = functions["_validate_azure_blob_sas_for_container"] + + container_sas = ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=sv=2022-11-02&spr=https&" + "st=2026-07-28T00%3A00%3A00Z&se=2030-07-28T00%3A00%3A00Z&" + "sr=c&sp=rl&sig=fake-container-signature" + ) + parsed_container_sas = parse_connection_string(container_sas) + validate_for_container(parsed_container_sas, "documents") + assert parsed_container_sas["account_url"] == "https://contosodata.blob.core.windows.net" + assert parsed_container_sas["endpoint_container_name"] == "documents" + assert parsed_container_sas["sas_token"].endswith("sig=fake-container-signature") + assert parsed_container_sas["sas_metadata"] == { + "credential_type": "sas", + "sas_scope": "container", + "permissions": "rl", + "starts_at": "2026-07-28T00:00:00+00:00", + "expires_at": "2030-07-28T00:00:00+00:00", + "https_only": True, + "stored_access_policy": False, + "signed_version": "2022-11-02", + "ip_range": "", + "services": "", + "resource_types": "", + "warnings": [], + } + assert "sig" not in parsed_container_sas["sas_metadata"] + assert "signature" not in str(parsed_container_sas["sas_metadata"]).lower() + + container_sas_url = ( + "https://contosodata.blob.core.windows.net/documents?" + "sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z&sig=fake-container-signature" + ) + parsed_sas_url = parse_credential(container_sas_url) + validate_for_container(parsed_sas_url, "documents", "https://contosodata.blob.core.windows.net") + assert parsed_sas_url["endpoint_container_name"] == "documents" + assert parsed_sas_url["sas_metadata"]["permissions"] == "rl" + + standalone_sas_token = ( + "?sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z&sig=fake-container-signature" + ) + parsed_sas_token = parse_credential( + standalone_sas_token, + account_url="https://contosodata.blob.core.windows.net", + container_name="documents", + ) + validate_for_container(parsed_sas_token, "documents", "https://contosodata.blob.core.windows.net") + assert parsed_sas_token["account_url"] == "https://contosodata.blob.core.windows.net" + assert parsed_sas_token["endpoint_container_name"] == "documents" + + read_only_sas_url = container_sas_url.replace("sp=rl", "sp=r") + try: + validate_for_container(parse_credential(read_only_sas_url), "documents") + raise AssertionError("Container SAS without List was accepted") + except ValueError as exc: + assert "Read and List" in str(exc) + + try: + validate_for_container(parsed_container_sas, "other-container") + raise AssertionError("Container SAS must match the selected container") + except ValueError as exc: + assert "selected container" in str(exc) + + extra_permission_container_sas = container_sas.replace("sr=c&sp=rl", "sr=c&sp=rlwd") + parsed_extra_permission_sas = parse_connection_string(extra_permission_container_sas) + validate_for_container(parsed_extra_permission_sas, "documents") + assert any( + "Write, Delete" in warning and "Read and List are sufficient" in warning + for warning in parsed_extra_permission_sas["sas_metadata"]["warnings"] + ) + + account_sas = ( + "BlobEndpoint=https://contosodata.blob.core.windows.net;" + "SharedAccessSignature=sv=2022-11-02&spr=https&ss=b&srt=co&" + "sp=rlwd&se=2030-07-28T00%3A00%3A00Z&sig=fake-account-signature" + ) + parsed_account_sas = parse_connection_string(account_sas) + validate_for_container(parsed_account_sas, "documents") + assert parsed_account_sas["sas_metadata"]["sas_scope"] == "account" + assert any("more access than this single-container source needs" in warning for warning in parsed_account_sas["sas_metadata"]["warnings"]) + assert any("Write, Delete" in warning for warning in parsed_account_sas["sas_metadata"]["warnings"]) + + blocked_sas_values = [ + ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents/report.pdf;" + "SharedAccessSignature=sv=2022-11-02&spr=https&sr=b&sp=r&" + "se=2030-07-28T00%3A00%3A00Z&sig=fake-blob-signature" + ), + ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=sv=2022-11-02&spr=https&sr=c&sp=r&" + "se=2030-07-28T00%3A00%3A00Z&sig=missing-list" + ), + ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2020-07-28T00%3A00%3A00Z&sig=expired" + ), + ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z" + ), + ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z&sig=missing-version" + ), + ] + for blocked_sas in blocked_sas_values: + try: + validate_for_container(parse_connection_string(blocked_sas), "documents") + raise AssertionError("Unsuitable SAS credential was accepted") + except ValueError as exc: + assert any( + expected_text in str(exc) + for expected_text in ("blob/object SAS", "Read and List", "expired", "incomplete") + ) + + +def test_azure_blob_container_sas_uses_container_client_without_token_disclosure(): + """Validate container SAS credentials construct only the selected container client.""" + class FakeContainerClient: + calls = [] + + def __init__(self, account_url, container_name, credential): + self.calls.append((account_url, container_name, credential)) + + class FakeBlobServiceClient: + calls = [] + + @classmethod + def from_connection_string(cls, connection_string): + cls.calls.append(connection_string) + raise AssertionError("Container SAS must not use BlobServiceClient.from_connection_string") + + connection_string = ( + "BlobEndpoint=https://contosodata.blob.core.windows.net/documents;" + "SharedAccessSignature=sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z&sig=fake-container-signature" + ) + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_azure_blob_endpoint_suffix_for_hostname", + "_normalize_azure_blob_url", + "_normalize_azure_container_name", + "_now", + "_parse_azure_blob_sas_parameters", + "_parse_azure_blob_sas_datetime", + "_azure_blob_sas_metadata", + "_parse_azure_blob_connection_string", + "_parse_azure_blob_sas_url", + "_parse_azure_blob_credential", + "_validate_azure_blob_connection_string", + "_validate_azure_blob_sas_for_container", + "_get_azure_blob_service_client", + "_get_azure_blob_container_client", + }, + { + "AZURE_STORAGE_ENDPOINT_SUFFIXES": ("core.windows.net",), + "ContainerClient": FakeContainerClient, + "BlobServiceClient": FakeBlobServiceClient, + "_get_identity_auth_for_source": lambda source: None, + "_resolved_auth_secret": lambda auth: str(auth.get("secret") or ""), + }, + ) + source = { + "connection": { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + }, + "auth": {"auth_type": "connection_string", "secret": connection_string}, + } + functions["_get_azure_blob_container_client"](source) + + assert FakeContainerClient.calls == [ + ( + "https://contosodata.blob.core.windows.net", + "documents", + "sv=2022-11-02&spr=https&sr=c&sp=rl&se=2030-07-28T00%3A00%3A00Z&sig=fake-container-signature", + ) + ] + assert FakeBlobServiceClient.calls == [] + + +def test_azure_blob_sas_url_hydrates_safe_connection_fields(): + """Validate pasted SAS URLs derive account/container without persisting the token in connection data.""" + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_get_file_sync_secret_value", + "_azure_blob_endpoint_suffix_for_hostname", + "_parse_azure_blob_sas_parameters", + "_parse_azure_blob_sas_datetime", + "_azure_blob_sas_metadata", + "_parse_azure_blob_sas_url", + "_parse_azure_blob_connection_string", + "_normalize_azure_blob_url", + "_normalize_azure_container_name", + "_parse_azure_blob_credential", + "_hydrate_azure_blob_connection_from_credential", + }, + { + "AZURE_STORAGE_ENDPOINT_SUFFIXES": ("core.windows.net",), + "ui_trigger_word": "__stored__", + }, + ) + sas_url = ( + "https://contosodata.blob.core.windows.net/documents?" + "sv=2022-11-02&spr=https&sr=c&sp=rl&" + "se=2030-07-28T00%3A00%3A00Z&sig=fake-container-signature" + ) + connection, credentials = functions["_hydrate_azure_blob_connection_from_credential"]( + {"account_url": sas_url}, + {}, + {}, + ) + + assert connection == { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + } + assert credentials["connection_string"] == sas_url + assert "sig=" not in json.dumps(connection) + + client_secret_connection, client_secret_credentials = functions["_hydrate_azure_blob_connection_from_credential"]( + { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + }, + {}, + {"auth_type": "client_secret", "secret": "not-a-sas-token"}, + ) + assert client_secret_connection["container_name"] == "documents" + assert client_secret_credentials["secret"] == "not-a-sas-token" + + standalone_connection, standalone_credentials = functions["_hydrate_azure_blob_connection_from_credential"]( + { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + }, + {}, + { + "auth_type": "connection_string", + "secret": "?sv=2022-11-02&spr=https&sr=c&sp=rl&se=2030-07-28T00%3A00%3A00Z&sig=fake", + }, + ) + assert standalone_connection["container_name"] == "documents" + assert standalone_credentials["auth_type"] == "connection_string" + + +def test_azure_blob_new_credentials_are_validated_before_secret_storage(): + """Validate rejected Blob credentials cannot write a new Key Vault secret version.""" + file_sync_text = read_text("application/single_app/functions_file_sync.py") + normalize_source_text = file_sync_text.split("def _normalize_source_payload", 1)[1].split( + "def create_file_sync_source", 1 + )[0] + + assert "def _prevalidate_new_azure_blob_credential" in file_sync_text + assert normalize_source_text.index("_prevalidate_new_azure_blob_credential(") < normalize_source_text.index( + "_prepare_auth_payload(" + ) + assert "and not identity_id" in normalize_source_text + + +def test_azure_blob_identity_and_read_fallback_guards_are_wired(): + """Validate identity rotations refresh metadata and fallback reads avoid false warnings.""" + file_sync_text = read_text("application/single_app/functions_file_sync.py") + file_sync_js = read_text("application/single_app/static/js/workspace/workspace-file-sync.js") + + assert "resolved_identity_auth = get_workspace_identity_auth" in file_sync_text + assert "tolerate_validation_errors=True" in file_sync_text + assert "if not read_verified:" in file_sync_text + assert "files_seen += 1" in file_sync_text + assert "if (identitySelect.value)" in file_sync_js + assert "credentials.connection_string = '';" in file_sync_js + + +def test_azure_blob_failure_diagnostics_are_non_secret_and_actionable(): + """Validate Azure failure logs contain useful codes but no signed URLs or tokens.""" + class FakeResponse: + status_code = 403 + headers = {"x-ms-request-id": "request-123"} + + class FakeAzureError(Exception): + class ErrorCode: + value = "AuthorizationPermissionMismatch" + + error_code = ErrorCode() + status_code = 403 + response = FakeResponse() + + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_normalize_azure_storage_error_code", + "_sanitize_azure_blob_credential_metadata", + "_azure_blob_error_diagnostics", + }, + ) + diagnostics = functions["_azure_blob_error_diagnostics"]( + FakeAzureError("signed URL must not be logged"), + { + "id": "source-1", + "credential_metadata": { + "credential_type": "sas", + "sas_scope": "container", + "permissions": "rl", + "expires_at": "2030-07-28T00:00:00+00:00", + "warnings": [], + }, + }, + ) + + assert diagnostics == { + "exception_type": "FakeAzureError", + "error_code": "authorizationpermissionmismatch", + "status_code": 403, + "request_id": "request-123", + "source_id": "source-1", + "auth_kind": "sas", + "scope_kind": "container", + "permissions": "rl", + } + assert "signed URL" not in json.dumps(diagnostics) + file_sync_text = read_text("application/single_app/functions_file_sync.py") + for public_message in [ + "A container SAS must include both Read and List permissions.", + "Check the account, container, signature, start time, and expiry.", + "Include the app's outbound addresses or adjust the storage network policy.", + "Azure could not find the selected Blob container", + ]: + assert public_message in file_sync_text + connection_test_block = file_sync_text.split("def _test_azure_blob_connection", 1)[1].split( + "def _azure_blob_error_diagnostics", 1 + )[0] + assert '"error": str(error)' not in connection_test_block + + +def test_azure_blob_connection_test_uses_only_list_and_read_operations(): + """Validate container SAS testing avoids unneeded container-properties access.""" + class FakeBlobClient: + def get_blob_properties(self): + return {"etag": "etag-1"} + + class FakeContainerClient: + def get_container_properties(self): + raise AssertionError("Get Container Properties is not required for File Sync") + + def walk_blobs(self, name_starts_with=None, delimiter=None): + assert name_starts_with == "reports/" + assert delimiter == "/" + return [{"name": "reports/example.pdf", "size": 10}] + + def list_blobs(self, name_starts_with=None): + raise AssertionError("Fallback listing should not run after a readable top-level blob") + + def get_blob_client(self, blob_name): + assert blob_name == "reports/example.pdf" + return FakeBlobClient() + + functions = load_functions( + "functions_file_sync.py", + { + "_normalize_text", + "_azure_blob_item_value", + "_azure_blob_item_name", + "_azure_blob_item_is_folder", + "_sanitize_azure_blob_credential_metadata", + "_test_azure_blob_connection", + }, + { + "_get_azure_blob_container_client": lambda source: FakeContainerClient(), + "FileSyncPublicValidationError": ValueError, + "AzureResourceNotFoundError": RuntimeError, + "log_event": lambda *args, **kwargs: None, + "logging": logging, + }, + ) + result = functions["_test_azure_blob_connection"]({ + "id": "source-1", + "source_type": "azure_blob", + "recursive": True, + "connection": {"blob_prefix": "reports"}, + "credential_metadata": { + "credential_type": "sas", + "sas_scope": "container", + "permissions": "rl", + "warnings": [], + }, + }) + + assert result["success"] is True + assert result["entries_checked"] == 1 + assert result["files_seen"] == 1 + assert result["read_verified"] is True + assert result["credential_metadata"]["warnings"] == [] + + +def test_azure_blob_sas_metadata_is_non_secret_and_frontend_visible(): + """Validate source serialization and UI expose SAS status without the token.""" + file_sync_text = read_text("application/single_app/functions_file_sync.py") + file_sync_js = read_text("application/single_app/static/js/workspace/workspace-file-sync.js") + + for marker in [ + '"credential_metadata": credential_metadata', + 'sanitized_source["credential_metadata"]', + 'source.get("credential_metadata")', + ]: + assert marker in file_sync_text + for marker in [ + "Container SAS", + "Account SAS", + "Blob connection string or SAS", + "Blob connection string, SAS URL, or SAS token", + "full container SAS URL", + "standalone SAS token", + "Expires", + "Expiry is controlled by a stored access policy", + "grant broader access", + "Each source syncs one container", + "Blob Sync", + ]: + assert marker in file_sync_js + + +def test_file_sync_routes_do_not_disclose_exception_details(): + """Validate detailed backend exceptions are logged but never returned.""" + route_text = read_text("application/single_app/route_backend_file_sync.py") + + assert "from functions_appinsights import log_event" in route_text + assert "FileSyncPublicValidationError" in route_text + assert "error.public_message" in route_text + assert '"[FileSync] Request failed."' in route_text + assert '"error": str(error)' in route_text + assert "return _error(str(error)" not in route_text + for public_message in [ + "You do not have permission to perform this File Sync operation.", + "The requested File Sync resource was not found.", + "The File Sync request could not be completed. Verify the source configuration and try again.", + "An unexpected error occurred while processing the File Sync request.", + ]: + assert public_message in route_text + + +def test_file_sync_run_and_item_errors_are_client_safe(): + """Validate persisted and serialized failure messages do not expose SDK details.""" + public_run_error = "File Sync run failed. Contact an administrator if the problem continues." + public_item_error = "File Sync could not process this item. Contact an administrator if the problem continues." + functions = load_functions( + "functions_file_sync.py", + {"sanitize_file_sync_run"}, + {"FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE": public_run_error}, + ) + sanitized_run = functions["sanitize_file_sync_run"]({ + "id": "run-1", + "status": "failed", + "error_message": "Request to https://internal.example?sig=secret failed", + }) + + assert sanitized_run["error_message"] == public_run_error + assert "internal.example" not in sanitized_run["error_message"] + assert "secret" not in sanitized_run["error_message"] + + file_sync_text = read_text("application/single_app/functions_file_sync.py") + assert f'FILE_SYNC_PUBLIC_RUN_ERROR_MESSAGE = "{public_run_error}"' in file_sync_text + assert f'FILE_SYNC_PUBLIC_ITEM_ERROR_MESSAGE = "{public_item_error}"' in file_sync_text + assert '"error_message": str(error)[:1000]' not in file_sync_text + assert 'item["error_message"] = str(delete_error)[:1000]' not in file_sync_text + assert '"run_failed", {"run_id": run["id"], "error": error_message}' not in file_sync_text + + +def test_azure_blob_secret_credentials_support_optional_key_vault_storage(): + """Validate Blob secrets use Key Vault when enabled and source storage otherwise.""" + store_calls = [] + disabled_functions = load_functions( + "functions_file_sync.py", + {"_as_bool", "_store_file_sync_secret"}, + { + "get_settings": lambda: { + "enable_key_vault_secret_storage": False, + "key_vault_name": "", + }, + "store_secret_in_key_vault": lambda **kwargs: store_calls.append(kwargs), + "_keyvault_scope": lambda scope_type: scope_type, + }, + ) + inline_secret = disabled_functions["_store_file_sync_secret"]( + "personal", "user-1", "source-1", "secret", "sas-secret" + ) + assert inline_secret == "sas-secret" + assert store_calls == [] + + enabled_functions = load_functions( + "functions_file_sync.py", + {"_as_bool", "_store_file_sync_secret"}, + { + "get_settings": lambda: { + "enable_key_vault_secret_storage": True, + "key_vault_name": "vault", + }, + "store_secret_in_key_vault": lambda **kwargs: "vault-secret-reference", + "_keyvault_scope": lambda scope_type: scope_type, + }, + ) + key_vault_secret = enabled_functions["_store_file_sync_secret"]( + "personal", "user-1", "source-1", "secret", "sas-secret" + ) + assert key_vault_secret == "vault-secret-reference" + + +def test_azure_blob_list_browse_and_stage_behavior(): + """Validate virtual paths, recursive filtering, metadata, and streamed staging.""" + names = { + "_azure_blob_item_value", + "_azure_blob_item_name", + "_azure_blob_item_is_folder", + "_azure_blob_item_size", + "_azure_blob_item_modified_at", + "_azure_blob_item_change_token", + "_join_azure_blob_path", + "_relative_azure_blob_path", + "_build_azure_blob_url", + "_browse_azure_blob_path", + "_list_azure_blobs", + "_stage_azure_blob_file", + } + + class BlobPrefix: + def __init__(self, name): + self.name = name + + class FakeDownloader: + def chunks(self): + return [b"blob ", b"content"] + + class FakeBlobClient: + def download_blob(self): + return FakeDownloader() + + class FakeContainerClient: + def __init__(self): + self.last_browse_prefix = None + self.last_blob_name = None + + def walk_blobs(self, name_starts_with=None, delimiter=None): + self.last_browse_prefix = name_starts_with + assert delimiter == "/" + return [ + BlobPrefix("incoming/reports/2025/"), + { + "name": "incoming/reports/summary.pdf", + "size": 12, + "last_modified": "2026-07-28T12:00:00+00:00", + "etag": "etag-summary", + }, + ] + + def list_blobs(self, name_starts_with=None): + assert name_starts_with == "incoming/reports" + return [ + { + "name": "incoming/reports/summary.pdf", + "size": 12, + "last_modified": "2026-07-28T12:00:00+00:00", + "etag": "etag-summary", + }, + { + "name": "incoming/reports/2025/detail.docx", + "size": 34, + "last_modified": "2026-07-28T12:01:00+00:00", + "etag": "etag-detail", + }, + {"name": "incoming/reports/empty/", "size": 0}, + { + "name": "incoming/reports/hns-directory", + "size": 0, + "metadata": {"hdi_isfolder": "true"}, + }, + ] + + def get_blob_client(self, blob_name): + self.last_blob_name = blob_name + return FakeBlobClient() + + container_client = FakeContainerClient() + functions = load_functions( + "functions_file_sync.py", + names, + { + "_format_smb_modified_at": lambda value: value, + "_get_azure_blob_container_client": lambda source: container_client, + }, + ) + source = { + "source_type": "azure_blob", + "recursive": True, + "connection": { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "documents", + "blob_prefix": "incoming/reports", + "selected_paths": [], + }, + } + config = { + "file_sync_allow_recursive_sources": True, + "file_sync_max_files_per_run": 100, + } + + browse_entries = functions["_browse_azure_blob_path"](source, "") + assert container_client.last_browse_prefix == "incoming/reports/" + assert browse_entries == [ + {"name": "2025", "path": "2025", "type": "folder", "size": 0, "modified_at": None}, + { + "name": "summary.pdf", + "path": "summary.pdf", + "type": "file", + "size": 12, + "modified_at": "2026-07-28T12:00:00+00:00", + }, + ] + + remote_files = functions["_list_azure_blobs"](source, config) + assert [item["relative_path"] for item in remote_files] == ["summary.pdf", "2025/detail.docx"] + assert remote_files[0]["remote_change_token"] == "etag-summary" + assert remote_files[0]["remote_path"] == "https://contosodata.blob.core.windows.net/documents/incoming/reports/summary.pdf" + + non_recursive_source = {**source, "recursive": False} + non_recursive_files = functions["_list_azure_blobs"](non_recursive_source, config) + assert [item["relative_path"] for item in non_recursive_files] == ["summary.pdf"] + + staged_path, content_hash = functions["_stage_azure_blob_file"](source, remote_files[0]) + try: + assert Path(staged_path).read_bytes() == b"blob content" + assert content_hash == hashlib.sha256(b"blob content").hexdigest() + assert container_client.last_blob_name == "incoming/reports/summary.pdf" + finally: + Path(staged_path).unlink(missing_ok=True) + + +def test_workspace_identity_catalog_supports_azure_blob(): + """Validate reusable identities can be scoped to Azure Blob File Sync.""" + identity_text = read_text("application/single_app/functions_workspace_identities.py") + identities_js = read_text("application/single_app/static/js/workspace/workspace-identities.js") + + assert '"file_sync": ["smb", "azure_files", "azure_blob",' in identity_text + assert "sourceTypes: ['smb', 'azure_files', 'azure_blob'," in identities_js + + +def test_frontend_source_workflow_supports_azure_blob(): + """Validate admins can enable Blob sources and all workspaces can configure them.""" + file_sync_js = read_text("application/single_app/static/js/workspace/workspace-file-sync.js") + admin_template = read_text("application/single_app/templates/admin_settings.html") + workspace_templates = { + "personal": read_text("application/single_app/templates/workspace.html"), + "group": read_text("application/single_app/templates/group_workspaces.html"), + "public": read_text("application/single_app/templates/manage_public_workspace.html"), + } + + for marker in [ + "value: 'azure_blob'", + "label: 'Azure Blob Storage'", + "azure_blob: ['managed_identity', 'client_secret', 'connection_string']", + "Blob service URL or account name", + "Container name", + "Blob prefix", + "A container SAS needs Read and List only.", + "To sync multiple containers, create one source per container.", + "container_name: containerNameField.input.value.trim()", + "blob_prefix: blobPrefixField.input.value.trim()", + ]: + assert marker in file_sync_js + + assert 'name="file_sync_visible_source_types" value="azure_blob"' in admin_template + assert "Azure Blob Storage" in admin_template + for scope_type, template_text in workspace_templates.items(): + assert f'data-scope="{scope_type}"' in template_text + assert "data-visible-source-types=" in template_text + assert "settings.file_sync_visible_source_types" in template_text + + +def test_synced_document_badges_include_azure_blob(): + """Validate synced-document source badges identify Azure Blob Storage.""" + workspace_utils = read_text("application/single_app/static/js/workspace/workspace-utils.js") + group_template = read_text("application/single_app/templates/group_workspaces.html") + public_js = read_text("application/single_app/static/js/public/public_workspace.js") + + for frontend_text in [workspace_utils, group_template, public_js]: + assert "azure_blob" in frontend_text + assert "Managed by File Sync from Azure Blob Storage" in frontend_text + + +def run_tests(): + """Run all tests in this file.""" + tests = [ + test_version_and_dependency_pin, + test_file_sync_backend_azure_blob_wiring, + test_azure_blob_connection_contract, + test_azure_blob_connection_normalization_behavior, + test_azure_blob_endpoints_block_server_side_request_forgery, + test_azure_blob_service_client_revalidates_endpoints_before_sdk_use, + test_azure_blob_container_sas_scope_permissions_and_expiry, + test_azure_blob_container_sas_uses_container_client_without_token_disclosure, + test_azure_blob_sas_url_hydrates_safe_connection_fields, + test_azure_blob_new_credentials_are_validated_before_secret_storage, + test_azure_blob_identity_and_read_fallback_guards_are_wired, + test_azure_blob_failure_diagnostics_are_non_secret_and_actionable, + test_azure_blob_connection_test_uses_only_list_and_read_operations, + test_azure_blob_sas_metadata_is_non_secret_and_frontend_visible, + test_file_sync_routes_do_not_disclose_exception_details, + test_file_sync_run_and_item_errors_are_client_safe, + test_azure_blob_secret_credentials_support_optional_key_vault_storage, + test_azure_blob_list_browse_and_stage_behavior, + test_workspace_identity_catalog_supports_azure_blob, + test_frontend_source_workflow_supports_azure_blob, + test_synced_document_badges_include_azure_blob, + ] + results = [] + for test in tests: + try: + test() + print(f"PASS {test.__name__}") + results.append(True) + except Exception as exc: + print(f"FAIL {test.__name__}: {exc}") + results.append(False) + return all(results) + + +if __name__ == "__main__": + sys.exit(0 if run_tests() else 1) \ No newline at end of file diff --git a/functional_tests/test_file_sync_azure_files_identity.py b/functional_tests/test_file_sync_azure_files_identity.py index 06fb1c60f..ee772c3a9 100644 --- a/functional_tests/test_file_sync_azure_files_identity.py +++ b/functional_tests/test_file_sync_azure_files_identity.py @@ -2,8 +2,12 @@ # test_file_sync_azure_files_identity.py """ Functional test for Azure Files File Sync identity support. -Version: 0.241.178 +Version: 0.250.070 Implemented in: 0.241.127 +Updated in: 0.250.067 +Updated in: 0.250.068 +Updated in: 0.250.069 +Updated in: 0.250.070 This test ensures Azure Files sync sources are wired to managed identity, service principal, and connection string authentication without requiring live @@ -39,7 +43,7 @@ def test_version_and_dependency_pin(): config_text = read_text("application/single_app/config.py") requirements_text = read_text("application/single_app/requirements.txt") - assert 'VERSION = "0.241.178"' in config_text + assert 'VERSION = "0.250.070"' in config_text assert "azure-storage-file-share==12.25.0" in requirements_text @@ -75,7 +79,7 @@ def test_workspace_identity_catalog_supports_azure_files(): """Validate reusable identities can describe Azure Files File Sync support.""" identity_text = read_text("application/single_app/functions_workspace_identities.py") - assert '"file_sync": ["smb", "azure_files", "onedrive", "google_drive", "google_shared_drive"]' in identity_text + assert '"file_sync": ["smb", "azure_files", "azure_blob", "onedrive", "google_drive", "google_shared_drive"]' in identity_text assert '"file_sync": {"anonymous", "client_secret", "connection_string", "managed_identity", "username_password"}' in identity_text assert "managed_identity_client_id" in identity_text assert "tenant_id" in identity_text @@ -105,7 +109,7 @@ def test_frontend_source_workflow_supports_azure_files(): assert marker in file_sync_js assert "Use this identity for File Sync sources" in identities_js - assert "sourceTypes: ['smb', 'azure_files', 'onedrive', 'google_drive', 'google_shared_drive']" in identities_js + assert "sourceTypes: ['smb', 'azure_files', 'azure_blob', 'onedrive', 'google_drive', 'google_shared_drive']" in identities_js assert "authTypes: ['username_password', 'anonymous', 'managed_identity', 'client_secret', 'connection_string']" in identities_js assert "file_sync_visible_source_type_azure_files" in admin_template assert "Azure Files" in admin_template diff --git a/functional_tests/test_file_sync_onedrive_personal.py b/functional_tests/test_file_sync_onedrive_personal.py index 90dce20cb..4a7688d16 100644 --- a/functional_tests/test_file_sync_onedrive_personal.py +++ b/functional_tests/test_file_sync_onedrive_personal.py @@ -2,8 +2,12 @@ # test_file_sync_onedrive_personal.py """ Functional test for personal OneDrive File Sync support. -Version: 0.241.178 +Version: 0.250.070 Implemented in: 0.241.128 +Updated in: 0.250.067 +Updated in: 0.250.068 +Updated in: 0.250.069 +Updated in: 0.250.070 This test ensures OneDrive sync source code remains wired as personal-only File Sync support while the admin source-type control keeps OneDrive marked as coming @@ -39,7 +43,7 @@ def test_version_and_source_defaults(): settings_text = read_text("application/single_app/functions_settings.py") file_sync_text = read_text("application/single_app/functions_file_sync.py") - assert 'VERSION = "0.241.178"' in config_text + assert 'VERSION = "0.250.070"' in config_text assert "FILE_SYNC_SOURCE_TYPE_ONEDRIVE = \"onedrive\"" in file_sync_text assert "FILE_SYNC_SOURCE_TYPE_ONEDRIVE" in file_sync_text assert "FILE_SYNC_SOURCE_TYPE_ONEDRIVE: {\"client_secret\"}" in file_sync_text @@ -84,14 +88,14 @@ def test_global_connector_identity_supports_cloud_drive_sync(): identity_js = read_text("application/single_app/static/js/workspace/workspace-identities.js") admin_template = read_text("application/single_app/templates/admin_settings.html") - assert '"file_sync": ["smb", "azure_files", "onedrive", "google_drive", "google_shared_drive"]' in identity_text + assert '"file_sync": ["smb", "azure_files", "azure_blob", "onedrive", "google_drive", "google_shared_drive"]' in identity_text assert 'allowed_usage_contexts = {"file_sync", "action"}' in identity_text assert "admin/workspace-identities/global" in admin_template assert 'data-capability-options="file_sync,action"' in admin_template assert "Cloud drive connector identities" in admin_template assert "OneDrive, SharePoint, and Google Workspace File Sync connectors are coming soon" in admin_template assert "admin-approved cloud drive connectors" in identity_js - assert "'smb', 'azure_files', 'onedrive', 'google_drive', 'google_shared_drive'" in identity_js + assert "'smb', 'azure_files', 'azure_blob', 'onedrive', 'google_drive', 'google_shared_drive'" in identity_js def test_file_sync_browse_routes_are_registered(): diff --git a/ui_tests/test_admin_file_sync_settings_ui.py b/ui_tests/test_admin_file_sync_settings_ui.py index 293c45de8..8f5c7e527 100644 --- a/ui_tests/test_admin_file_sync_settings_ui.py +++ b/ui_tests/test_admin_file_sync_settings_ui.py @@ -2,10 +2,14 @@ """ UI test for Admin Settings File Sync management. -Version: 0.241.180 +Version: 0.250.070 Implemented in: 0.241.073 Updated in: 0.241.178 Updated in: 0.241.180 +Updated in: 0.250.067 +Updated in: 0.250.068 +Updated in: 0.250.069 +Updated in: 0.250.070 This test ensures the Admin Settings File Sync tab renders as its own section, uses personal app-role and workspace assignment gate controls, stacks scope cards @@ -143,8 +147,11 @@ def handle_public_workspace_assignment_search(route): expect(file_sync_section.get_by_text("Visible Source Types")).to_be_visible() expect(page.get_by_label("SMB Share")).to_be_visible() expect(page.get_by_label("Azure Files")).to_be_visible() + expect(page.get_by_label("Azure Blob Storage")).to_be_visible() if not page.get_by_label("Azure Files").is_checked(): page.get_by_label("Azure Files").check() + if page.get_by_label("Azure Blob Storage").is_checked(): + page.get_by_label("Azure Blob Storage").uncheck() expect(page.get_by_label("OneDrive")).to_be_visible() expect(page.get_by_label("On-prem SharePoint")).to_be_visible() expect(page.get_by_label("Google Workspace")).to_be_visible() diff --git a/ui_tests/test_workspace_file_sync_ui.py b/ui_tests/test_workspace_file_sync_ui.py index 55c66360c..c1e0eb057 100644 --- a/ui_tests/test_workspace_file_sync_ui.py +++ b/ui_tests/test_workspace_file_sync_ui.py @@ -1,8 +1,12 @@ # test_workspace_file_sync_ui.py """ UI test for workspace File Sync tab. -Version: 0.241.129 +Version: 0.250.070 Implemented in: 0.241.042 +Updated in: 0.250.067 +Updated in: 0.250.068 +Updated in: 0.250.069 +Updated in: 0.250.070 This test ensures the workspace Sync tab renders, loads source rows, opens the source workflow modal, and queues a manual sync without browser console errors. @@ -61,6 +65,34 @@ def test_workspace_file_sync_tab(): "remote_delete_policy": "ignore", "last_run_status": "completed", "last_run_counts": {"queued": 2, "unchanged": 4, "skipped": 1, "failed": 0}, + }, + { + "id": "source-2", + "name": "Research Blob", + "source_type": "azure_blob", + "enabled": True, + "recursive": True, + "connection": { + "account_url": "https://contosodata.blob.core.windows.net", + "container_name": "research", + "blob_prefix": "reports", + "selected_paths": [], + }, + "credentials": {"auth_type": "connection_string", "secret_stored": True}, + "credential_metadata": { + "credential_type": "sas", + "sas_scope": "container", + "permissions": "rld", + "expires_at": "2030-07-28T00:00:00+00:00", + "https_only": True, + "stored_access_policy": False, + "warnings": ["Container SAS includes permissions File Sync does not need: Delete. Read and List are sufficient."], + }, + "filters": {"include_patterns": [], "exclude_patterns": [], "allowed_extensions": [], "fixed_tags": [], "folder_tag_mode": "parent"}, + "schedule": {"enabled": False, "interval_minutes": 60}, + "remote_delete_policy": "ignore", + "last_run_status": None, + "last_run_counts": {}, } ] } @@ -97,6 +129,28 @@ def handle_file_sync(route): if request.method == "GET" and request.url.endswith("/sources"): route.fulfill(status=200, content_type="application/json", body=json.dumps(source_state)) return + if request.method == "POST" and request.url.endswith("/test-connection"): + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({ + "connection": { + "success": True, + "entries_checked": 2, + "read_verified": True, + "credential_metadata": { + "credential_type": "sas", + "sas_scope": "container", + "permissions": "rld", + "expires_at": "2030-07-28T00:00:00+00:00", + "https_only": True, + "stored_access_policy": False, + "warnings": ["Container SAS includes permissions File Sync does not need: Delete. Read and List are sufficient."], + }, + } + }), + ) + return if request.method == "GET" and request.url.endswith("/runs"): route.fulfill( status=200, @@ -202,13 +256,25 @@ def handle_documents(route): expect(identity_view_modal).to_be_hidden() page.locator("#sync-tab-btn").click() + page.locator("#file-sync-root").evaluate("""root => { + root.dataset.visibleSourceTypes = 'smb,azure_files,azure_blob'; + delete root.dataset.fileSyncInitialized; + root.replaceChildren(); + window.initializeFileSyncRoot(root); + }""") expect(page.get_by_role("button", name="Add Source")).to_be_visible() expect(page.get_by_role("button", name="Identities")).to_have_count(0) expect(page.get_by_text("Finance Share")).to_be_visible() expect(page.get_by_text("SMB Share")).to_be_visible() expect(page.get_by_text("queued 2, unchanged 4, skipped 1, failed 0")).to_be_visible() + blob_row = page.locator("tr", has_text="Research Blob") + expect(blob_row.get_by_text("Container SAS")).to_be_visible() + expect(blob_row.get_by_text("Permissions: Read, List, Delete")).to_be_visible() + expect(blob_row.get_by_text(re.compile(r"Expires .*days remaining"))).to_be_visible() + expect(blob_row.get_by_text("Container SAS includes permissions File Sync does not need: Delete. Read and List are sufficient.")).to_be_visible() - page.get_by_role("button", name="Delete").click() + finance_row = page.locator("tr", has_text="Finance Share") + finance_row.get_by_role("button", name="Delete").click() expect(page.get_by_role("heading", name="Delete File Sync Source")).to_be_visible() expect(page.get_by_role("button", name="Delete Sync Source")).to_be_visible() expect(page.get_by_role("button", name="Delete All Files")).to_be_visible() @@ -218,9 +284,15 @@ def handle_documents(route): source_modal = page.locator('[data-file-sync-source-modal="true"]') expect(source_modal.get_by_role("heading", name="Add Sync Source")).to_be_visible() expect(source_modal.get_by_text("SMB Share")).to_be_visible() + expect(source_modal.get_by_text("Azure Blob Storage")).to_be_visible() + source_modal.get_by_role("button", name=re.compile("^Azure Blob Storage")).click() source_modal.get_by_role("button", name="Configure Source").click() expect(source_modal.get_by_text("Source Type")).to_be_visible() - expect(source_modal.get_by_label("UNC path")).to_be_visible() + expect(source_modal.get_by_label("Blob service URL or account name")).to_be_visible() + expect(source_modal.get_by_label("Container name")).to_be_visible() + expect(source_modal.get_by_label("Blob prefix")).to_be_visible() + expect(source_modal.get_by_text("A container SAS needs Read and List only.")).to_be_visible() + expect(source_modal.get_by_text("To sync multiple containers, create one source per container.")).to_be_visible() expect(source_modal.get_by_text("Identity and Authentication")).to_be_visible() expect(source_modal.get_by_label("Reusable identity")).to_be_visible() expect(source_modal.get_by_text("Selection, Subfolders, and Filters")).to_be_visible() @@ -234,13 +306,27 @@ def handle_documents(route): expect(source_modal.get_by_text("Schedule interval minutes")).to_be_hidden() expect(source_modal.locator("#file-sync-recursive")).to_be_visible() expect(source_modal.get_by_role("button", name="Test Connection")).to_be_visible() + source_modal.get_by_label("Authentication").select_option("connection_string") + expect(source_modal.get_by_label("Authentication").locator("option:checked")).to_have_text("Blob connection string or SAS") + source_modal.get_by_label("Blob connection string, SAS URL, or SAS token").fill( + "https://contosodata.blob.core.windows.net/research?sv=fake&spr=https&sr=c&sp=rl&se=2030-07-28T00%3A00%3A00Z&sig=fake" + ) + expect(source_modal.get_by_label("Blob service URL or account name")).to_have_value("https://contosodata.blob.core.windows.net") + expect(source_modal.get_by_label("Container name")).to_have_value("research") + expect(source_modal.get_by_label("Source name")).to_have_value("research Blob Sync") + source_modal.get_by_role("button", name="Test Connection").click() + expect(source_modal.get_by_text("Connection OK. Checked 2 top-level item(s).")).to_be_visible() + expect(source_modal.get_by_text("Container SAS")).to_be_visible() + expect(source_modal.get_by_text("Permissions: Read, List, Delete")).to_be_visible() + expect(source_modal.get_by_text(re.compile(r"Expires .*days remaining"))).to_be_visible() + expect(source_modal.get_by_text("Container SAS includes permissions File Sync does not need: Delete. Read and List are sufficient.")).to_be_visible() source_modal.get_by_role("button", name="Cancel").click() expect(source_modal).to_be_hidden() - page.get_by_role("button", name="History").click() + finance_row.get_by_role("button", name="History").click() expect(page.get_by_role("heading", name="Sync History")).to_be_visible() - page.get_by_role("button", name="Sync").click() + finance_row.get_by_role("button", name="Sync").click() expect(page.get_by_text("Sync run queued.")).to_be_visible() assert console_errors == [] finally: