Add support for Azure managed identity

docs/azure.md

@@ -418,6 +418,34 @@ azure {
 }
 ```
 
+(azure-managed-identities)=
+
+## Managed identities
+
+:::{versionadded} 24.03.0-edge
+:::
+
+An Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) can be used to authenticate with Azure Blob storage without using secrets such as a storage account key.
+
+The managed identity can be specified as follows:
+
+```groovy
+azure {
+    managedIdentity {
+        clientId = '<YOUR MANAGED IDENTITY>'
+    }
+
+    storage {
+        accountName = '<YOUR STORAGE ACCOUNT NAME>'
+    }
+
+    batch {
+        accountName = '<YOUR BATCH ACCOUNT NAME>'
+        location = '<YOUR BATCH ACCOUNT LOCATION>'
+    }
+}
+```
+
 ## Advanced configuration
 
 Read the {ref}`Azure configuration<config-azure>` section to learn more about advanced configuration options.

docs/config.md

@@ -373,29 +373,24 @@ The following settings are available:
 : Enable autoscaling feature for the pool identified with `<name>`.
 
 `azure.batch.pools.<name>.fileShareRootPath`
-: *New in `nf-azure` version `0.11.0`*
 : If mounting File Shares, this is the internal root mounting point. Must be `/mnt/resource/batch/tasks/fsmounts` for CentOS nodes or `/mnt/batch/tasks/fsmounts` for Ubuntu nodes (default is for CentOS).
 
 `azure.batch.pools.<name>.lowPriority`
-: *New in `nf-azure` version `1.4.0`*
 : Enable the use of low-priority VMs (default: `false`).
 
 `azure.batch.pools.<name>.maxVmCount`
 : Specify the max of virtual machine when using auto scale option.
 
 `azure.batch.pools.<name>.mountOptions`
-: *New in `nf-azure` version `0.11.0`*
 : Specify the mount options for mounting the file shares (default: `-o vers=3.0,dir_mode=0777,file_mode=0777,sec=ntlmssp`).
 
 `azure.batch.pools.<name>.offer`
-: *New in `nf-azure` version `0.11.0`*
 : Specify the offer type of the virtual machine type used by the pool identified with `<name>` (default: `centos-container`).
 
 `azure.batch.pools.<name>.privileged`
 : Enable the task to run with elevated access. Ignored if `runAs` is set (default: `false`).
 
 `azure.batch.pools.<name>.publisher`
-: *New in `nf-azure` version `0.11.0`*
 : Specify the publisher of virtual machine type used by the pool identified with `<name>` (default: `microsoft-azure-batch`).
 
 `azure.batch.pools.<name>.runAs`
@@ -411,7 +406,6 @@ The following settings are available:
 : Specify the scheduling policy for the pool identified with `<name>`. It can be either `spread` or `pack` (default: `spread`).
 
 `azure.batch.pools.<name>.sku`
-: *New in `nf-azure` version `0.11.0`*
 : Specify the ID of the Compute Node agent SKU which the pool identified with `<name>` supports (default: `batch.node.centos 8`).
 
 `azure.batch.pools.<name>.startTask.script`
@@ -435,16 +429,16 @@ The following settings are available:
 `azure.batch.pools.<name>.vmType`
 : Specify the virtual machine type used by the pool identified with `<name>`.
 
+`azure.managedIdentity.clientId`
+: Specify the client ID for an Azure [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview). See {ref}`azure-managed-identities` for more details.
+
 `azure.registry.server`
-: *New in `nf-azure` version `0.9.8`*
 : Specify the container registry from which to pull the Docker images (default: `docker.io`).
 
 `azure.registry.userName`
-: *New in `nf-azure` version `0.9.8`*
 : Specify the username to connect to a private container registry.
 
 `azure.registry.password`
-: *New in `nf-azure` version `0.9.8`*
 : Specify the password to connect to a private container registry.
 
 `azure.retryPolicy.delay`

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy

@@ -269,24 +269,20 @@ class AzBatchService implements Closeable {
     }
 
 
-    protected createBatchCredentialsWithKey() {
+    protected BatchCredentials createBatchCredentialsWithKey() {
         log.debug "[AZURE BATCH] Creating Azure Batch client using shared key creddentials"
 
-        if (config.batch().endpoint || config.batch().accountKey || config.batch().accountName) {
-            // Create batch client
-            if (!config.batch().endpoint)
-                throw new IllegalArgumentException("Missing Azure Batch endpoint -- Specify it in the nextflow.config file using the setting 'azure.batch.endpoint'")
-            if (!config.batch().accountName)
-                throw new IllegalArgumentException("Missing Azure Batch account name -- Specify it in the nextflow.config file using the setting 'azure.batch.accountName'")
-            if (!config.batch().accountKey)
-                throw new IllegalArgumentException("Missing Azure Batch account key -- Specify it in the nextflow.config file using the setting 'azure.batch.accountKey'")
+        if( !config.batch().endpoint )
+            throw new IllegalArgumentException("Missing Azure Batch endpoint -- Specify it in the nextflow.config file using the setting 'azure.batch.endpoint'")
+        if( !config.batch().accountName )
+            throw new IllegalArgumentException("Missing Azure Batch account name -- Specify it in the nextflow.config file using the setting 'azure.batch.accountName'")
+        if( !config.batch().accountKey )
+            throw new IllegalArgumentException("Missing Azure Batch account key -- Specify it in the nextflow.config file using the setting 'azure.batch.accountKey'")
 
-            return new BatchSharedKeyCredentials(config.batch().endpoint, config.batch().accountName, config.batch().accountKey)
-
-        }
+        return new BatchSharedKeyCredentials(config.batch().endpoint, config.batch().accountName, config.batch().accountKey)
     }
 
-    protected createBatchCredentialsWithServicePrincipal() {
+    protected BatchCredentials createBatchCredentialsWithServicePrincipal() {
         log.debug "[AZURE BATCH] Creating Azure Batch client using Service Principal credentials"
 
         final batchEndpoint = "https://batch.core.windows.net/";
@@ -307,12 +303,15 @@ class AzBatchService implements Closeable {
     protected BatchClient createBatchClient() {
         log.debug "[AZURE BATCH] Executor options=${config.batch()}"
 
-        def cred = config.activeDirectory().isConfigured()
-                ? createBatchCredentialsWithServicePrincipal()
-                : createBatchCredentialsWithKey()
+        BatchCredentials cred
+
+        if( config.activeDirectory().isConfigured() )
+            cred = createBatchCredentialsWithServicePrincipal()
+        else if( config.batch().endpoint || config.batch().accountKey || config.batch().accountName )
+            cred = createBatchCredentialsWithKey()
 
         // Create batch client
-        def client = BatchClient.open(cred as BatchCredentials)
+        def client = BatchClient.open(cred)
 
         Global.onCleanup((it)->client.protocolLayer().restClient().close())
 

plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzHelper.groovy

@@ -18,6 +18,8 @@ package nextflow.cloud.azure.batch
 import java.nio.file.Path
 import java.time.OffsetDateTime
 
+import com.azure.identity.ClientSecretCredentialBuilder
+import com.azure.identity.DefaultAzureCredentialBuilder
 import com.azure.storage.blob.BlobContainerClient
 import com.azure.storage.blob.BlobServiceClient
 import com.azure.storage.blob.BlobServiceClientBuilder
@@ -130,7 +132,7 @@ class AzHelper {
         return delegationKey
     }
 
-        static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
+    static String generateContainerUserDelegationSas(BlobContainerClient client, Duration duration, UserDelegationKey key) {
 
         final startTime = OffsetDateTime.now()
         final indicatedExpiryTime = startTime.plusHours(duration.toHours())
@@ -153,7 +155,7 @@ class AzHelper {
     }
 
     static String generateAccountSas(BlobServiceClient client, Duration duration) {
-        final expiryTime = OffsetDateTime.now().plusSeconds(duration.toSeconds());
+        final expiryTime = OffsetDateTime.now().plusSeconds(duration.toSeconds())
         final signature = new AccountSasSignatureValues(
                 expiryTime,
                 ACCOUNT_PERMS,
@@ -172,8 +174,8 @@ class AzHelper {
     static synchronized BlobServiceClient getOrCreateBlobServiceWithKey(String accountName, String accountKey) {
         log.debug "Creating Azure blob storage client -- accountName=$accountName; accountKey=${accountKey?.substring(0,5)}.."
 
-        final credential = new StorageSharedKeyCredential(accountName, accountKey);
-        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
+        final credential = new StorageSharedKeyCredential(accountName, accountKey)
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
 
         return new BlobServiceClientBuilder()
                 .endpoint(endpoint)
@@ -190,12 +192,49 @@ class AzHelper {
 
         log.debug "Creating Azure blob storage client -- accountName: $accountName; sasToken: ${sasToken?.substring(0,10)}.."
 
-        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName);
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
 
         return new BlobServiceClientBuilder()
                 .endpoint(endpoint)
                 .sasToken(sasToken)
                 .buildClient()
     }
 
+    @Memoized
+    static synchronized BlobServiceClient getOrCreateBlobServiceWithServicePrincipal(String accountName, String clientId, String clientSecret, String tenantId) {
+        log.debug "Creating Azure Blob storage client using Service Principal credentials"
+
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
+
+        final credential = new ClientSecretCredentialBuilder()
+                .clientId(clientId)
+                .clientSecret(clientSecret)
+                .tenantId(tenantId)
+                .build()
+
+        return new BlobServiceClientBuilder()
+                .credential(credential)
+                .endpoint(endpoint)
+                .buildClient()
+    }
+
+    @Memoized
+    static synchronized BlobServiceClient getOrCreateBlobServiceWithManagedIdentity(String accountName, String clientId) {
+        if( !clientId )
+            throw new IllegalArgumentException("Missing Azure blob managed identity client ID")
+
+        log.debug "Creating Azure blob storage client -- accountName: $accountName; clientId: ${clientId}"
+
+        final endpoint = String.format(Locale.ROOT, "https://%s.blob.core.windows.net", accountName)
+
+        final credential = new DefaultAzureCredentialBuilder()
+                .managedIdentityClientId(clientId)
+                .build()
+
+        return new BlobServiceClientBuilder()
+                .credential(credential)
+                .endpoint(endpoint)
+                .buildClient()
+    }
+
 }

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzActiveDirectoryOpts.groovy

@@ -19,7 +19,7 @@ import groovy.transform.CompileStatic
 import nextflow.cloud.azure.nio.AzFileSystemProvider
 
 /**
- * Model Azure identity options from nextflow config file
+ * Model Azure Entra (formerly Active Directory) config options
  *
  * @author Abhinav Sharma <abhi18av@outlook.com>
  */

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzConfig.groovy

@@ -40,13 +40,16 @@ class AzConfig {
 
     private AzActiveDirectoryOpts activeDirectoryOpts
 
+    private AzManagedIdentityOpts managedIdentityOpts
+
     AzConfig(Map azure) {
         this.batchOpts = new AzBatchOpts( (Map)azure.batch ?: Collections.emptyMap() )
         this.storageOpts = new AzStorageOpts( (Map)azure.storage ?: Collections.emptyMap() )
         this.registryOpts = new AzRegistryOpts( (Map)azure.registry ?: Collections.emptyMap() )
         this.azcopyOpts = new AzCopyOpts( (Map)azure.azcopy ?: Collections.emptyMap() )
         this.retryConfig = new AzRetryConfig( (Map)azure.retryPolicy ?: Collections.emptyMap() )
         this.activeDirectoryOpts = new AzActiveDirectoryOpts((Map) azure.activeDirectory ?: Collections.emptyMap())
+        this.managedIdentityOpts = new AzManagedIdentityOpts((Map) azure.managedIdentity ?: Collections.emptyMap())
     }
 
     AzCopyOpts azcopy() { azcopyOpts }
@@ -61,6 +64,8 @@ class AzConfig {
 
     AzActiveDirectoryOpts activeDirectory() { activeDirectoryOpts }
 
+    AzManagedIdentityOpts managedIdentity() { managedIdentityOpts }
+
     static AzConfig getConfig(Session session) {
         if( !session )
             throw new IllegalStateException("Missing Nextflow session")

plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzManagedIdentityOpts.groovy

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nextflow.cloud.azure.config
+
+import groovy.transform.CompileStatic
+import nextflow.cloud.azure.nio.AzFileSystemProvider
+
+/**
+ * Model Azure managed identity config options
+ *
+ * @author Ben Sherman <bentshermann@gmail.com>
+ */
+@CompileStatic
+class AzManagedIdentityOpts {
+
+    String clientId
+
+    AzManagedIdentityOpts(Map config) {
+        assert config != null
+        this.clientId = config.clientId
+    }
+
+    Map<String, Object> getEnv() {
+        Map<String, Object> props = new HashMap<>();
+        props.put(AzFileSystemProvider.AZURE_MANAGED_IDENTITY, clientId)
+        return props
+    }
+
+}

plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzPathFactory.groovy

@@ -49,10 +49,10 @@ class AzPathFactory extends FileSystemPathFactory {
             throw new IllegalArgumentException("Invalid Azure path URI - make sure the schema prefix does not container more than two slash characters - offending value: $uri")
 
         final storageConfigEnv = AzConfig.getConfig().storage().getEnv()
-
         final activeDirectoryConfigEnv = AzConfig.getConfig().activeDirectory().getEnv()
+        final managedIdentityConfigEnv = AzConfig.getConfig().managedIdentity().getEnv()
 
-        final configEnv = storageConfigEnv + activeDirectoryConfigEnv
+        final configEnv = storageConfigEnv + activeDirectoryConfigEnv + managedIdentityConfigEnv
 
         // find the related file system
         final fs = getFileSystem(uri0(uri), configEnv)