[UIAM] Cloud API key authentication

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -286,6 +286,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ILM_ADD_SKIP_SETTING = def(9_089_0_00);
     public static final TransportVersion ML_INFERENCE_MISTRAL_CHAT_COMPLETION_ADDED = def(9_090_0_00);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST = def(9_091_0_00);
+    public static final TransportVersion SECURITY_CLOUD_API_KEY_REALM_AND_TYPE = def(9_092_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/module-info.java

@@ -230,6 +230,7 @@
     exports org.elasticsearch.xpack.core.watcher.trigger;
     exports org.elasticsearch.xpack.core.watcher.watch;
     exports org.elasticsearch.xpack.core.watcher;
+    exports org.elasticsearch.xpack.core.security.authc.apikey;
 
     provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber
         with

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authc.apikey.CustomApiKeyAuthenticator;
 import org.elasticsearch.xpack.core.security.authc.service.NodeLocalServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
@@ -128,6 +129,10 @@ default ServiceAccountTokenStore getServiceAccountTokenStore(SecurityComponents
         return null;
     }
 
+    default CustomApiKeyAuthenticator getCustomApiKeyAuthenticator(SecurityComponents components) {
+        return null;
+    }
+
     /**
      * Returns a authorization engine for authorizing requests, or null to use the default authorization mechanism.
      *

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -61,6 +61,7 @@
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newAnonymousRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newApiKeyRealmRef;
+import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newCloudApiKeyRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newCrossClusterAccessRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newInternalAttachRealmRef;
 import static org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef.newInternalFallbackRealmRef;
@@ -71,6 +72,8 @@
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.ATTACH_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.ATTACH_REALM_TYPE;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.CLOUD_API_KEY_REALM_NAME;
+import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.CLOUD_API_KEY_REALM_TYPE;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.CROSS_CLUSTER_ACCESS_AUTHENTICATION_KEY;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.CROSS_CLUSTER_ACCESS_REALM_NAME;
 import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.CROSS_CLUSTER_ACCESS_REALM_TYPE;
@@ -264,6 +267,16 @@ public Authentication maybeRewriteForOlderVersion(TransportVersion olderVersion)
                     + "]"
             );
         }
+        if (isCloudApiKey() && olderVersion.before(TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE)) {
+            throw new IllegalArgumentException(
+                "versions of Elasticsearch before ["
+                    + TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE.toReleaseVersion()
+                    + "] can't handle cloud API key authentication and attempted to rewrite for ["
+                    + olderVersion.toReleaseVersion()
+                    + "]"
+            );
+        }
+
         final Map<String, Object> newMetadata = maybeRewriteMetadata(olderVersion, this);
 
         final Authentication newAuthentication;
@@ -528,6 +541,10 @@ public boolean isApiKey() {
         return effectiveSubject.getType() == Subject.Type.API_KEY;
     }
 
+    public boolean isCloudApiKey() {
+        return effectiveSubject.getType() == Subject.Type.CLOUD_API_KEY;
+    }
+
     public boolean isCrossClusterAccess() {
         return effectiveSubject.getType() == Subject.Type.CROSS_CLUSTER_ACCESS;
     }
@@ -625,6 +642,16 @@ private static void doWriteTo(Subject effectiveSubject, Subject authenticatingSu
                     + "]"
             );
         }
+        if (effectiveSubject.getType() == Subject.Type.CLOUD_API_KEY
+            && out.getTransportVersion().before(TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE)) {
+            throw new IllegalArgumentException(
+                "versions of Elasticsearch before ["
+                    + TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE.toReleaseVersion()
+                    + "] can't handle cloud API key authentication and attempted to send to ["
+                    + out.getTransportVersion().toReleaseVersion()
+                    + "]"
+            );
+        }
         final boolean isRunAs = authenticatingSubject != effectiveSubject;
         if (isRunAs) {
             final User outerUser = effectiveSubject.getUser();
@@ -772,6 +799,7 @@ public void toXContentFragment(XContentBuilder builder) throws IOException {
                 builder.field("api_key", Map.of("id", apiKeyId, "name", apiKeyName));
             }
         }
+        // TODO cloud API key fields such as managed_by
     }
 
     public static Authentication getAuthenticationFromCrossClusterAccessMetadata(Authentication authentication) {
@@ -891,11 +919,17 @@ private void checkConsistencyForInternalAuthenticationType() {
 
     private void checkConsistencyForApiKeyAuthenticationType() {
         final RealmRef authenticatingRealm = authenticatingSubject.getRealm();
-        if (false == authenticatingRealm.isApiKeyRealm() && false == authenticatingRealm.isCrossClusterAccessRealm()) {
+        if (false == authenticatingRealm.isApiKeyRealm()
+            && false == authenticatingRealm.isCrossClusterAccessRealm()
+            && false == authenticatingRealm.isCloudApiKeyRealm()) {
             throw new IllegalArgumentException(
                 Strings.format("API key authentication cannot have realm type [%s]", authenticatingRealm.type)
             );
         }
+        if (authenticatingRealm.isCloudApiKeyRealm()) {
+            // TODO consistency check for cloud API keys
+            return;
+        }
         checkConsistencyForApiKeyAuthenticatingSubject("API key");
         if (Subject.Type.CROSS_CLUSTER_ACCESS == authenticatingSubject.getType()) {
             if (authenticatingSubject.getMetadata().get(CROSS_CLUSTER_ACCESS_AUTHENTICATION_KEY) == null) {
@@ -1183,6 +1217,10 @@ private boolean isAnonymousRealm() {
             return ANONYMOUS_REALM_NAME.equals(name) && ANONYMOUS_REALM_TYPE.equals(type);
         }
 
+        private boolean isCloudApiKeyRealm() {
+            return CLOUD_API_KEY_REALM_NAME.equals(name) && CLOUD_API_KEY_REALM_TYPE.equals(type);
+        }
+
         private boolean isApiKeyRealm() {
             return API_KEY_REALM_NAME.equals(name) && API_KEY_REALM_TYPE.equals(type);
         }
@@ -1212,6 +1250,11 @@ static RealmRef newServiceAccountRealmRef(String nodeName) {
             return new Authentication.RealmRef(ServiceAccountSettings.REALM_NAME, ServiceAccountSettings.REALM_TYPE, nodeName, null);
         }
 
+        static RealmRef newCloudApiKeyRealmRef(String nodeName) {
+            // no domain for cloud API key tokens
+            return new RealmRef(CLOUD_API_KEY_REALM_NAME, CLOUD_API_KEY_REALM_TYPE, nodeName, null);
+        }
+
         static RealmRef newApiKeyRealmRef(String nodeName) {
             // no domain for API Key tokens
             return new RealmRef(API_KEY_REALM_NAME, API_KEY_REALM_TYPE, nodeName, null);
@@ -1293,6 +1336,16 @@ public static Authentication newRealmAuthentication(User user, RealmRef realmRef
         return authentication;
     }
 
+    public static Authentication newCloudApiKeyAuthentication(AuthenticationResult<User> authResult, String nodeName) {
+        assert authResult.isAuthenticated() : "cloud API Key authn result must be successful";
+        final User apiKeyUser = authResult.getValue();
+        final Authentication.RealmRef authenticatedBy = newCloudApiKeyRealmRef(nodeName);
+        return new Authentication(
+            new Subject(apiKeyUser, authenticatedBy, TransportVersion.current(), authResult.getMetadata()),
+            AuthenticationType.API_KEY
+        );
+    }
+
     public static Authentication newApiKeyAuthentication(AuthenticationResult<User> authResult, String nodeName) {
         assert authResult.isAuthenticated() : "API Key authn result must be successful";
         final User apiKeyUser = authResult.getValue();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java

@@ -13,6 +13,9 @@ public final class AuthenticationField {
     public static final String PRIVILEGE_CATEGORY_VALUE_OPERATOR = "operator";
     public static final String PRIVILEGE_CATEGORY_VALUE_EMPTY = "__empty";
 
+    public static final String CLOUD_API_KEY_REALM_NAME = "_cloud_api_key";
+    public static final String CLOUD_API_KEY_REALM_TYPE = "_cloud_api_key";
+
     public static final String API_KEY_REALM_NAME = "_es_api_key";
     public static final String API_KEY_REALM_TYPE = "_es_api_key";
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java

@@ -47,6 +47,7 @@ public class Subject {
     public enum Type {
         USER,
         API_KEY,
+        CLOUD_API_KEY,
         SERVICE_ACCOUNT,
         CROSS_CLUSTER_ACCESS,
     }
@@ -72,6 +73,9 @@ public Subject(User user, Authentication.RealmRef realm, TransportVersion versio
         } else if (AuthenticationField.API_KEY_REALM_TYPE.equals(realm.getType())) {
             assert AuthenticationField.API_KEY_REALM_NAME.equals(realm.getName()) : "api key realm name mismatch";
             this.type = Type.API_KEY;
+        } else if (AuthenticationField.CLOUD_API_KEY_REALM_TYPE.equals(realm.getType())) {
+            assert AuthenticationField.CLOUD_API_KEY_REALM_NAME.equals(realm.getName()) : "cloud api key realm name mismatch";
+            this.type = Type.CLOUD_API_KEY;
         } else if (ServiceAccountSettings.REALM_TYPE.equals(realm.getType())) {
             assert ServiceAccountSettings.REALM_NAME.equals(realm.getName()) : "service account realm name mismatch";
             this.type = Type.SERVICE_ACCOUNT;
@@ -105,19 +109,12 @@ public TransportVersion getTransportVersion() {
     }
 
     public RoleReferenceIntersection getRoleReferenceIntersection(@Nullable AnonymousUser anonymousUser) {
-        switch (type) {
-            case USER:
-                return buildRoleReferencesForUser(anonymousUser);
-            case API_KEY:
-                return buildRoleReferencesForApiKey();
-            case SERVICE_ACCOUNT:
-                return new RoleReferenceIntersection(new RoleReference.ServiceAccountRoleReference(user.principal()));
-            case CROSS_CLUSTER_ACCESS:
-                return buildRoleReferencesForCrossClusterAccess();
-            default:
-                assert false : "unknown subject type: [" + type + "]";
-                throw new IllegalStateException("unknown subject type: [" + type + "]");
-        }
+        return switch (type) {
+            case CLOUD_API_KEY, USER -> buildRoleReferencesForUser(anonymousUser);
+            case API_KEY -> buildRoleReferencesForApiKey();
+            case SERVICE_ACCOUNT -> new RoleReferenceIntersection(new RoleReference.ServiceAccountRoleReference(user.principal()));
+            case CROSS_CLUSTER_ACCESS -> buildRoleReferencesForCrossClusterAccess();
+        };
     }
 
     public boolean canAccessResourcesOf(Subject resourceCreatorSubject) {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/apikey/CustomApiKeyAuthenticator.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.apikey;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+
+/**
+ * An extension point to provide a custom API key authenticator implementation.
+ * The implementation is wrapped by a core `Authenticator` class and included in the authenticator chain _before_ the
+ * default API key authenticator.
+ */
+public interface CustomApiKeyAuthenticator {
+    String name();
+
+    AuthenticationToken extractCredentials(@Nullable SecureString apiKeyCredentials);
+
+    void authenticate(@Nullable AuthenticationToken authenticationToken, ActionListener<AuthenticationResult<Authentication>> listener);
+
+    /**
+     * A no-op implementation of {@link CustomApiKeyAuthenticator} that is effectively skipped in the authenticator chain.
+     */
+    class Noop implements CustomApiKeyAuthenticator {
+        @Override
+        public String name() {
+            return "noop";
+        }
+
+        @Override
+        public AuthenticationToken extractCredentials(@Nullable SecureString apiKeyCredentials) {
+            return null;
+        }
+
+        @Override
+        public void authenticate(
+            @Nullable AuthenticationToken authenticationToken,
+            ActionListener<AuthenticationResult<Authentication>> listener
+        ) {
+            listener.onResponse(AuthenticationResult.notHandled());
+        }
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/xcontent/XContentUtils.java

@@ -128,6 +128,9 @@ private static void addSubjectInfo(XContentBuilder builder, Subject subject) thr
                 }
                 builder.endObject();
             }
+            case CLOUD_API_KEY -> {
+                // TODO Add cloud API key information here
+            }
         }
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java

@@ -115,6 +115,52 @@ public void testWriteToWithCrossClusterAccessThrowsOnUnsupportedVersion() throws
         }
     }
 
+    public void testWriteToAndReadFromWithCloudApiKeyAuthentication() throws Exception {
+        final Authentication authentication = Authentication.newCloudApiKeyAuthentication(
+            AuthenticationResult.success(new User(randomAlphanumericOfLength(5), "superuser"), Map.of()),
+            randomAlphanumericOfLength(10)
+        );
+
+        assertThat(authentication.isCloudApiKey(), is(true));
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        authentication.writeTo(output);
+        final Authentication readFrom = new Authentication(output.bytes().streamInput());
+        assertThat(readFrom.isCloudApiKey(), is(true));
+
+        assertThat(readFrom, not(sameInstance(authentication)));
+        assertThat(readFrom, equalTo(authentication));
+    }
+
+    public void testWriteToWithCloudApiKeyThrowsOnUnsupportedVersion() {
+        final Authentication authentication = Authentication.newCloudApiKeyAuthentication(
+            AuthenticationResult.success(new User(randomAlphanumericOfLength(5), "superuser"), Map.of()),
+            randomAlphanumericOfLength(10)
+        );
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            final TransportVersion version = TransportVersionUtils.randomVersionBetween(
+                random(),
+                TransportVersions.V_8_0_0,
+                TransportVersionUtils.getPreviousVersion(TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE)
+            );
+            out.setTransportVersion(version);
+
+            final var ex = expectThrows(IllegalArgumentException.class, () -> authentication.writeTo(out));
+            assertThat(
+                ex.getMessage(),
+                containsString(
+                    "versions of Elasticsearch before ["
+                        + TransportVersions.SECURITY_CLOUD_API_KEY_REALM_AND_TYPE.toReleaseVersion()
+                        + "] can't handle cloud API key authentication and attempted to send to ["
+                        + out.getTransportVersion().toReleaseVersion()
+                        + "]"
+                )
+            );
+        }
+
+    }
+
     public void testSystemUserReadAndWrite() throws Exception {
         BytesStreamOutput output = new BytesStreamOutput();
 

x-pack/plugin/security/src/main/java/module-info.java

@@ -69,7 +69,7 @@
     exports org.elasticsearch.xpack.security.action.settings to org.elasticsearch.server;
     exports org.elasticsearch.xpack.security.operator to org.elasticsearch.internal.operator, org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.authz to org.elasticsearch.internal.security;
-    exports org.elasticsearch.xpack.security.authc to org.elasticsearch.xcontent;
+    exports org.elasticsearch.xpack.security.authc to org.elasticsearch.xcontent, org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.authc.saml to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.slowlog to org.elasticsearch.server;
     exports org.elasticsearch.xpack.security.authc.support to org.elasticsearch.internal.security;