8360937: Enhance certificate handling

Reviewed-by: fferrari
Backport-of: f2fba5a55176ca82985ca42996cef36be7b7500a

Author: Alexey Bakhtin

src/java.base/share/classes/sun/security/util/DerValue.java

@@ -852,6 +852,22 @@ public String getUniversalString() throws IOException {
         return readStringInternal(tag_UniversalString, new UTF_32BE());
     }
 
+    /**
+     * Checks that the BMPString does not contain any surrogate characters,
+     * which are outside the Basic Multilingual Plane.
+     *
+     * @throws IOException if illegal characters are detected
+     */
+    public void validateBMPString() throws IOException {
+        String bmpString = getBMPString();
+        for (int i = 0; i < bmpString.length(); i++) {
+            if (Character.isSurrogate(bmpString.charAt(i))) {
+                throw new IOException(
+                    "Illegal character in BMPString, index: " + i);
+            }
+         }
+     }
+
     /**
      * Reads the ASN.1 NULL value
      */

src/java.base/share/classes/sun/security/x509/AVA.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,10 +28,13 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.Reader;
+import java.nio.charset.Charset;
 import java.text.Normalizer;
 import java.util.*;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.charset.StandardCharsets.UTF_16BE;
 
 import sun.security.action.GetBooleanAction;
 import sun.security.util.*;
@@ -594,6 +597,10 @@ private static boolean trailingSpace(Reader in) throws IOException {
             throw new IOException("AVA, extra bytes = "
                 + derval.data.available());
         }
+
+        if (value.tag == DerValue.tag_BMPString) {
+            value.validateBMPString();
+        }
     }
 
     AVA(DerInputStream in) throws IOException {
@@ -735,7 +742,7 @@ public String toRFC2253String(Map<String, String> oidMap) {
              */
             String valStr = null;
             try {
-                valStr = new String(value.getDataBytes(), UTF_8);
+                valStr = new String(value.getDataBytes(), getCharset(value, false));
             } catch (IOException ie) {
                 throw new IllegalArgumentException("DER Value conversion");
             }
@@ -870,7 +877,7 @@ public String toRFC2253CanonicalString() {
              */
             String valStr = null;
             try {
-                valStr = new String(value.getDataBytes(), UTF_8);
+                valStr = new String(value.getDataBytes(), getCharset(value, true));
             } catch (IOException ie) {
                 throw new IllegalArgumentException("DER Value conversion");
             }
@@ -977,6 +984,39 @@ private static boolean isDerString(DerValue value, boolean canonical) {
         }
     }
 
+    /*
+     * Returns the charset that should be used to decode each DN string type.
+     *
+     * This method ensures that multi-byte (UTF8String and BMPString) types
+     * are decoded using the correct charset and the String forms represent
+     * the correct characters. For 8-bit ASCII-based types (PrintableString
+     * and IA5String), we return ISO_8859_1 rather than ASCII, so that the
+     * complete range of characters can be represented, as many certificates
+     * do not comply with the Internationalized Domain Name ACE format.
+     *
+     * NOTE: this method only supports DirectoryStrings of the types returned
+     * by isDerString().
+     */
+    private static Charset getCharset(DerValue value, boolean canonical) {
+        if (canonical) {
+            return switch (value.tag) {
+                case DerValue.tag_PrintableString -> ISO_8859_1;
+                case DerValue.tag_UTF8String -> UTF_8;
+                default -> throw new Error("unexpected tag: " + value.tag);
+            };
+        }
+
+        return switch (value.tag) {
+            case DerValue.tag_PrintableString,
+                 DerValue.tag_T61String,
+                 DerValue.tag_IA5String,
+                 DerValue.tag_GeneralString -> ISO_8859_1;
+            case DerValue.tag_BMPString -> UTF_16BE;
+            case DerValue.tag_UTF8String -> UTF_8;
+            default -> throw new Error("unexpected tag: " + value.tag);
+        };
+    }
+
     boolean hasRFC2253Keyword() {
         return AVAKeyword.hasKeyword(oid, RFC2253);
     }

test/lib/jdk/test/lib/security/CertificateBuilder.java

@@ -54,6 +54,7 @@
 import sun.security.x509.SubjectAlternativeNameExtension;
 import sun.security.x509.URIName;
 import sun.security.x509.KeyIdentifier;
+import sun.security.x509.X500Name;
 
 /**
  * Helper class that builds and signs X.509 certificates.
@@ -90,7 +91,7 @@
 public class CertificateBuilder {
     private final CertificateFactory factory;
 
-    private X500Principal subjectName = null;
+    private X500Name subjectName = null;
     private BigInteger serialNumber = null;
     private PublicKey publicKey = null;
     private Date notBefore = null;
@@ -116,7 +117,7 @@ public CertificateBuilder() throws CertificateException {
      * on this certificate.
      */
     public CertificateBuilder setSubjectName(X500Principal name) {
-        subjectName = name;
+        subjectName = X500Name.asX500Name(name);
         return this;
     }
 
@@ -126,7 +127,23 @@ public CertificateBuilder setSubjectName(X500Principal name) {
      * @param name The subject name in RFC 2253 format
      */
     public CertificateBuilder setSubjectName(String name) {
-        subjectName = new X500Principal(name);
+        try {
+            subjectName = new X500Name(name);
+        } catch (IOException ioe) {
+            throw new IllegalArgumentException(ioe);
+        }
+        return this;
+    }
+
+    /**
+     * Set the subject name for the certificate. This method is useful when
+     * you need more control over the contents of the subject name.
+     *
+     * @param name an {@code X500Name} to be used as the subject name
+     * on this certificate
+     */
+    public CertificateBuilder setSubjectName(X500Name name) {
+        subjectName = name;
         return this;
     }