From 6b3beeca3b6ffe8078c367570e1845879ba2d514 Mon Sep 17 00:00:00 2001
From: Matthieu Gallien <matthieu.gallien@nextcloud.com>
Date: Tue, 22 Aug 2023 22:08:19 +0200
Subject: [PATCH] fixed the decryption issue with hardware token being used

switch to another encryption method (not sure this is really necessary)

fixed missing base64 conversion

fixed the check for decrypted length

Signed-off-by: Matthieu Gallien <matthieu.gallien@nextcloud.com>
---
 src/libsync/clientsideencryption.cpp | 59 +++++++++++++++++++++++-----
 1 file changed, 50 insertions(+), 9 deletions(-)

diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp
index 7797d4b1e0d7d..c166a7ceffc5f 100644
--- a/src/libsync/clientsideencryption.cpp
+++ b/src/libsync/clientsideencryption.cpp
@@ -665,6 +665,9 @@ namespace internals {
                                                                 EVP_PKEY *publicKey,
                                                                 const QByteArray& binaryData);
 
+[[nodiscard]] std::optional<QByteArray> encryptStringAsymmetricWithToken(EVP_PKEY *publicKey,
+                                                                         const QByteArray& binaryData);
+
 [[nodiscard]] std::optional<QByteArray> decryptStringAsymmetric(ENGINE *sslEngine,
                                                                 EVP_PKEY *privateKey,
                                                                 const QByteArray& binaryData);
@@ -684,17 +687,11 @@ std::optional<QByteArray> encryptStringAsymmetric(const ClientSideEncryption &en
             qCWarning(lcCseEncryption()) << "encryptStringAsymmetric" << "hardware public key" << publicKey;
         }
 
-        {
-            Bio publicKeyBio;
-            PEM_write_bio_PKCS8PrivateKey(publicKeyBio, publicKey, nullptr, nullptr, 0, nullptr, nullptr);
-            const auto publicKeyBase64 = BIO2ByteArray(publicKeyBio).toBase64();
-            qCInfo(lcCseDecryption()) << "decryptStringAsymmetric" << "hardware public key to base64" << publicKeyBase64;
-        }
-
         qCInfo(lcCseEncryption()) << "encryptStringAsymmetric:"
                                   << "data:" << binaryData.toBase64();
 
-        auto encryptedBase64Result = internals::encryptStringAsymmetric(encryptionEngine.sslEngine(), publicKey, binaryData);
+        auto encryptedBase64Result = internals::encryptStringAsymmetricWithToken(PKCS11_get_public_key(encryptionEngine.getTokenPublicKey()),
+                                                                                 binaryData);
 
         if (!encryptedBase64Result) {
             qCWarning(lcCseEncryption()) << "encrypt failed";
@@ -1008,6 +1005,39 @@ void debugOpenssl()
 
 namespace internals {
 
+std::optional<QByteArray> encryptStringAsymmetricWithToken(EVP_PKEY *publicKey,
+                                                           const QByteArray& binaryData)
+{
+    const auto initialLength = RSA_size(EVP_PKEY_get0_RSA(publicKey));
+    auto out = QByteArray{static_cast<int>(initialLength), 0};
+
+    const auto rc = RSA_public_encrypt(binaryData.size(),
+                                       reinterpret_cast<const unsigned char*>(binaryData.data()),
+                                       reinterpret_cast<unsigned char*>(out.data()),
+                                       const_cast<RSA*>(EVP_PKEY_get0_RSA(publicKey)),
+                                       RSA_PKCS1_PADDING);
+
+    if (rc != initialLength) {
+        qCInfo(lcCseEncryption()) << "data to encrypt:" << binaryData.toBase64()
+                                  << "size:" << binaryData.size()
+                                  << "key size:" << initialLength;
+        qCCritical(lcCseEncryption()) << "RSA_public_encrypt failed"
+                                      << "expected encrypted length" << initialLength
+                                      << "result" << rc << ERR_reason_error_string(ERR_get_error());
+
+        return {};
+    } else {
+        qCInfo(lcCseEncryption()) << "data to encrypt:" << binaryData.toBase64()
+                                  << "size:" << binaryData.size()
+                                  << "key size:" << initialLength;
+        qCInfo(lcCseEncryption()) << "data encrypted:" << out.toBase64()
+                                  << "size:" << out.size()
+                                  << "key size:" << initialLength;
+    }
+
+    return out.toBase64();
+}
+
 std::optional<QByteArray> decryptStringAsymmetricWithToken(PKCS11_KEY *privateKey,
                                                            EVP_PKEY *publicKey,
                                                            const QByteArray &binaryData)
@@ -1020,7 +1050,11 @@ std::optional<QByteArray> decryptStringAsymmetricWithToken(PKCS11_KEY *privateKe
                                            reinterpret_cast<unsigned char*>(decryptedData.data()),
                                            privateKey,
                                            RSA_PKCS1_PADDING);
-    if (rc != initialLength) {
+
+    if (rc < 0) {
+        qCInfo(lcCseDecryption()) << "data to decrypt:" << binaryData.toBase64()
+                                  << "size:" << binaryData.size()
+                                  << "key size:" << initialLength;
         qCCritical(lcCseDecryption()) << "PKCS11_private_decrypt failed"
                                       << "expected decrypted length" << initialLength
                                       << "result" << rc << ERR_reason_error_string(ERR_get_error());
@@ -1030,6 +1064,13 @@ std::optional<QByteArray> decryptStringAsymmetricWithToken(PKCS11_KEY *privateKe
                                   << "need login:" << (privateKey->needLogin ? "true" : "false");
 
         return {};
+    } else {
+        qCInfo(lcCseDecryption()) << "data to decrypt:" << binaryData.toBase64()
+                                  << "size:" << binaryData.size()
+                                  << "key size:" << initialLength;
+        qCInfo(lcCseDecryption()) << "data decrypted:" << decryptedData.toBase64()
+                                  << "size:" << decryptedData.size()
+                                  << "key size:" << initialLength;
     }
 
     qCInfo(lcCseDecryption()) << "result to base64" << decryptedData.toBase64();