diff --git a/CHANGELOG b/CHANGELOG
index accb4f198..d3294ee1a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,7 @@
+=== master
+
+* Stop using base64 library in column_encryption plugin (jeremyevans)
+
 === 5.72.0 (2023-09-01)
 
 * Sort caches before marshalling when using schema_caching, index_caching, static_cache_cache, and pg_auto_constraint_validations (jeremyevans)
diff --git a/lib/sequel/plugins/column_encryption.rb b/lib/sequel/plugins/column_encryption.rb
index a0f1fc893..c4a065b51 100644
--- a/lib/sequel/plugins/column_encryption.rb
+++ b/lib/sequel/plugins/column_encryption.rb
@@ -31,7 +31,6 @@
   # :nocov:
 end
 
-require 'base64'
 require 'securerandom'
 
 module Sequel
@@ -375,7 +374,7 @@ def initialize(keys)
         # Decrypt using any supported format and any available key.
         def decrypt(data)
           begin
-            data = Base64.urlsafe_decode64(data)
+            data = urlsafe_decode64(data)
           rescue ArgumentError
             raise Error, "Unable to decode encrypted column: invalid base64"
           end
@@ -448,7 +447,7 @@ def case_insensitive_searchable_encrypt(data)
         # The prefix string of columns for the given search type and the first configured encryption key.
         # Used to find values that do not use this prefix in order to perform reencryption.
         def current_key_prefix(search_type)
-          Base64.urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
+          urlsafe_encode64("#{search_type.chr}\0#{@key_id.chr}")
         end
 
         # The prefix values to search for the given data (an array of strings), assuming the column uses
@@ -472,11 +471,40 @@ def regular_and_lowercase_search_prefixes(data)
 
         private
 
+        if RUBY_VERSION >= '2.4'
+          def decode64(str)
+            str.unpack1("m0")
+          end
+        # :nocov:
+        else
+          def decode64(str)
+            str.unpack("m0")[0]
+          end
+        # :nocov:
+        end
+
+        def urlsafe_encode64(bin)
+          str = [bin].pack("m0")
+          str.chomp!("=") unless str.chomp!("==")
+          str.tr!("+/", "-_")
+          str
+        end
+
+        def urlsafe_decode64(str)
+          if str.length % 4 == 0
+            str = str.tr("-_", "+/")
+          else
+            str = str.ljust((str.length + 3) & ~3, "=")
+            str.tr!("-_", "+/")
+          end
+          decode64(str)
+        end
+
         # An array of strings, one for each configured encryption key, to find encypted values matching
         # the given data and search format.
         def _search_prefixes(data, search_type)
           @key_map.map do |key_id, (key, _)|
-            Base64.urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
+            urlsafe_encode64(_search_prefix(data, search_type, key_id, key))
           end
         end
 
@@ -509,7 +537,7 @@ def _encrypt(data, prefix)
           cipher_text << cipher.update(data) if data_size > 0
           cipher_text << cipher.final
 
-          Base64.urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
+          urlsafe_encode64("#{prefix}#{random_data}#{cipher_iv}#{cipher.auth_tag}#{cipher_text}")
         end
       end
 
diff --git a/spec/extensions/column_encryption_spec.rb b/spec/extensions/column_encryption_spec.rb
index afc975e30..8b2808c3b 100644
--- a/spec/extensions/column_encryption_spec.rb
+++ b/spec/extensions/column_encryption_spec.rb
@@ -395,16 +395,16 @@ def obj.save(*); end
     @obj[:enc] = enc.dup.tap{|x| x[0] = '%'}
     proc{@obj.enc}.must_raise Sequel::Error # invalid base-64
 
-    @obj[:enc] = enc.dup.tap{|x| x[0,4] = Base64.urlsafe_encode64("\4\0\0")}
+    @obj[:enc] = enc.dup.tap{|x| x[0,4] = "BAAA"} # "\4\0\0" base64
     proc{@obj.enc}.must_raise Sequel::Error # invalid flags
 
-    @obj[:enc] = enc.dup.tap{|x| x[0,4] = Base64.urlsafe_encode64("\0\1\0")}
+    @obj[:enc] = enc.dup.tap{|x| x[0,4] = "AAEA"} # "\0\1\0" base64
     proc{@obj.enc}.must_raise Sequel::Error # invalid reserved byte
 
-    @obj[:enc] = enc.dup.tap{|x| x[0,4] = Base64.urlsafe_encode64("\0\0\1")}
+    @obj[:enc] = enc.dup.tap{|x| x[0,4] = "AAAB"} # "\0\0\1" base64
     proc{@obj.enc}.must_raise Sequel::Error # invalid key id
 
-    @obj[:enc] = enc.dup.tap{|x| x[0,4] = Base64.urlsafe_encode64("\1\0\0")}
+    @obj[:enc] = enc.dup.tap{|x| x[0,4] = "AQAA"} # "\1\0\0" base64
     proc{@obj.enc}.must_raise Sequel::Error # invalid minimum size for searchable
 
     @obj[:enc] = enc.dup.tap{|x| x.slice!(60, 1000)}