switch JSON properties to custom JSONProperty that works in web conso…

…le UI

googleapis/python-ndb#874 (comment)

activitypub.py

@@ -65,7 +65,7 @@ def actor(domain):
 
     # TODO: unify with common.actor()
     actor = {
-        **common.postprocess_as2(json_loads(user.actor_as2), user=user),
+        **common.postprocess_as2(user.actor_as2, user=user),
         'id': host_url(domain),
         # This has to be the domain for Mastodon etc interop! It seems like it
         # should be the custom username from the acct: u-url in their h-card,
@@ -126,10 +126,8 @@ def inbox(domain=None):
             return msg, 200
 
     activity_unwrapped = redirect_unwrap(activity)
-    activity_obj = Object(
-        id=id,
-        as2=json_dumps(activity_unwrapped),
-        source_protocol='activitypub')
+    activity_obj = Object(id=id, as2=activity_unwrapped,
+                          source_protocol='activitypub')
     activity_obj.put()
 
     if type == 'Accept':  # eg in response to a Follow
@@ -161,7 +159,7 @@ def inbox(domain=None):
         elif digest.removeprefix('SHA-256=') != expected:
             logger.warning('Invalid Digest header, required for HTTP Signature')
         else:
-            key_actor = json_loads(common.get_object(keyId, user=user).as2)
+            key_actor = common.get_object(keyId, user=user).as2
             key = key_actor.get("publicKey", {}).get('publicKeyPem')
             logger.info(f'Verifying signature for {request.path} with key {key}')
             try:
@@ -190,10 +188,7 @@ def inbox(domain=None):
             error("Couldn't find obj_id of object to update")
 
         obj = Object.get_by_id(obj_id) or Object(id=obj_id)
-        obj.populate(
-            as2=json_dumps(obj_as2),
-            source_protocol='activitypub',
-        )
+        obj.populate(as2=obj_as2, source_protocol='activitypub')
         obj.put()
 
         activity_obj.status = 'complete'
@@ -226,20 +221,20 @@ def inbox(domain=None):
     # fetch actor if necessary so we have name, profile photo, etc
     if actor and isinstance(actor, str):
         actor = activity['actor'] = activity_unwrapped['actor'] = \
-            json_loads(common.get_object(actor, user=user).as2)
+            common.get_object(actor, user=user).as2
 
     # fetch object if necessary so we can render it in feeds
     inner_obj = activity_unwrapped.get('object')
     if type in FETCH_OBJECT_TYPES and isinstance(inner_obj, str):
         obj = Object.get_by_id(inner_obj) or common.get_object(inner_obj, user=user)
         obj_as2 = activity['object'] = activity_unwrapped['object'] = \
-            json_loads(obj.as2) if obj.as2 else as2.from_as1(json_loads(obj.as1))
+            obj.as2 if obj.as2 else as2.from_as1(obj.as1)
 
     if type == 'Follow':
         resp = accept_follow(activity, activity_unwrapped, user)
 
     # send webmentions to each target
-    activity_obj.as2 = json_dumps(activity_unwrapped)
+    activity_obj.as2 = activity_unwrapped
     common.send_webmentions(as2.to_as1(activity), activity_obj, proxy=True)
 
     # deliver original posts and reposts to followers
@@ -305,7 +300,7 @@ def accept_follow(follow, follow_unwrapped, user):
 
     # store Follower
     follower = Follower.get_or_create(dest=user.key.id(), src=follower_id,
-                                      last_follow=json_dumps(follow))
+                                      last_follow=follow)
     follower.status = 'active'
     follower.put()
 

common.py

@@ -158,8 +158,7 @@ def get_object(id, user=None):
       logging.warning(f'Wiping out mf2 property: {obj.mf2}')
       obj.mf2 = None
 
-    obj.populate(as2=json_dumps(obj_as2),
-                 source_protocol='activitypub')
+    obj.populate(as2=obj_as2, source_protocol='activitypub')
     obj.put()
     return obj
 

follow.py

@@ -156,7 +156,7 @@ def finish(self, auth_entity, state=None):
             flash(f"Couldn't find ActivityPub profile link for {addr}")
             return redirect(f'/user/{domain}/following')
 
-        followee = json_loads(common.get_object(as2_url, user=user).as2)
+        followee = common.get_object(as2_url, user=user).as2
         id = followee.get('id')
         inbox = followee.get('inbox')
         if not id or not inbox:
@@ -175,11 +175,10 @@ def finish(self, auth_entity, state=None):
        }
         common.signed_post(inbox, user=user, data=follow_as2)
 
-        follow_json = json_dumps(follow_as2, sort_keys=True)
         Follower.get_or_create(dest=id, src=domain, status='active',
-                                last_follow=follow_json)
+                                last_follow=follow_as2)
         Object(id=follow_id, domains=[domain], labels=['user', 'activity'],
-               source_protocol='ui', status='complete', as2=follow_json,
+               source_protocol='ui', status='complete', as2=follow_as2,
                ).put()
 
         link = common.pretty_link(util.get_url(followee) or id, text=addr)
@@ -229,13 +228,12 @@ def finish(self, auth_entity, state=None):
             error(f'Bad state {state}')
 
         followee_id = follower.dest
-        last_follow = json_loads(follower.last_follow)
-        followee = last_follow['object']
+        followee = follower.last_follow['object']
 
         if isinstance(followee, str):
             # fetch as AS2 to get full followee with inbox
             followee_id = followee
-            followee = json_loads(common.get_object(followee_id, user=user).as2)
+            followee = common.get_object(followee_id, user=user).as2
 
         inbox = followee.get('inbox')
         if not inbox:
@@ -249,15 +247,14 @@ def finish(self, auth_entity, state=None):
             'type': 'Undo',
             'id': unfollow_id,
             'actor': common.host_url(domain),
-            'object': last_follow,
+            'object': follower.last_follow,
        }
         common.signed_post(inbox, user=user, data=unfollow_as2)
 
         follower.status = 'inactive'
         follower.put()
         Object(id=unfollow_id, domains=[domain], labels=['user', 'activity'],
-               source_protocol='ui', status='complete',
-               as2=json_dumps(unfollow_as2, sort_keys=True),
+               source_protocol='ui', status='complete', as2=unfollow_as2,
                ).put()
 
         link = common.pretty_link(util.get_url(followee) or followee_id)

models.py

@@ -14,7 +14,7 @@
 from google.cloud import ndb
 from granary import as1, as2, bluesky, microformats2
 from oauth_dropins.webutil.appengine_info import DEBUG
-from oauth_dropins.webutil.models import StringIdModel
+from oauth_dropins.webutil.models import JsonProperty, StringIdModel
 from oauth_dropins.webutil import util
 from oauth_dropins.webutil.util import json_dumps, json_loads
 
@@ -64,7 +64,7 @@ class User(StringIdModel):
     has_redirects = ndb.BooleanProperty()
     redirects_error = ndb.TextProperty()
     has_hcard = ndb.BooleanProperty()
-    actor_as2 = ndb.TextProperty()
+    actor_as2 = JsonProperty()
     use_instead = ndb.KeyProperty()
 
     created = ndb.DateTimeProperty(auto_now_add=True)
@@ -130,7 +130,7 @@ def private_pem(self):
     def to_as1(self):
         """Returns this user as an AS1 actor dict, if possible."""
         if self.actor_as2:
-            return as2.to_as1(json_loads(self.actor_as2))
+            return as2.to_as1(self.actor_as2)
 
     def username(self):
         """Returns the user's preferred username.
@@ -143,9 +143,8 @@ def username(self):
         domain = self.key.id()
 
         if self.actor_as2:
-            actor = json_loads(self.actor_as2)
             for url in [u.get('value') if isinstance(u, dict) else u
-                        for u in util.get_list(actor, 'url')]:
+                        for u in util.get_list(self.actor_as2, 'url')]:
                 if url and url.startswith('acct:'):
                     urluser, urldomain = util.parse_acct_uri(url)
                     if urldomain == domain:
@@ -177,7 +176,7 @@ def is_homepage(self, url):
     def user_page_link(self):
         """Returns a pretty user page link with the user's name and profile picture."""
         domain = self.key.id()
-        actor = util.json_loads(self.actor_as2) if self.actor_as2 else {}
+        actor = self.actor_as2 or {}
         name = (actor.get('name') or
                 # prettify if domain, noop if username
                 util.domain_from_link(self.username()))
@@ -236,8 +235,7 @@ def verify(self):
 
         # check home page
         try:
-            _, _, actor_as2 = common.actor(self)
-            self.actor_as2 = json_dumps(actor_as2)
+            _, _, self.actor_as2 = common.actor(self)
             self.has_hcard = True
         except (BadRequest, NotFound):
             self.actor_as2 = None
@@ -280,22 +278,22 @@ class Object(StringIdModel):
     source_protocol = ndb.StringProperty(choices=PROTOCOLS)
     labels = ndb.StringProperty(repeated=True, choices=LABELS)
 
-    # these are all JSON. They're TextProperty, and not JsonProperty, so that
-    # their plain text is visible in the App Engine admin console. (JsonProperty
-    # uses a blob.)
-    as2 = ndb.TextProperty()  # only one of the rest will be populated...
-    bsky = ndb.TextProperty() # Bluesky / AT Protocol
-    mf2 = ndb.TextProperty()  # HTML microformats2
+    # TODO: switch back to ndb.JsonProperty if/when they fix it for the web console
+    # https://github.com/googleapis/python-ndb/issues/874
+    as2 = JsonProperty()  # only one of the rest will be populated...
+    bsky = JsonProperty() # Bluesky / AT Protocol
+    mf2 = JsonProperty()  # HTML microformats2
 
     @ndb.ComputedProperty
     def as1(self):
-        assert bool(self.as2) ^ bool(self.bsky) ^ bool(self.mf2), \
-            f'{bool(self.as2)} {bool(self.bsky)} {bool(self.mf2)}'
-        return (as2.to_as1(common.redirect_unwrap(json_loads(self.as2))) if self.as2
-                else bluesky.to_as1(json_loads(self.bsky)) if self.bsky
-                else microformats2.json_to_object(json_loads(self.mf2))
-                  if self.mf2
-                else None)
+        assert (self.as2 is not None) ^ (self.bsky is not None) ^ (self.mf2 is not None), \
+            f'{self.as2} {self.bsky} {self.mf2}'
+        if self.as2 is not None:
+            return as2.to_as1(common.redirect_unwrap(self.as2))
+        elif self.bsky is not None:
+            return bluesky.to_as1(self.bsky)
+        elif self.mf2 is not None:
+            return microformats2.json_to_object(self.mf2)
 
     @ndb.ComputedProperty
     def type(self):  # AS1 objectType, or verb if it's an activity
@@ -371,7 +369,7 @@ class Follower(StringIdModel):
     dest = ndb.StringProperty()
     # Most recent AP (AS2) JSON Follow activity. If inbound, must have a
     # composite actor object with an inbox, publicInbox, or sharedInbox.
-    last_follow = ndb.TextProperty()
+    last_follow = JsonProperty()
     status = ndb.StringProperty(choices=STATUSES, default='active')
 
     created = ndb.DateTimeProperty(auto_now_add=True)
@@ -403,7 +401,4 @@ def to_as1(self):
     def to_as2(self):
         """Returns this follower as an AS2 actor dict, if possible."""
         if self.last_follow:
-            last_follow = json_loads(self.last_follow)
-            person = last_follow.get('actor' if util.is_web(self.src) else 'object')
-            if person:
-                return person
+            return self.last_follow.get('actor' if util.is_web(self.src) else 'object')

pages.py

@@ -209,7 +209,7 @@ def fetch_objects(query, user):
         if isinstance(inner_obj, str):
             inner_obj = Object.get_by_id(inner_obj)
             if inner_obj:
-                inner_obj = json_loads(inner_obj.as1)
+                inner_obj = inner_obj.as1
 
         content = (inner_obj.get('content')
                    or inner_obj.get('displayName')

render.py

@@ -9,7 +9,6 @@
 from oauth_dropins.webutil import flask_util
 from oauth_dropins.webutil.flask_util import error
 from oauth_dropins.webutil import util
-from oauth_dropins.webutil.util import json_loads
 
 from app import app, cache
 import common

tests/test_activitypub.py

@@ -202,8 +202,7 @@ class ActivityPubTest(testutil.TestCase):
 
     def setUp(self):
         super().setUp()
-        self.user = User.get_or_create('foo.com', has_hcard=True,
-                                       actor_as2=json_dumps(ACTOR))
+        self.user = User.get_or_create('foo.com', has_hcard=True, actor_as2=ACTOR)
 
     def test_actor(self, *_):
         got = self.client.get('/foo.com')
@@ -369,7 +368,7 @@ def test_repost_of_federated_post(self, mock_head, mock_get, mock_post):
             'url': 'https://foo.com/orig',
         }
         with app.test_request_context('/'):
-            Object(id=orig_url, as2=json_dumps(note)).put()
+            Object(id=orig_url, as2=note).put()
 
         repost = {
             **REPOST_FULL,
@@ -559,7 +558,7 @@ def test_inbox_follow_accept_with_id(self, *mocks):
                            object_ids=[FOLLOW['object']])
 
         follower = Follower.query().get()
-        self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, json_loads(follower.last_follow))
+        self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, follower.last_follow)
 
     def test_inbox_follow_accept_with_object(self, *mocks):
         wrapped_user = {
@@ -583,7 +582,7 @@ def test_inbox_follow_accept_with_object(self, *mocks):
 
         follower = Follower.query().get()
         follow['actor'] = ACTOR
-        self.assertEqual(follow, json_loads(follower.last_follow))
+        self.assertEqual(follow, follower.last_follow)
 
         follow.update({
             'object': unwrapped_user,
@@ -652,7 +651,7 @@ def test_inbox_follow_use_instead_strip_www(self, mock_head, mock_get, mock_post
         # check that the Follower doesn't have www
         follower = Follower.get_by_id(f'foo.com {ACTOR["id"]}')
         self.assertEqual('active', follower.status)
-        self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, json_loads(follower.last_follow))
+        self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, follower.last_follow)
 
     def test_inbox_undo_follow(self, mock_head, mock_get, mock_post):
         mock_head.return_value = requests_response(url='https://foo.com/')
@@ -850,7 +849,7 @@ def test_delete_actor(self, _, mock_get, ___):
         self.assertEqual('active', other.key.get().status)
 
     def test_delete_note(self, _, mock_get, ___):
-        obj = Object(id='http://an/obj', as2='{}')
+        obj = Object(id='http://an/obj', as2={})
         obj.put()
 
         mock_get.side_effect = [
@@ -872,7 +871,7 @@ def test_delete_note(self, _, mock_get, ___):
         self.assert_entities_equal(obj, common.get_object.cache['http://an/obj'])
 
     def test_update_note(self, *mocks):
-        Object(id='https://a/note', as2='{}').put()
+        Object(id='https://a/note', as2={}).put()
         self._test_update(*mocks)
 
     def test_update_unknown(self, *mocks):
@@ -928,7 +927,7 @@ def test_inbox_no_webmention_endpoint(self, mock_head, mock_get, mock_post):
                            object_ids=[LIKE['object']])
 
     def test_inbox_id_already_seen(self, *mocks):
-        obj_key = Object(id=FOLLOW_WRAPPED['id'], as2='{}').put()
+        obj_key = Object(id=FOLLOW_WRAPPED['id'], as2={}).put()
 
         got = self.client.post('/foo.com/inbox', json=FOLLOW_WRAPPED)
         self.assertEqual(200, got.status_code)
@@ -964,10 +963,10 @@ def test_followers_collection_empty(self, *args):
 
     def store_followers(self):
         Follower.get_or_create('foo.com', 'https://bar.com',
-                               last_follow=json_dumps(FOLLOW_WITH_ACTOR))
+                               last_follow=FOLLOW_WITH_ACTOR)
         Follower.get_or_create('http://other/actor', 'foo.com')
         Follower.get_or_create('foo.com', 'https://baz.com',
-                               last_follow=json_dumps(FOLLOW_WITH_ACTOR))
+                               last_follow=FOLLOW_WITH_ACTOR)
         Follower.get_or_create('foo.com', 'baj.com', status='inactive')
 
     def test_followers_collection(self, *args):
@@ -1032,10 +1031,10 @@ def test_following_collection_empty(self, *args):
 
     def store_following(self):
         Follower.get_or_create('https://bar.com', 'foo.com',
-                               last_follow=json_dumps(FOLLOW_WITH_OBJECT))
+                               last_follow=FOLLOW_WITH_OBJECT)
         Follower.get_or_create('foo.com', 'http://other/actor')
         Follower.get_or_create('https://baz.com', 'foo.com',
-                               last_follow=json_dumps(FOLLOW_WITH_OBJECT))
+                               last_follow=FOLLOW_WITH_OBJECT)
         Follower.get_or_create('baj.com', 'foo.com', status='inactive')
 
     def test_following_collection(self, *args):