switch Object.as1 to be a ComputedProperty

activitypub.py

@@ -129,7 +129,6 @@ def inbox(domain=None):
     activity_obj = Object(
         id=id,
         as2=json_dumps(activity_unwrapped),
-        as1=json_dumps(as2.to_as1(activity_unwrapped)),
         source_protocol='activitypub')
     activity_obj.put()
 
@@ -193,7 +192,6 @@ def inbox(domain=None):
         obj = Object.get_by_id(obj_id) or Object(id=obj_id)
         obj.populate(
             as2=json_dumps(obj_as2),
-            as1=json_dumps(as2.to_as1(obj_as2)),
             source_protocol='activitypub',
         )
         obj.put()
@@ -231,17 +229,17 @@ def inbox(domain=None):
             json_loads(common.get_object(actor, user=user).as2)
 
     # fetch object if necessary so we can render it in feeds
-    if type in FETCH_OBJECT_TYPES and isinstance(activity.get('object'), str):
+    inner_obj = activity_unwrapped.get('object')
+    if type in FETCH_OBJECT_TYPES and isinstance(inner_obj, str):
+        obj = Object.get_by_id(inner_obj) or common.get_object(inner_obj, user=user)
         obj_as2 = activity['object'] = activity_unwrapped['object'] = \
-            json_loads(common.get_object(activity['object'], user=user).as2)
+            json_loads(obj.as2) if obj.as2 else as2.from_as1(json_loads(obj.as1))
 
     if type == 'Follow':
         resp = accept_follow(activity, activity_unwrapped, user)
 
     # send webmentions to each target
-    activity_as1 = as2.to_as1(activity_unwrapped)
-    activity_obj.populate(as2=json_dumps(activity_unwrapped),
-                          as1=json_dumps(activity_as1))
+    activity_obj.as2 = json_dumps(activity_unwrapped)
     common.send_webmentions(as2.to_as1(activity), activity_obj, proxy=True)
 
     # deliver original posts and reposts to followers
@@ -265,10 +263,10 @@ def inbox(domain=None):
                 if activity_obj.domains and 'feed' not in activity_obj.labels:
                     activity_obj.labels.append('feed')
 
-    if (activity_as1.get('objectType') == 'activity'
+    if (activity_obj.as1.get('objectType') == 'activity'
         and 'activity' not in activity_obj.labels):
-
         activity_obj.labels.append('activity')
+
     activity_obj.put()
     return 'OK'
 

common.py

@@ -153,8 +153,12 @@ def get_object(id, user=None):
 
     logger.info(f'Object not in datastore or has no as2: {id}')
     obj_as2 = get_as2(id, user=user).json()
+
+    if obj.mf2:
+      logging.warning(f'Wiping out mf2 property: {obj.mf2}')
+      obj.mf2 = None
+
     obj.populate(as2=json_dumps(obj_as2),
-                 as1=json_dumps(as2.to_as1(obj_as2)),
                  source_protocol='activitypub')
     obj.put()
     return obj
@@ -320,7 +324,7 @@ def send_webmentions(activity_wrapped, obj, proxy=None):
 
     Returns: boolean, True if any webmentions were sent, False otherwise
     """
-    activity_unwrapped = json_loads(obj.as1)
+    activity_unwrapped = obj.as1
 
     verb = activity_unwrapped.get('verb')
     if verb and verb not in SUPPORTED_VERBS:

follow.py

@@ -180,7 +180,6 @@ def finish(self, auth_entity, state=None):
                                 last_follow=follow_json)
         Object(id=follow_id, domains=[domain], labels=['user', 'activity'],
                source_protocol='ui', status='complete', as2=follow_json,
-               as1=json_dumps(as2.to_as1(follow_as2), sort_keys=True),
                ).put()
 
         link = common.pretty_link(util.get_url(followee) or id, text=addr)
@@ -259,7 +258,6 @@ def finish(self, auth_entity, state=None):
         Object(id=unfollow_id, domains=[domain], labels=['user', 'activity'],
                source_protocol='ui', status='complete',
                as2=json_dumps(unfollow_as2, sort_keys=True),
-               as1=json_dumps(as2.to_as1(unfollow_as2), sort_keys=True),
                ).put()
 
         link = common.pretty_link(util.get_url(followee) or followee_id)

models.py

@@ -12,7 +12,7 @@
 from Crypto.Util import number
 from flask import request
 from google.cloud import ndb
-from granary import as1, as2, microformats2
+from granary import as1, as2, bluesky, microformats2
 from oauth_dropins.webutil.appengine_info import DEBUG
 from oauth_dropins.webutil.models import StringIdModel
 from oauth_dropins.webutil import util
@@ -283,17 +283,26 @@ class Object(StringIdModel):
     # these are all JSON. They're TextProperty, and not JsonProperty, so that
     # their plain text is visible in the App Engine admin console. (JsonProperty
     # uses a blob.)
-    as1 = ndb.TextProperty(required=True)  # converted from source data
     as2 = ndb.TextProperty()  # only one of the rest will be populated...
     bsky = ndb.TextProperty() # Bluesky / AT Protocol
     mf2 = ndb.TextProperty()  # HTML microformats2
 
+    @ndb.ComputedProperty
+    def as1(self):
+        assert bool(self.as2) ^ bool(self.bsky) ^ bool(self.mf2), \
+            f'{bool(self.as2)} {bool(self.bsky)} {bool(self.mf2)}'
+        return (as2.to_as1(common.redirect_unwrap(json_loads(self.as2))) if self.as2
+                else bluesky.to_as1(json_loads(self.bsky)) if self.bsky
+                else microformats2.json_to_object(json_loads(self.mf2))
+                  if self.mf2
+                else None)
+
     @ndb.ComputedProperty
     def type(self):  # AS1 objectType, or verb if it's an activity
-        return as1.object_type(json_loads(self.as1))
+        return as1.object_type(self.as1)
 
     def _object_ids(self):  # id(s) of inner objects
-        return as1.get_ids(json_loads(self.as1), 'object')
+        return as1.get_ids(self.as1, 'object')
     object_ids = ndb.ComputedProperty(_object_ids, repeated=True)
 
     deleted = ndb.BooleanProperty()
@@ -329,9 +338,8 @@ def actor_link(self, user=None):
             # outbound; show a nice link to the user
             return user.user_page_link()
 
-        activity = json_loads(self.as1)
-        actor = (util.get_first(activity, 'actor')
-                 or util.get_first(activity, 'author')
+        actor = (util.get_first(self.as1, 'actor')
+                 or util.get_first(self.as1, 'author')
                  or {})
         if isinstance(actor, str):
             return common.pretty_link(actor, user=user)

pages.py

@@ -142,7 +142,7 @@ def feed(domain):
         Object.domains == domain, Object.labels == 'feed') \
         .order(-Object.created) \
         .fetch_page(PAGE_SIZE)
-    activities = [json_loads(obj.as1) for obj in objects if not obj.deleted]
+    activities = [obj.as1 for obj in objects if not obj.deleted]
 
     actor = {
       'displayName': domain,
@@ -184,10 +184,8 @@ def fetch_objects(query, user):
 
     # synthesize human-friendly content for objects
     for i, obj in enumerate(objects):
-        obj_as1 = json_loads(obj.as1)
-
         # synthesize text snippet
-        type = as1.object_type(obj_as1)
+        type = as1.object_type(obj.as1)
         phrases = {
             'article': 'posted',
             'comment': 'replied',
@@ -207,7 +205,7 @@ def fetch_objects(query, user):
         obj.phrase = phrases.get(type)
 
         # TODO: unify inner object loading? optionally fetch external?
-        inner_obj = util.get_first(obj_as1, 'object') or {}
+        inner_obj = util.get_first(obj.as1, 'object') or {}
         if isinstance(inner_obj, str):
             inner_obj = Object.get_by_id(inner_obj)
             if inner_obj:
@@ -220,17 +218,17 @@ def fetch_objects(query, user):
         if (obj.domains and
               inner_obj.get('id', '').strip('/') == f'https://{obj.domains[0]}'):
             obj.phrase = 'updated'
-            obj_as1.update({
+            obj.as1.update({
                 'content': 'their profile',
                 'url': f'https://{obj.domains[0]}',
             })
         elif url:
             content = common.pretty_link(url, text=content, user=user)
 
-        obj.content = (obj_as1.get('content')
-                       or obj_as1.get('displayName')
-                       or obj_as1.get('summary'))
-        obj.url = util.get_first(obj_as1, 'url')
+        obj.content = (obj.as1.get('content')
+                       or obj.as1.get('displayName')
+                       or obj.as1.get('summary'))
+        obj.url = util.get_first(obj.as1, 'url')
 
         if (type in ('like', 'follow', 'repost', 'share') or
             not obj.content):

redirect.py

@@ -82,7 +82,7 @@ def redir(to):
                 obj = Object.get_by_id(to)
                 if not obj or obj.deleted:
                     return f'Object not found: {to}', 404
-                ret = postprocess_as2(as2.from_as1(json_loads(obj.as1)),
+                ret = postprocess_as2(as2.from_as1(obj.as1),
                                       user, create=False)
                 logger.info(f'Returning: {json_dumps(ret, indent=2)}')
                 return ret, {

render.py

@@ -29,13 +29,11 @@ def render():
     elif not obj.as1:
         error(f'Stored object for {id} has no AS1', status=404)
 
-    as1 = json_loads(obj.as1)
-
     # TODO: uncomment once we're storing inner objects separately
-    # if (as1.get('objectType') == 'activity' and
-    #     as1.get('verb') in ('post', 'update', 'delete')):
+    # if (obj.as1.get('objectType') == 'activity' and
+    #     obj.as1.get('verb') in ('post', 'update', 'delete')):
     #     # redirect to inner object
-    #     obj_id = as1.get('object')
+    #     obj_id = obj.as1.get('object')
     #     if isinstance(obj_id, dict):
     #         obj_id = obj_id.get('id')
     #     if not obj_id:
@@ -45,9 +43,9 @@ def render():
 
     # add HTML meta redirect to source page. should trigger for end users in
     # browsers but not for webmention receivers (hopefully).
-    html = microformats2.activities_to_html([as1])
+    html = microformats2.activities_to_html([obj.as1])
     utf8 = '<meta charset="utf-8">'
-    url = util.get_url(as1)
+    url = util.get_url(obj.as1)
     if url:
         refresh = f'<meta http-equiv="refresh" content="0;url={url}">'
         html = html.replace(utf8, utf8 + '\n' + refresh)

tests/test_activitypub.py

@@ -9,7 +9,7 @@
 import urllib.parse
 
 from google.cloud import ndb
-from granary import as2
+from granary import as2, microformats2
 from httpsig import HeaderSigner
 from oauth_dropins.webutil import util
 from oauth_dropins.webutil.testutil import requests_response
@@ -18,6 +18,7 @@
 from urllib3.exceptions import ReadTimeoutError
 
 import activitypub
+from app import app
 import common
 from models import Follower, Object, User
 from . import testutil
@@ -301,7 +302,6 @@ def _test_inbox_reply(self, reply, expected_props, mock_head, mock_get, mock_pos
                            domains=['or.ig'],
                            source_protocol='activitypub',
                            status='complete',
-                           as1=as2.to_as1(expected_props['as2']),
                            delivered=['http://or.ig/post'],
                            **expected_props)
 
@@ -349,7 +349,6 @@ def _test_inbox_create_obj(self, path, mock_head, mock_get, mock_post):
         self.assert_object('http://th.is/note/as2',
                            source_protocol='activitypub',
                            as2=expected_as2,
-                           as1=as2.to_as1(expected_as2),
                            domains=['foo.com', 'baz.com'],
                            type='post',
                            labels=['activity', 'feed'],
@@ -369,7 +368,8 @@ def test_repost_of_federated_post(self, mock_head, mock_get, mock_post):
             **NOTE_OBJECT,
             'url': 'https://foo.com/orig',
         }
-        Object(id=orig_url, mf2='{}', as1=json_dumps(as2.to_as1(note))).put()
+        with app.test_request_context('/'):
+            Object(id=orig_url, as2=json_dumps(note)).put()
 
         repost = {
             **REPOST_FULL,
@@ -390,7 +390,6 @@ def test_repost_of_federated_post(self, mock_head, mock_get, mock_post):
             },
         )
 
-        note.pop('cc')
         repost['object'] = note
         self.assert_object(REPOST_FULL['id'],
                            source_protocol='activitypub',
@@ -400,7 +399,7 @@ def test_repost_of_federated_post(self, mock_head, mock_get, mock_post):
                            domains=['foo.com'],
                            delivered=['https://foo.com/orig'],
                            type='share',
-                           labels=['activity', 'notification'],
+                           labels=['activity', 'feed', 'notification'],
                            object_ids=[NOTE_OBJECT['id']])
 
     def test_shared_inbox_repost(self, mock_head, mock_get, mock_post):
@@ -435,7 +434,6 @@ def test_shared_inbox_repost(self, mock_head, mock_get, mock_post):
                            source_protocol='activitypub',
                            status='complete',
                            as2=REPOST_FULL,
-                           as1=as2.to_as1(REPOST_FULL),
                            domains=['foo.com', 'baz.com', 'th.is'],
                            type='share',
                            labels=['activity', 'feed', 'notification'],
@@ -507,7 +505,6 @@ def _test_inbox_mention(self, mention, expected_props, mock_head, mock_get, mock
                            source_protocol='activitypub',
                            status='complete',
                            as2=expected_as2,
-                           as1=as2.to_as1(expected_as2),
                            delivered=['https://tar.get/'],
                            **expected_props)
 
@@ -540,7 +537,6 @@ def test_inbox_like(self, mock_head, mock_get, mock_post):
                            source_protocol='activitypub',
                            status='complete',
                            as2=LIKE_WITH_ACTOR,
-                           as1=as2.to_as1(LIKE_WITH_ACTOR),
                            delivered=['http://or.ig/post'],
                            type='like',
                            labels=['notification', 'activity'],
@@ -557,7 +553,6 @@ def test_inbox_follow_accept_with_id(self, *mocks):
                            source_protocol='activitypub',
                            status='complete',
                            as2=follow,
-                           as1=as2.to_as1(follow),
                            delivered=['https://foo.com/'],
                            type='follow',
                            labels=['notification', 'activity'],
@@ -599,7 +594,6 @@ def test_inbox_follow_accept_with_object(self, *mocks):
                            source_protocol='activitypub',
                            status='complete',
                            as2=follow,
-                           as1=as2.to_as1(follow),
                            delivered=['https://foo.com/'],
                            type='follow',
                            labels=['notification', 'activity'],
@@ -856,7 +850,7 @@ def test_delete_actor(self, _, mock_get, ___):
         self.assertEqual('active', other.key.get().status)
 
     def test_delete_note(self, _, mock_get, ___):
-        obj = Object(id='http://an/obj', as1='{}')
+        obj = Object(id='http://an/obj', as2='{}')
         obj.put()
 
         mock_get.side_effect = [
@@ -870,15 +864,15 @@ def test_delete_note(self, _, mock_get, ___):
         resp = self.client.post('/inbox', json=delete)
         self.assertEqual(200, resp.status_code)
         self.assertTrue(obj.key.get().deleted)
-        self.assert_object(delete['id'], as2=delete, as1=as2.to_as1(delete),
+        self.assert_object(delete['id'], as2=delete,
                            type='delete', source_protocol='activitypub',
                            status='complete')
 
         obj.deleted = True
         self.assert_entities_equal(obj, common.get_object.cache['http://an/obj'])
 
     def test_update_note(self, *mocks):
-        Object(id='https://a/note', as1='{}').put()
+        Object(id='https://a/note', as2='{}').put()
         self._test_update(*mocks)
 
     def test_update_unknown(self, *mocks):
@@ -894,10 +888,9 @@ def _test_update(self, _, mock_get, ___):
 
         obj = UPDATE_NOTE['object']
         self.assert_object('https://a/note', type='note', as2=obj,
-                           as1=as2.to_as1(obj), source_protocol='activitypub')
+                           source_protocol='activitypub')
         self.assert_object(UPDATE_NOTE['id'], source_protocol='activitypub',
-                           type='update', status='complete', as2=UPDATE_NOTE,
-                           as1=as2.to_as1(UPDATE_NOTE))
+                           type='update', status='complete', as2=UPDATE_NOTE)
 
         self.assert_entities_equal(Object.get_by_id('https://a/note'),
                                    common.get_object.cache['https://a/note'])
@@ -930,13 +923,12 @@ def test_inbox_no_webmention_endpoint(self, mock_head, mock_get, mock_post):
                            source_protocol='activitypub',
                            status='complete',
                            as2=LIKE_WITH_ACTOR,
-                           as1=as2.to_as1(LIKE_WITH_ACTOR),
                            type='like',
                            labels=['notification', 'activity'],
                            object_ids=[LIKE['object']])
 
     def test_inbox_id_already_seen(self, *mocks):
-        obj_key = Object(id=FOLLOW_WRAPPED['id'], as1='{}').put()
+        obj_key = Object(id=FOLLOW_WRAPPED['id'], as2='{}').put()
 
         got = self.client.post('/foo.com/inbox', json=FOLLOW_WRAPPED)
         self.assertEqual(200, got.status_code)