Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .codex-plugin/plugin.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "codex-plugin",
"version": "0.2.28",
"version": "0.2.43",
"description": "Inkbox bridge for Codex over email, SMS, iMessage, and voice.",
"author": {
"name": "Inkbox AI",
Expand Down
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog

## 0.2.43 (unreleased)

- Wait up to 60 seconds for incoming voice-note transcripts, then read the message once before continuing with its original attachments.
- Register transcription-completion notifications automatically and label generated voice-note text distinctly from typed messages.
- Retain pending voice-note receipts and their original deadline across gateway restarts without waking twice for late completion notifications.

## 0.2.42

- Retain managed inbound work when final reply delivery cannot be confirmed.
Expand Down
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1168,3 +1168,7 @@ report; the key itself is not needed in the terminal. Runtime processes still
require their own signing key. Managed delivery does not require ordinary webhook
subscriptions. Standalone diagnostics are unchanged. These checks do not send
messages or prove end-to-end delivery.

### Incoming voice notes

Incoming audio keeps its original attachment. When transcription is pending, the gateway waits up to 60 seconds for its completion notification, then reads the message once. Available transcripts are labeled as auto-generated voice-note text, not typed messages. If transcription fails, remains pending, or cannot be read, the original attachment follows the existing message-handling path. Completion notifications are registered automatically and do not create a second agent turn.
2 changes: 1 addition & 1 deletion inkbox_codex/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""Inkbox bridge for Codex — email, SMS, iMessage, and voice."""

__version__ = "0.2.42"
__version__ = "0.2.43"
9 changes: 7 additions & 2 deletions inkbox_codex/companion.py
Original file line number Diff line number Diff line change
Expand Up @@ -421,10 +421,11 @@ def close(self):


class Receiver:
def __init__(self, *, cfg, client, sessions, sender_allowed: Callable, mail_body: Callable, prepare_whatsapp: Callable | None = None):
def __init__(self, *, cfg, client, sessions, sender_allowed: Callable, mail_body: Callable, prepare_whatsapp: Callable | None = None, resolve_voice_note: Callable | None = None):
self.cfg, self.client, self.sessions = cfg, client, sessions
self.sender_allowed, self.mail_body = sender_allowed, mail_body
self.prepare_whatsapp = prepare_whatsapp
self.resolve_voice_note = resolve_voice_note
namespace = hashlib.sha256(f"{cfg.base_url}|{cfg.identity}".encode()).hexdigest()
self.namespace = namespace
self.inbox = Inbox(inbox_path(cfg))
Expand Down Expand Up @@ -983,6 +984,9 @@ async def process(self, event):
# Match ordinary inbound filtering: discard this input without
# blocking a later, permitted sponsor in the same scope.
return
if self.resolve_voice_note is not None and event.channel in {"phone", "imessage", "whatsapp"}:
field = "text_message" if event.channel == "phone" else "message"
event.envelope["data"][field] = await self.resolve_voice_note(event.channel, event.envelope["data"][field])
if event.channel == "whatsapp" and not event.envelope["_codex_whatsapp_source"].get("whatsapp_enriched"):
# Receipt admission/HTTP acknowledgement must not wait on attachment
# downloads or contact lookups. Persist enrichment for restart replay.
Expand Down Expand Up @@ -1043,7 +1047,8 @@ async def live_text(self, event):
raise CompanionError("Invalid Companion Slack message")
return incoming[1]
message = event.envelope["data"]["text_message" if event.channel == "phone" else "message"]
text = event.text
from .voice_notes import transcript_note
text = event.text + transcript_note(message.get("media") or [])
if event.channel == "whatsapp":
text = event.envelope["_codex_whatsapp_source"]["whatsapp_body"]
if event.channel == "mail":
Expand Down
85 changes: 79 additions & 6 deletions inkbox_codex/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
from .codex_client import CodexTurnResult, probe_codex
from .a2a_delegations import find_by_task as find_a2a_delegation
from .media import download_media, inbound_media_note
from .voice_notes import FINISHED
from .hosted_sms_guard import hosted_sms_attempt_state
from .prompts import contact_marker, inject_contact_memories, normalize_contact_memories, strip_markdown
from .realtime import (
Expand Down Expand Up @@ -126,6 +127,7 @@
from codex_client import CodexTurnResult, probe_codex
from a2a_delegations import find_by_task as find_a2a_delegation
from media import download_media, inbound_media_note
from voice_notes import FINISHED
from hosted_sms_guard import hosted_sms_attempt_state
from prompts import contact_marker, inject_contact_memories, normalize_contact_memories, strip_markdown
from realtime import (
Expand Down Expand Up @@ -793,6 +795,13 @@ def _delivery_failure_reply_instruction(
# Mail: inbound plus the two delivery-failure transitions that feed the loop
# (_on_mail_delivery_failed). The success transitions stay unsubscribed — they
# would pay signature cost on every outbound email for no behaviour.
TRANSCRIPTION_EVENTS = [FINISHED]


def _subscription_family(event_type: str) -> str:
return event_type if event_type in TRANSCRIPTION_EVENTS else event_type.split(".", 1)[0]


MAIL_EVENTS = ["message.received", "message.bounced", "message.failed"]


Expand Down Expand Up @@ -979,6 +988,7 @@ def __init__(self, cfg: BridgeConfig):
self._managed_draining = False
self._managed_ingress = 0
self.sessions: Optional[SessionManager] = None
self._voice_note_closing = False
self._slack_activity = None
self._companion_receiver = None
self._scheduler = None
Expand Down Expand Up @@ -1043,6 +1053,7 @@ async def _run(self) -> None:
None
"""
self._a2a_closing = False
self._voice_note_closing = False
if not AIOHTTP_AVAILABLE:
raise RuntimeError("aiohttp is not installed; run: pip install aiohttp")
if not INKBOX_AVAILABLE:
Expand Down Expand Up @@ -1132,6 +1143,7 @@ async def _run(self) -> None:
retire(self.cfg)
await self._recover_imessage_inputs()
receiver.recover()
self._voice_note_inbox().recover()
await asyncio.to_thread(self._patch_identity_objects)
await self._catch_up_a2a_tasks()
await self._recover_hosted_call_completions()
Expand Down Expand Up @@ -1275,7 +1287,8 @@ def _reconcile(
subscription_url: str = webhook_url,
) -> None:
whatsapp = any(event.startswith("whatsapp.") for event in event_types)
lookup_kw = {**owner_kw, "scope": "identity"} if whatsapp else owner_kw
identity_scope = whatsapp or event_types == TRANSCRIPTION_EVENTS
lookup_kw = {**owner_kw, "scope": "identity"} if identity_scope else owner_kw
subscriptions = self._inkbox.webhooks.subscriptions
try:
existing = subscriptions.list(**lookup_kw)
Expand All @@ -1287,9 +1300,9 @@ def _reconcile(
return
raise
desired_families = {
event_type.split(".", 1)[0] for event_type in event_types
_subscription_family(event_type) for event_type in event_types
}
if whatsapp and any(sub.url == subscription_url and set(event_types).issubset(sub.event_types)
if identity_scope and any(sub.url == subscription_url and set(event_types).issubset(sub.event_types)
for sub in existing):
return
for sub in existing:
Expand All @@ -1299,12 +1312,12 @@ def _reconcile(
):
return # already wired
existing_families = {
event_type.split(".", 1)[0] for event_type in sub.event_types
_subscription_family(event_type) for event_type in sub.event_types
}
if (
sub.url.split("?", 1)[0].endswith(DEFAULT_WEBHOOK_PATH)
and desired_families & existing_families
and (not whatsapp or (existing_families <= desired_families
and (not identity_scope or (existing_families <= desired_families
and sub.url == subscription_url))
):
# Replace only this event channel. One identity may have
Expand All @@ -1313,13 +1326,19 @@ def _reconcile(
try:
subscriptions.create(url=subscription_url, event_types=event_types, **owner_kw)
except Exception as exc:
if whatsapp and getattr(exc, "status_code", None) == 409:
if identity_scope and getattr(exc, "status_code", None) == 409:
# Another startup may have installed the same receiver.
for sub in subscriptions.list(**lookup_kw):
if sub.url == subscription_url and set(event_types).issubset(sub.event_types):
return
raise

try:
_reconcile({"agent_identity_id": identity.id}, TRANSCRIPTION_EVENTS)
except ValueError as exc:
if not str(exc).startswith("Identity-wide webhook subscriptions are not supported by this server yet."):
raise
logger.warning("Transcription callbacks are unavailable on this API version")
if identity.mailbox is not None:
_reconcile({"mailbox_id": identity.mailbox.id}, MAIL_EVENTS)
logger.info("[bridge] mailbox %s → %s", identity.mailbox.email_address, webhook_url)
Expand Down Expand Up @@ -1404,7 +1423,41 @@ def _reconcile(
_reconcile({"agent_identity_id": identity.id}, CALL_EVENTS)
logger.info("[bridge] identity events for %s → %s", self.cfg.identity, webhook_url)

def _voice_note_inbox(self):
if getattr(self, "_voice_note_closing", False):
raise RuntimeError("Voice-note intake is shutting down")
from .voice_notes import VoiceNotes
inbox = getattr(self, "_voice_note_state", None)
if inbox is None:
root = Path(os.getenv("INKBOX_CODEX_HOME") or (Path.home() / ".inkbox-codex"))
identity_id = str(self._identity.id)
inbox = self._voice_note_state = VoiceNotes(
root / f"voice-notes-{identity_id}.sqlite3", self._identity,
self._inkbox, self._deliver_voice_note,
)
return inbox

async def _deliver_voice_note(self, envelope):
event_type = envelope.get("event_type")
if event_type == "text.received":
return await self._on_text_received(envelope)
if event_type == "imessage.received":
return await self._on_imessage_received(envelope)
from .whatsapp import receive
return await receive(self, envelope)

def _voice_note_receipt(self, event_type, envelope):
from .voice_notes import RECEIVED, message_for, pending
if event_type == FINISHED:
self._voice_note_inbox().complete(envelope)
return web.json_response({"ok": True})
if event_type in RECEIVED and (pending(message_for(envelope)) or getattr(self, "_voice_note_state", None) is not None):
if self._voice_note_inbox().defer(envelope):
return web.json_response({"ok": True, "pending_transcription": True})
return None

async def _cleanup(self) -> None:
self._voice_note_closing = True
if self._managed_control is not None:
await self._managed_control.close()
self._managed_control = None
Expand Down Expand Up @@ -1448,6 +1501,9 @@ async def _cleanup(self) -> None:
await asyncio.gather(*jobs, return_exceptions=True)
if getattr(self, "_companion_receiver", None) is not None:
await self._companion_receiver.close()
if getattr(self, "_voice_note_state", None) is not None:
await self._voice_note_state.close()
self._voice_note_state = None
if self.sessions is not None:
await self.sessions.close_all()
if getattr(self, "_slack_activity", None) is not None:
Expand Down Expand Up @@ -2119,6 +2175,12 @@ def _is_known_inkbox_event(self, event_type: "str | None", envelope: Dict[str, A
or (envelope.get("direction") == "inbound" and envelope.get("local_phone_number"))
)

async def _resolve_companion_voice_note(self, channel, message):
from .voice_notes import pending
if not pending(message):
return message
return await self._voice_note_inbox().resolve_message("text" if channel == "phone" else channel, message)

async def _prepare_whatsapp_companion(self, envelope, *, enrich=False):
from .whatsapp import prepare_companion
return await prepare_companion(self, envelope, enrich=enrich)
Expand All @@ -2131,6 +2193,7 @@ def _companion(self):
cfg=self.cfg, client=self._inkbox, sessions=self.sessions,
sender_allowed=self._sender_allowed, mail_body=self._fetch_mail_body,
prepare_whatsapp=self._prepare_whatsapp_companion,
resolve_voice_note=self._resolve_companion_voice_note,
)
return self._companion_receiver

Expand Down Expand Up @@ -2218,6 +2281,9 @@ async def _accept_webhook(self, request: "web.Request") -> "web.Response":
):
return web.Response(status=401, text="WhatsApp requires a valid Inkbox signature")

if source == "inkbox" and envelope.get("event_type") == FINISHED:
return self._voice_note_receipt(envelope["event_type"], envelope)

if envelope.get("companion") is not None:
# Context grants require a verified Inkbox signature even when the
# operator permits unsigned ordinary webhooks for local testing.
Expand Down Expand Up @@ -2297,6 +2363,9 @@ async def _route_inkbox_event(

if event_type in (*SLACK_INCOMING_EVENTS, SLACK_STOP_EVENT):
return await self._on_slack_received(envelope)
voice_receipt = self._voice_note_receipt(event_type, envelope)
if voice_receipt is not None:
return voice_receipt
if event_type == "message.received":
return await self._on_mail_received(envelope)
if event_type == "text.received":
Expand Down Expand Up @@ -4510,13 +4579,17 @@ async def _with_media(self, text: str, media: List[Dict[str, Any]], *, prefix: s
"""
if not media:
return text
from .voice_notes import transcript_note
text += transcript_note(media)
saved = await download_media(media, prefix=prefix)
if image_paths is not None:
image_paths.extend(item["path"] for item in saved if item["content_type"].startswith("image/"))
from .slack_media import transcribe_audio
for item in saved:
if not item["content_type"].startswith("audio/"):
continue
if (item.get("transcription") or {}).get("status") == "completed":
continue
key = getattr(getattr(self.cfg, "realtime", None), "api_key", "")
try:
if not key:
Expand Down
25 changes: 23 additions & 2 deletions inkbox_codex/managed_ingress.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ async def loop(self, *, protocol_only=False):
async def refresh_media(gateway, payload):
"""Refresh expiring media at dispatch, including deliveries with no retry.

Read through the identity-scoped SDK. Only media is replaced; routing and
Read through the identity-scoped SDK. Only media URLs are refreshed; routing and
sender authorization continue to use the accepted delivery. Lookup failures
propagate so a media-only message cannot silently run without its files.
"""
Expand All @@ -82,8 +82,29 @@ async def refresh_media(gateway, payload):
raise ValueError('Media message identity required')
lookup = gateway._identity.get_text if channel == 'text.received' else gateway._identity.get_imessage
fresh = await asyncio.to_thread(lookup, message_id)
fresh_id = fresh.get('id') if isinstance(fresh, dict) else getattr(fresh, 'id', None)
if fresh_id is not None and str(fresh_id) != str(message_id):
raise ValueError('Fresh media message identity mismatch')
media = fresh.get('media') if isinstance(fresh, dict) else fresh.media
if not media:
raise ValueError('Fresh media unavailable')
media = [asdict(item) if is_dataclass(item) else dict(item) for item in media]
return {**payload, 'data': {**payload['data'], key: {**message, 'media': media}}}
accepted = message['media']
modern = any(item.get('id') for item in media)
by_id = {str(item['id']): item for item in media if item.get('id')}
refreshed = []
for index, item in enumerate(accepted):
match = by_id.get(str(item['id'])) if modern and item.get('id') else None
if match is None and not modern and len(media) == len(accepted):
candidate = media[index]
if not (item.get('content_type') and candidate.get('content_type')
and item['content_type'] != candidate['content_type']):
match = candidate
if match is None:
raise ValueError('Fresh attachment identity unavailable')
current = {**item, 'url': match.get('url')}
result = match.get('transcription')
if isinstance(result, dict) and result.get('status') in {'completed', 'failed', 'skipped'}:
current['transcription'] = result
refreshed.append(current)
return {**payload, 'data': {**payload['data'], key: {**message, 'media': refreshed}}}
1 change: 1 addition & 0 deletions inkbox_codex/media.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ async def download_media(items: List[Any], *, prefix: str) -> List[Dict[str, Any
"path": str(target),
"content_type": content_type or "application/octet-stream",
"size": _item_field(item, "size"),
**({"transcription": _item_field(item, "transcription")} if _item_field(item, "transcription") is not None else {}),
})
return saved

Expand Down
Loading
Loading