http timeouts (#1970)

* fix: raise the http timeouts

I have a feeling this is preventing bufconnect to write the response sometimes, causing empty unary responses.
This could be the reason why our ratelimiting is flaky right now.

* refactor(cache): remove TODO comment in NewOtter function
fix(ratelimit): handle error and retry push pull operation in syncWithOrigin

apps/agent/pkg/cache/otter.go

@@ -77,7 +77,6 @@ func NewOtter[T any](config OtterConfig[T]) (Cache[T], error) {
 	go o.runRefreshing()
 
 	repeat.Every(time.Minute, func() {
-		// TODO: andreas replace with metrics
 		o.metrics.Record(CacheStats{
 			Resource: o.resource,
 			Size:     o.cache.Size(),

apps/agent/services/ratelimit/sync_with_origin.go

@@ -60,7 +60,12 @@ func (s *service) syncWithOrigin(req syncWithOriginRequest) {
 	if err != nil {
 		tracing.RecordError(span, err)
 		s.logger.Warn().Err(err).Interface("peer", peer).Str("peerId", peer.Id).Msg("failed to push pull")
-		return
+		res, err = c.PushPull(ctx, connectReq)
+		if err != nil {
+			tracing.RecordError(span, err)
+			s.logger.Error().Err(err).Str("peerId", peer.Id).Msg("failed to push pull again")
+			return
+		}
 	}
 
 	if len(req.events) != len(res.Msg.Updates) {