Skip to content

Commit

Permalink
#207 Delete expired messages (backend)
Browse files Browse the repository at this point in the history
  • Loading branch information
@falconandy
falconandy committed May 7, 2021
1 parent 0d2a2db commit 54d051e
Show file tree
Hide file tree
Showing 22 changed files with 1,377 additions and 767 deletions.
10 changes: 5 additions & 5 deletions backend/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/ansel1/merry v1.5.1
github.com/appleboy/go-fcm v0.1.5
github.com/aws/aws-sdk-go v1.38.28
github.com/aws/aws-sdk-go v1.38.35
github.com/containerd/containerd v1.4.3 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/disintegration/imaging v1.6.2
Expand All @@ -17,7 +17,6 @@ require (
github.com/go-chi/chi v4.1.2+incompatible
github.com/go-chi/cors v1.2.0
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang/protobuf v1.5.2
github.com/gorilla/mux v1.7.3 // indirect
github.com/gorilla/sessions v1.2.1
github.com/h2non/filetype v1.1.1
Expand All @@ -34,12 +33,13 @@ require (
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/sirupsen/logrus v1.7.0 // indirect
github.com/stretchr/testify v1.7.0
github.com/tidwall/gjson v1.7.5
github.com/tidwall/sjson v1.1.6
github.com/twitchtv/twirp v7.2.0+incompatible
github.com/twitchtv/twirp v8.0.0+incompatible
github.com/vmware-labs/yaml-jsonpointer v0.1.1
github.com/xhit/go-simple-mail/v2 v2.8.1
github.com/xhit/go-simple-mail/v2 v2.9.0
github.com/ziutek/mymysql v1.5.4 // indirect
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
google.golang.org/protobuf v1.26.0
Expand Down
22 changes: 11 additions & 11 deletions backend/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ github.com/appleboy/go-fcm v0.1.5 h1:fKbcZf/7vwGsvDkcop8a+kCHnK+tt4wXX0X7uEzwI6E
github.com/appleboy/go-fcm v0.1.5/go.mod h1:MSxZ4LqGRsnywOjnlXJXMqbjZrG4vf+0oHitfC9HRH0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.38.28 h1:2ZzgEupSluR18ClxUnHwXKyuADheZpMblXRAsHqF0tI=
github.com/aws/aws-sdk-go v1.38.28/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.38.35 h1:7AlAO0FC+8nFjxiGKEmq0QLpiA8/XFr6eIxgRTwkdTg=
github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
Expand Down Expand Up @@ -96,9 +96,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -267,22 +266,23 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/gjson v1.7.4 h1:19cchw8FOxkG5mdLRkGf9jqIqEyqdZhPqW60XfyFxk8=
github.com/tidwall/gjson v1.7.4/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
github.com/tidwall/gjson v1.7.5 h1:zmAN/xmX7OtpAkv4Ovfso60r/BiCi5IErCDYGNJu+uc=
github.com/tidwall/gjson v1.7.5/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk=
github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/sjson v1.1.6 h1:8fDdlahON04OZBlTQCIatW8FstSFJz8oxidj5h0rmSQ=
github.com/tidwall/sjson v1.1.6/go.mod h1:KN3FZ7odvXIHPbJdhNorK/M9lWweVUbXsXXhrJ/kGOA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twitchtv/twirp v7.2.0+incompatible h1:cXERdTtJqg8+OZdPCPGG2xWW8g+IKQ6zYjQTk9tWcCk=
github.com/twitchtv/twirp v7.2.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=
github.com/twitchtv/twirp v8.0.0+incompatible h1:uYHA8+9cit/+LUfQjL6zo/0QDKTo4U2H/WAnJ6LfhBU=
github.com/twitchtv/twirp v8.0.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/vmware-labs/yaml-jsonpointer v0.1.1 h1:gO9fa7NgCEGUM/+MhpHlrmh3kNTMzFTmoeBmWSagLz0=
github.com/vmware-labs/yaml-jsonpointer v0.1.1/go.mod h1:JDtZ8Hd3hkVzyCAoBXxOSfP9aVfUMAG9OSPfgQTvCQk=
github.com/xhit/go-simple-mail/v2 v2.8.1 h1:40ZWoTbU6eTq170ulrELSlCMCzKuce+JeKLwo6Qzk1E=
github.com/xhit/go-simple-mail/v2 v2.8.1/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4=
github.com/xhit/go-simple-mail/v2 v2.9.0 h1:vN4fb1Aw5BDtMeJuV/aTP82ufjdT8q0GmqiBjMKPN6I=
github.com/xhit/go-simple-mail/v2 v2.9.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
Expand All @@ -299,8 +299,8 @@ golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
Expand Down
23 changes: 6 additions & 17 deletions backend/messagehub/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"crypto/rsa"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
Expand All @@ -21,6 +22,7 @@ import (
"github.com/mreider/koto/backend/messagehub/config"
"github.com/mreider/koto/backend/messagehub/migrate"
"github.com/mreider/koto/backend/messagehub/repo"
"github.com/mreider/koto/backend/messagehub/services"
"github.com/mreider/koto/backend/token"
)

Expand Down Expand Up @@ -83,7 +85,10 @@ func main() {
s3Cleaner := common.NewS3Cleaner(db, s3Storage)
go s3Cleaner.Clean(context.Background())

go cleanOldGuestMessages(repos.Message)
messageCleaner := services.NewMessageCleaner(cfg.ExternalAddress,
fmt.Sprintf("%s/rpc.MessageHubInternalService/ExpirationDays", cfg.UserHubAddress),
tokenGenerator)
go messageCleaner.DeleteExpiredMessages(repos.Message)

server := messagehub.NewServer(cfg, repos, tokenParser, s3Storage, tokenGenerator, string(publicKeyPEM), db)
err = server.Run()
Expand Down Expand Up @@ -146,19 +151,3 @@ func loadConfig(execDir string) (config.Config, error) {

return cfg, nil
}

func cleanOldGuestMessages(messageRepo repo.MessageRepo) {
ticker := time.NewTicker(time.Hour * 1)
defer ticker.Stop()

for range ticker.C {
func() {
defer func() {
if r := recover(); r != nil {
log.Println("can't delete old guest messages:", r)
}
}()
messageRepo.DeleteOldGuestMessages(time.Now().UTC().AddDate(0, 0, -30))
}()
}
}
40 changes: 23 additions & 17 deletions backend/messagehub/repo/message_repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package repo
import (
"database/sql"
"errors"
"log"
"time"

"github.com/ansel1/merry"
Expand Down Expand Up @@ -82,7 +83,7 @@ type MessageRepo interface {
DeleteReportedMessage(reportID string) bool
BlockReportedUser(reportID string) bool
ResolveMessageReport(reportID string) bool
DeleteOldGuestMessages(until time.Time)
DeleteExpiredMessages(until time.Time, onlyGuestMessages bool)
MarkRead(userID string, messageIDs []string)
Counts(currentUserID string, userIDs []string) Counts
GroupCounts(currentUserID string, groupIDs []string) map[string]Counts
Expand Down Expand Up @@ -645,81 +646,86 @@ func (r *messageRepo) ResolveMessageReport(reportID string) bool {
return true
}

func (r *messageRepo) DeleteOldGuestMessages(until time.Time) {
func (r *messageRepo) DeleteExpiredMessages(until time.Time, onlyGuestMessages bool) {
until = until.UTC()
err := common.RunInTransaction(r.db, func(tx *sqlx.Tx) error {
_, err := tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
delete from message_likes
where message_id in (select id from mc);`,
until)
onlyGuestMessages, until)
if err != nil {
return merry.Wrap(err)
}

_, err = tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
delete from message_visibility
where message_id in (select id from mc);`,
until)
onlyGuestMessages, until)
if err != nil {
return merry.Wrap(err)
}

_, err = tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
delete from message_reports
where message_id in (select id from mc);`,
until)
onlyGuestMessages, until)
if err != nil {
return merry.Wrap(err)
}

_, err = tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
delete from message_reads
where message_id in (select id from mc);`,
until)
onlyGuestMessages, until)
if err != nil {
return merry.Wrap(err)
}

_, err = tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
insert into blob_pending_deletes(blob_id, deleted_at)
select attachment_id, $2::timestamptz
select attachment_id, $3::timestamptz
from messages
where id in (select id from mc) and attachment_id != ''
union
select attachment_thumbnail_id, $2::timestamptz
select attachment_thumbnail_id, $3::timestamptz
from messages
where id in (select id from mc) and attachment_thumbnail_id != '';`,
until, common.CurrentTimestamp())
onlyGuestMessages, until, common.CurrentTimestamp())
if err != nil {
return merry.Wrap(err)
}

_, err = tx.Exec(`
with m as (select id from messages where parent_id is null and is_guest = true and created_at < $1),
res, err := tx.Exec(`
with m as (select id from messages where parent_id is null and ($1 = false or is_guest = true) and created_at < $2),
c as (select id from messages where parent_id in (select id from m)),
mc as (select id from m union select id from c)
delete from messages
where id in (select id from mc);`,
until)
onlyGuestMessages, until)
if err != nil {
return merry.Wrap(err)
}

rowsAffected, err := res.RowsAffected()
if err == nil {
log.Printf("Deleted %d expired messages", rowsAffected)
}

return nil
})
if err != nil {
Expand Down
Loading

0 comments on commit 54d051e

Please sign in to comment.