From 9498f39f78c18d55fe44022d2ccaa2308a8860e0 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Mon, 26 Feb 2024 13:13:03 +0100 Subject: [PATCH 01/22] remove util.ArrayContains Since Go 1.21, there is a slices package in the standard library, that implements the same functionality more comprehensively. Replace the uses of ArrayContains with slices.Contains. --- cmd/plugins/plugin_antispam_ocr/main.go | 3 ++- thumbnailing/i/bmp.go | 7 +++---- thumbnailing/i/heif.go | 4 ++-- thumbnailing/i/jpg.go | 7 +++---- thumbnailing/i/mp4.go | 11 +++++------ thumbnailing/thumbnail.go | 6 +++--- util/arrays.go | 11 ----------- util/mime.go | 3 ++- 8 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 util/arrays.go diff --git a/cmd/plugins/plugin_antispam_ocr/main.go b/cmd/plugins/plugin_antispam_ocr/main.go index d641488a..4b16d284 100644 --- a/cmd/plugins/plugin_antispam_ocr/main.go +++ b/cmd/plugins/plugin_antispam_ocr/main.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "regexp" + "slices" "strings" "time" @@ -77,7 +78,7 @@ func (a *AntispamOCR) CheckForSpam(b64 string, filename string, contentType stri if len(b) < a.minSize || len(b) > a.maxSize { return false, nil } - if !util.ArrayContains(a.contentTypes, contentType) { + if !slices.Contains(a.contentTypes, contentType) { return false, nil } if !a.userIdRegex.MatchString(userId) { diff --git a/thumbnailing/i/bmp.go b/thumbnailing/i/bmp.go index bbf6d50d..eb45514c 100644 --- a/thumbnailing/i/bmp.go +++ b/thumbnailing/i/bmp.go @@ -3,15 +3,14 @@ package i import ( "errors" "io" + "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" "golang.org/x/image/bmp" ) -type bmpGenerator struct { -} +type bmpGenerator struct{} func (d bmpGenerator) supportedContentTypes() []string { return []string{"image/bmp", "image/x-bmp"} @@ -22,7 +21,7 @@ func (d bmpGenerator) supportsAnimation() bool { } func (d bmpGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d bmpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { diff --git a/thumbnailing/i/heif.go b/thumbnailing/i/heif.go index 8d4b005e..c6f2fb9a 100644 --- a/thumbnailing/i/heif.go +++ b/thumbnailing/i/heif.go @@ -4,11 +4,11 @@ import ( "errors" "image" "io" + "slices" _ "github.com/strukturag/libheif/go/heif" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" ) type heifGenerator struct { @@ -23,7 +23,7 @@ func (d heifGenerator) supportsAnimation() bool { } func (d heifGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d heifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { diff --git a/thumbnailing/i/jpg.go b/thumbnailing/i/jpg.go index f05d8190..de20b188 100644 --- a/thumbnailing/i/jpg.go +++ b/thumbnailing/i/jpg.go @@ -5,17 +5,16 @@ import ( "image" _ "image/jpeg" "io" + "slices" "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/readers" ) -type jpgGenerator struct { -} +type jpgGenerator struct{} func (d jpgGenerator) supportedContentTypes() []string { return []string{"image/jpeg", "image/jpg"} @@ -26,7 +25,7 @@ func (d jpgGenerator) supportsAnimation() bool { } func (d jpgGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d jpgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { diff --git a/thumbnailing/i/mp4.go b/thumbnailing/i/mp4.go index 99d69ad4..cd59d66e 100644 --- a/thumbnailing/i/mp4.go +++ b/thumbnailing/i/mp4.go @@ -6,14 +6,13 @@ import ( "os" "os/exec" "path" + "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" ) -type mp4Generator struct { -} +type mp4Generator struct{} func (d mp4Generator) supportedContentTypes() []string { return []string{"video/mp4"} @@ -24,7 +23,7 @@ func (d mp4Generator) supportsAnimation() bool { } func (d mp4Generator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d mp4Generator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { @@ -44,7 +43,7 @@ func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width i defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { return nil, errors.New("mp4: error creating temp video file: " + err.Error()) } @@ -57,7 +56,7 @@ func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width i return nil, errors.New("mp4: error converting video file: " + err.Error()) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { return nil, errors.New("mp4: error reading temp png file: " + err.Error()) } diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index 2323aa06..30ad858a 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -4,20 +4,20 @@ import ( "errors" "io" "reflect" + "slices" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/i" "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/readers" ) var ErrUnsupported = errors.New("unsupported thumbnail type") func IsSupported(contentType string) bool { - return util.ArrayContains(i.GetSupportedContentTypes(), contentType) + return slices.Contains(i.GetSupportedContentTypes(), contentType) } func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { @@ -26,7 +26,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h ctx.Log.Debugf("Unsupported content type '%s'", contentType) return nil, ErrUnsupported } - if !util.ArrayContains(ctx.Config.Thumbnails.Types, contentType) { + if !slices.Contains(ctx.Config.Thumbnails.Types, contentType) { ctx.Log.Debugf("Disabled content type '%s'", contentType) return nil, ErrUnsupported } diff --git a/util/arrays.go b/util/arrays.go deleted file mode 100644 index 4c9255bb..00000000 --- a/util/arrays.go +++ /dev/null @@ -1,11 +0,0 @@ -package util - -func ArrayContains(a []string, v string) bool { - for _, e := range a { - if e == v { - return true - } - } - - return false -} diff --git a/util/mime.go b/util/mime.go index 8d34d03f..1f6c4743 100644 --- a/util/mime.go +++ b/util/mime.go @@ -2,6 +2,7 @@ package util import ( "mime" + "slices" "strings" ) @@ -19,7 +20,7 @@ func ExtensionForContentType(ct string) string { func CanInline(ct string) bool { ct = FixContentType(ct) - return ArrayContains(InlineContentTypes, ct) + return slices.Contains(InlineContentTypes, ct) } var InlineContentTypes = []string{ From 8b0a8e6755bcf4032bacd467354e8fe3ac3c5655 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Mon, 26 Feb 2024 15:33:03 +0100 Subject: [PATCH 02/22] use format strings for string building --- api/_responses/errors.go | 18 +++++--- api/_routers/00-install-params.go | 14 +----- api/_routers/03-host_detection.go | 7 +-- api/_routers/97-optional-access-token.go | 2 +- api/_routers/97-require-access-token.go | 2 +- api/_routers/98-use-rcontext.go | 8 ++-- api/custom/datastores.go | 26 ++++++----- api/custom/exports.go | 59 ++++++++++++------------ api/custom/federation.go | 9 ++-- api/custom/imports.go | 24 +++++----- api/custom/media_attributes.go | 17 +++---- api/custom/purge.go | 47 ++++++++++--------- api/custom/quarantine.go | 17 +++---- api/custom/tasks.go | 15 +++--- api/custom/usage.go | 41 ++++++++-------- api/r0/download.go | 10 ++-- api/r0/identicon.go | 5 +- api/r0/logout.go | 9 ++-- api/r0/preview_url.go | 16 +++---- api/r0/thumbnail.go | 22 ++++----- api/r0/upload_async.go | 4 +- api/r0/upload_sync.go | 2 +- api/router.go | 37 +++++++-------- api/unstable/info.go | 8 ++-- api/unstable/local_copy.go | 10 ++-- api/v1/create.go | 3 +- database/db.go | 34 +++++++------- database/table_expiring_media.go | 11 +++-- database/table_export_parts.go | 11 +++-- database/table_exports.go | 9 ++-- database/table_last_access.go | 6 +-- database/table_media.go | 43 ++++++++--------- database/table_media_attributes.go | 7 +-- database/table_media_hold.go | 8 ++-- database/table_reserved_media.go | 7 +-- database/table_tasks.go | 15 +++--- database/table_thumbnails.go | 19 ++++---- database/table_url_previews.go | 9 ++-- database/table_user_stats.go | 5 +- database/virtualtable_metadata.go | 16 +++---- datastores/buffer.go | 9 ++-- matrix/server_discovery.go | 38 ++++++++------- pipelines/_steps/meta/flag_access.go | 2 +- pipelines/_steps/upload/lock.go | 3 +- plugins/manager.go | 4 +- tasks/exec.go | 2 +- test/test_internals/deps.go | 6 +-- test/test_internals/deps_minio.go | 2 +- test/test_internals/deps_mmr.go | 4 +- test/test_internals/deps_network.go | 2 +- thumbnailing/i/apng.go | 8 ++-- thumbnailing/i/bmp.go | 4 +- thumbnailing/i/flac.go | 9 ++-- thumbnailing/i/gif.go | 10 ++-- thumbnailing/i/heif.go | 7 ++- thumbnailing/i/jpegxl.go | 16 +++---- thumbnailing/i/jpg.go | 8 ++-- thumbnailing/i/mp3.go | 12 ++--- thumbnailing/i/mp4.go | 12 ++--- thumbnailing/i/ogg.go | 9 ++-- thumbnailing/i/png.go | 6 +-- thumbnailing/i/svg.go | 16 +++---- thumbnailing/i/tiff.go | 4 +- thumbnailing/i/wav.go | 6 +-- thumbnailing/i/webp.go | 4 +- thumbnailing/thumbnail.go | 3 +- thumbnailing/u/exif.go | 12 ++--- thumbnailing/u/framing.go | 2 +- thumbnailing/u/metadata.go | 6 +-- thumbnailing/u/sample.go | 3 +- 70 files changed, 443 insertions(+), 418 deletions(-) diff --git a/api/_responses/errors.go b/api/_responses/errors.go index f22b33d1..1471b207 100644 --- a/api/_responses/errors.go +++ b/api/_responses/errors.go @@ -1,6 +1,10 @@ package _responses -import "github.com/t2bot/matrix-media-repo/common" +import ( + "fmt" + + "github.com/t2bot/matrix-media-repo/common" +) type ErrorResponse struct { Code string `json:"errcode"` @@ -8,12 +12,12 @@ type ErrorResponse struct { InternalCode string `json:"mr_errcode"` } -func InternalServerError(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown} +func InternalServerError(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeUnknown} } -func BadGatewayError(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown} +func BadGatewayError(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeUnknown} } func MethodNotAllowed() *ErrorResponse { @@ -48,8 +52,8 @@ func GuestAuthFailed() *ErrorResponse { return &ErrorResponse{common.ErrCodeNoGuests, "Guests cannot use this endpoint", common.ErrCodeNoGuests} } -func BadRequest(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeBadRequest} +func BadRequest(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeBadRequest} } func QuotaExceeded() *ErrorResponse { diff --git a/api/_routers/00-install-params.go b/api/_routers/00-install-params.go index 11b42f33..2f58f178 100644 --- a/api/_routers/00-install-params.go +++ b/api/_routers/00-install-params.go @@ -2,25 +2,15 @@ package _routers import ( "context" - "errors" "net/http" "regexp" "github.com/julienschmidt/httprouter" ) -func localCompile(expr string) *regexp.Regexp { - r, err := regexp.Compile(expr) - if err != nil { - panic(errors.New("error compiling expression: " + expr + " | " + err.Error())) - } - return r -} - -var ServerNameRegex = localCompile("[a-zA-Z0-9.:\\-_]+") - -//var NumericIdRegex = localCompile("[0-9]+") +var ServerNameRegex = regexp.MustCompile("[a-zA-Z0-9.:\\-_]+") +// var NumericIdRegex = regexp.MustCompile("[0-9]+") func GetParam(name string, r *http.Request) string { p := httprouter.ParamsFromContext(r.Context()) if p == nil { diff --git a/api/_routers/03-host_detection.go b/api/_routers/03-host_detection.go index 830e48f0..7e908559 100644 --- a/api/_routers/03-host_detection.go +++ b/api/_routers/03-host_detection.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net" "net/http" "strings" @@ -60,11 +61,11 @@ func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { }).Inc() logger.Warnf("The server name provided ('%s') in the Host header is not configured, or the request was made directly to the media repo. Please specify a Host header and check your reverse proxy configuration. The request is being rejected.", r.Host) w.WriteHeader(http.StatusBadGateway) - if b, err := json.Marshal(_responses.BadGatewayError("Review server logs to continue")); err != nil { - panic(errors.New("error preparing BadGatewayError: " + err.Error())) + if b, err := json.Marshal(_responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { + panic(fmt.Errorf("error preparing BadGatewayError: %v", err)) } else { if _, err = w.Write(b); err != nil { - panic(errors.New("error sending BadGatewayError: " + err.Error())) + panic(fmt.Errorf("error sending BadGatewayError: %v", err)) } } return // don't call next handler diff --git a/api/_routers/97-optional-access-token.go b/api/_routers/97-optional-access-token.go index 9950b5bc..5735d6eb 100644 --- a/api/_routers/97-optional-access-token.go +++ b/api/_routers/97-optional-access-token.go @@ -39,7 +39,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { if !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError("unexpected error validating access token") + return _responses.InternalServerError(errors.New("unexpected error validating access token")) } ctx.Log.Warn("Failed to verify token (non-fatal): ", err) diff --git a/api/_routers/97-require-access-token.go b/api/_routers/97-require-access-token.go index ba7dda5e..995071a7 100644 --- a/api/_routers/97-require-access-token.go +++ b/api/_routers/97-require-access-token.go @@ -45,7 +45,7 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { if err != nil && !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError("unexpected error validating access token") + return _responses.InternalServerError(errors.New("unexpected error validating access token")) } return _responses.AuthFailed() } diff --git a/api/_routers/98-use-rcontext.go b/api/_routers/98-use-rcontext.go index 35e36369..3b0a5422 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/_routers/98-use-rcontext.go @@ -88,7 +88,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { r = writeStatusCode(w, r, http.StatusOK) if _, err := w.Write([]byte(htmlRes.HTML)); err != nil { - panic(errors.New("error sending HtmlResponse: " + err.Error())) + panic(fmt.Errorf("error sending HtmlResponse: %w", err)) } return // don't continue } @@ -104,16 +104,16 @@ beforeParseDownload: ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes) if errors.Is(err, http_range.ErrInvalid) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("invalid range header") + res = _responses.BadRequest(errors.New("invalid range header")) goto beforeParseDownload // reprocess `res` } else if errors.Is(err, http_range.ErrNoOverlap) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("out of range") + res = _responses.BadRequest(errors.New("out of range")) goto beforeParseDownload // reprocess `res` } if len(ranges) > 1 { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("only 1 range is supported") + res = _responses.BadRequest(errors.New("only 1 range is supported")) goto beforeParseDownload // reprocess `res` } diff --git a/api/custom/datastores.go b/api/custom/datastores.go index 5985611d..9e43907c 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -1,6 +1,11 @@ package custom import ( + "errors" + "fmt" + "net/http" + "strconv" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" @@ -9,9 +14,6 @@ import ( "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" - "net/http" - "strconv" - "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -29,7 +31,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { sentry.CaptureException(err) rctx.Log.Error("Error getting datastore URI: ", err) - return _responses.InternalServerError("unexpected error getting datastore information") + return _responses.InternalServerError(errors.New("unexpected error getting datastore information")) } dsMap := make(map[string]interface{}) dsMap["type"] = ds.Type @@ -47,7 +49,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -61,20 +63,20 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use }) if sourceDsId == targetDsId { - return _responses.BadRequest("Source and target datastore cannot be the same") + return _responses.BadRequest(errors.New("Source and target datastore cannot be the same")) } if _, ok := datastores.Get(rctx, sourceDsId); !ok { - return _responses.BadRequest("Source datastore does not appear to exist") + return _responses.BadRequest(errors.New("Source datastore does not appear to exist")) } if _, ok := datastores.Get(rctx, targetDsId); !ok { - return _responses.BadRequest("Target datastore does not appear to exist") + return _responses.BadRequest(errors.New("Target datastore does not appear to exist")) } estimate, err := datastores.SizeOfDsIdWithAge(rctx, sourceDsId, beforeTs) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error getting storage estimate") + return _responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } rctx.Log.Infof("User %s has started a datastore media transfer", user.UserId) @@ -82,7 +84,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error starting migration") + return _responses.InternalServerError(errors.New("Unexpected error starting migration")) } migration := &DatastoreMigration{ @@ -100,7 +102,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -115,7 +117,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error getting storage estimate") + return _responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } return &_responses.DoNotCacheResponse{Payload: result} } diff --git a/api/custom/exports.go b/api/custom/exports.go index 59a89af0..6fd71c8d 100644 --- a/api/custom/exports.go +++ b/api/custom/exports.go @@ -2,6 +2,7 @@ package custom import ( "bytes" + "errors" "net/http" "strconv" @@ -39,7 +40,7 @@ type ExportMetadata struct { func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared @@ -52,7 +53,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta userId := _routers.GetParam("userId", r) if !isAdmin && user.UserId != userId { - return _responses.BadRequest("cannot export data for another user") + return _responses.BadRequest(errors.New("cannot export data for another user")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -63,7 +64,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting export") + return _responses.InternalServerError(errors.New("fatal error starting export")) } return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ @@ -74,7 +75,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared @@ -91,7 +92,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime // We won't be able to check unless we know about the homeserver though if !util.IsServerOurs(serverName) { - return _responses.BadRequest("cannot export data for another server") + return _responses.BadRequest(errors.New("cannot export data for another server")) } isLocalAdmin, err := matrix.IsUserAdmin(rctx, serverName, user.AccessToken, r.RemoteAddr) @@ -100,7 +101,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime isLocalAdmin = false } if !isLocalAdmin { - return _responses.BadRequest("cannot export data for another server") + return _responses.BadRequest(errors.New("cannot export data for another server")) } } @@ -112,7 +113,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting export") + return _responses.InternalServerError(errors.New("fatal error starting export")) } return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ @@ -123,13 +124,13 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + _responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -143,7 +144,7 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get entity for export ID") + return _responses.InternalServerError(errors.New("failed to get entity for export ID")) } if entityId == "" { return _responses.NotFoundError() @@ -153,14 +154,14 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return _responses.InternalServerError(errors.New("failed to get export parts")) } template, err := templating.GetTemplate("view_export") if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get template") + return _responses.InternalServerError(errors.New("failed to get template")) } model := &templating.ViewExportModel{ @@ -183,7 +184,7 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to render template") + return _responses.InternalServerError(errors.New("failed to render template")) } return &_responses.HtmlResponse{HTML: html.String()} @@ -191,13 +192,13 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + _responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -211,14 +212,14 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get entity for export ID") + return _responses.InternalServerError(errors.New("failed to get entity for export ID")) } parts, err := partsDb.GetForExport(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return _responses.InternalServerError(errors.New("failed to get export parts")) } metadata := &ExportMetadata{ @@ -238,20 +239,20 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apim func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) pid := _routers.GetParam("partId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + _responses.BadRequest(errors.New("invalid export ID")) } partId, err := strconv.Atoi(pid) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest("invalid part index") + return _responses.BadRequest(errors.New("invalid part index")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -264,7 +265,7 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get part") + return _responses.InternalServerError(errors.New("failed to get part")) } if part == nil { @@ -274,13 +275,13 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api dsConf, ok := datastores.Get(rctx, part.DatastoreId) if !ok { sentry.CaptureMessage("failed to locate datastore") - return _responses.InternalServerError("failed to locate datastore") + return _responses.InternalServerError(errors.New("failed to locate datastore")) } s, err := datastores.Download(rctx, dsConf, part.Location) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to start download") + return _responses.InternalServerError(errors.New("failed to start download")) } return &_responses.DownloadResponse{ @@ -294,13 +295,13 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + _responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -315,7 +316,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return _responses.InternalServerError(errors.New("failed to get export parts")) } for _, part := range parts { @@ -324,7 +325,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export part") + return _responses.InternalServerError(errors.New("failed to delete export part")) } } @@ -333,13 +334,13 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export parts") + return _responses.InternalServerError(errors.New("failed to delete export parts")) } err = exportDb.Delete(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export record") + return _responses.InternalServerError(errors.New("failed to delete export record")) } return _responses.EmptyResponse{} diff --git a/api/custom/federation.go b/api/custom/federation.go index 48a0fb92..f0b51eae 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -2,6 +2,7 @@ package custom import ( "encoding/json" + "errors" "net/http" "github.com/getsentry/sentry-go" @@ -18,7 +19,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -29,7 +30,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return _responses.InternalServerError(err) } versionUrl := url + "/_matrix/federation/v1/version" @@ -37,7 +38,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return _responses.InternalServerError(err) } decoder := json.NewDecoder(versionResponse.Body) @@ -46,7 +47,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return _responses.InternalServerError(err) } resp := make(map[string]interface{}) diff --git a/api/custom/imports.go b/api/custom/imports.go index a667834a..50edfe0e 100644 --- a/api/custom/imports.go +++ b/api/custom/imports.go @@ -23,24 +23,24 @@ type ImportStarted struct { func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } task, importId, err := tasks.RunImport(rctx) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting import") + return _responses.InternalServerError(errors.New("fatal error starting import")) } err = task_runner.AppendImportFile(rctx, importId, r.Body) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error appending first file to import") + return _responses.InternalServerError(errors.New("error appending first file to import")) } return &_responses.DoNotCacheResponse{Payload: &ImportStarted{ @@ -51,16 +51,16 @@ func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Us func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } importId := _routers.GetParam("importId", r) if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest("invalid import ID") + return _responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.AppendImportFile(rctx, importId, r.Body) @@ -70,7 +70,7 @@ func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error appending to import") + return _responses.InternalServerError(errors.New("error appending to import")) } return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} @@ -78,16 +78,16 @@ func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return _responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } importId := _routers.GetParam("importId", r) if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest("invalid import ID") + return _responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.FinishImport(rctx, importId) @@ -97,7 +97,7 @@ func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error stopping import") + return _responses.InternalServerError(errors.New("error stopping import")) } return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go index 8f9abda1..a31a884d 100644 --- a/api/custom/media_attributes.go +++ b/api/custom/media_attributes.go @@ -2,6 +2,7 @@ package custom import ( "encoding/json" + "errors" "net/http" "github.com/getsentry/sentry-go" @@ -37,7 +38,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest("invalid origin") + return _responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -55,7 +56,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get media record") + return _responses.InternalServerError(errors.New("failed to get media record")) } if media == nil { return _responses.NotFoundError() @@ -66,7 +67,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get attributes record") + return _responses.InternalServerError(errors.New("failed to get attributes record")) } retAttrs := &Attributes{ Purpose: database.PurposeNone, @@ -83,7 +84,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest("invalid origin") + return _responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -102,7 +103,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to read attributes") + return _responses.InternalServerError(errors.New("failed to read attributes")) } attrDb := database.GetInstance().MediaAttributes.Prepare(rctx) @@ -110,18 +111,18 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get attributes") + return _responses.InternalServerError(errors.New("failed to get attributes")) } if attrs == nil || attrs.Purpose != newAttrs.Purpose { if !database.IsPurpose(newAttrs.Purpose) { - return _responses.BadRequest("unknown purpose") + return _responses.BadRequest(errors.New("unknown purpose")) } err = attrDb.UpsertPurpose(origin, mediaId, newAttrs.Purpose) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to update attributes: purpose") + return _responses.InternalServerError(errors.New("failed to update attributes: purpose")) } } diff --git a/api/custom/purge.go b/api/custom/purge.go index cc8ef012..f5b8b997 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -2,6 +2,7 @@ package custom import ( "errors" + "fmt" "net/http" "strconv" @@ -26,11 +27,11 @@ type MediaPurgedResponse struct { func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr == "" { - return _responses.BadRequest("Missing before_ts argument") + return _responses.BadRequest(errors.New("Missing before_ts argument")) } beforeTs, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -42,7 +43,7 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error("Error purging remote media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Error purging remote media") + return _responses.InternalServerError(errors.New("Error purging remote media")) } return &_responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} @@ -55,7 +56,7 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _ mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -75,7 +76,7 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _ } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} @@ -98,7 +99,7 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return _responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -110,7 +111,7 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} @@ -123,7 +124,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -132,7 +133,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if includeLocalStr != "" { includeLocal, err = strconv.ParseBool(includeLocalStr) if err != nil { - return _responses.BadRequest("Error parsing include_local: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing include_local: %w", err)) } } @@ -151,7 +152,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return _responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, &task_runner.PurgeAuthContext{}, &task_runner.QuarantineThis{ @@ -163,7 +164,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} @@ -181,7 +182,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -194,9 +195,9 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta _, userDomain, err := util.SplitUserId(userId) if err != nil { - rctx.Log.Error("Error parsing user ID ("+userId+"): ", err) + rctx.Log.Errorf("Error parsing user ID (%s): %v", userId, err) sentry.CaptureException(err) - return _responses.InternalServerError("error parsing user ID") + return _responses.InternalServerError(errors.New("error parsing user ID")) } if !isGlobalAdmin && userDomain != r.Host { @@ -208,7 +209,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return _responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -220,7 +221,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} @@ -238,7 +239,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -253,7 +254,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media in room") + return _responses.InternalServerError(errors.New("error retrieving media in room")) } mxcs := make([]string, 0) @@ -293,7 +294,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} @@ -311,14 +312,14 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %f", err)) } } serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -335,7 +336,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return _responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -347,7 +348,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go index 26954935..4c0d7bd9 100644 --- a/api/custom/quarantine.go +++ b/api/custom/quarantine.go @@ -1,6 +1,7 @@ package custom import ( + "errors" "net/http" "github.com/getsentry/sentry-go" @@ -37,7 +38,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media in room") + return _responses.InternalServerError(errors.New("error retrieving media in room")) } var mxcs []string @@ -68,7 +69,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error parsing user ID ("+userId+"): ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error parsing user ID") + return _responses.InternalServerError(errors.New("error parsing user ID")) } if !allowOtherHosts && userDomain != r.Host { @@ -80,7 +81,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error while listing media for the user: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media for user") + return _responses.InternalServerError(errors.New("error retrieving media for user")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -97,7 +98,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -114,7 +115,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ if err != nil { rctx.Log.Error("Error while listing media for the server: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media for server") + return _responses.InternalServerError(errors.New("error retrieving media for server")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -132,7 +133,7 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimet mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -142,7 +143,7 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimet }) if !allowOtherHosts && r.Host != server { - return _responses.BadRequest("unable to quarantine media on other homeservers") + return _responses.BadRequest(errors.New("unable to quarantine media on other homeservers")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -163,7 +164,7 @@ func performQuarantineRequest(ctx rcontext.RequestContext, host string, allowOth if err != nil { ctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error quarantining media") + return _responses.InternalServerError(errors.New("error quarantining media")) } return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} diff --git a/api/custom/tasks.go b/api/custom/tasks.go index 2002ab9f..b3814f25 100644 --- a/api/custom/tasks.go +++ b/api/custom/tasks.go @@ -1,15 +1,16 @@ package custom import ( + "errors" + "net/http" + "strconv" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/database" - "net/http" - "strconv" - "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -29,7 +30,7 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserIn taskId, err := strconv.Atoi(taskIdStr) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest("invalid task ID") + return _responses.BadRequest(errors.New("invalid task ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -42,7 +43,7 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserIn if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get task information") + return _responses.InternalServerError(errors.New("failed to get task information")) } if task == nil { return _responses.NotFoundError() @@ -66,7 +67,7 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get background tasks") + return _responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) @@ -92,7 +93,7 @@ func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get background tasks") + return _responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) diff --git a/api/custom/usage.go b/api/custom/usage.go index 3025ebdb..48cff4ea 100644 --- a/api/custom/usage.go +++ b/api/custom/usage.go @@ -1,6 +1,8 @@ package custom import ( + "errors" + "fmt" "net/http" "strconv" "strings" @@ -56,7 +58,7 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -69,14 +71,14 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get byte usage for server") + return _responses.InternalServerError(errors.New("Failed to get byte usage for server")) } mediaCount, thumbCount, err := db.CountUsageForServer(serverName) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get count usage for server") + return _responses.InternalServerError(errors.New("Failed to get count usage for server")) } return &_responses.DoNotCacheResponse{ @@ -104,7 +106,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U userIds := r.URL.Query()["user_id"] if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -124,7 +126,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get media records for users") + return _responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*UserUsageEntry) @@ -163,7 +165,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet mxcs := r.URL.Query()["mxc"] if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -183,11 +185,11 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Error parsing MXC " + mxc) + return _responses.InternalServerError(fmt.Errorf("Error parsing MXC %s", mxc)) } if o != serverName { - return _responses.BadRequest("MXC URIs must match the requested server") + return _responses.BadRequest(errors.New("MXC URIs must match the requested server")) } split = append(split, i) @@ -198,7 +200,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get media records for users") + return _responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*MediaUsageEntry) @@ -232,7 +234,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ } if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + return _responses.BadRequest(errors.New("invalid server name")) } isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user) @@ -245,14 +247,14 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ orderBy = database.DefaultSynStatUserOrderBy } if !database.IsSynStatUserOrderBy(orderBy) { - return _responses.BadRequest("Query parameter 'order_by' must be one of the accepted values") + return _responses.BadRequest(errors.New("Query parameter 'order_by' must be one of the accepted values")) } var start int64 = 0 if len(qs["from"]) > 0 { start, err = strconv.ParseInt(qs.Get("from"), 10, 64) if err != nil || start < 0 { - return _responses.BadRequest("Query parameter 'from' must be a non-negative integer") + return _responses.BadRequest(errors.New("Query parameter 'from' must be a non-negative integer")) } } @@ -260,7 +262,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["limit"]) > 0 { limit, err = strconv.ParseInt(qs.Get("limit"), 10, 64) if err != nil || limit < 0 { - return _responses.BadRequest("Query parameter 'limit' must be a non-negative integer") + return _responses.BadRequest(errors.New("Query parameter 'limit' must be a non-negative integer")) } } if limit > 50 { @@ -272,7 +274,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["from_ts"]) > 0 { fromTS, err = strconv.ParseInt(qs.Get("from_ts"), 10, 64) if err != nil || fromTS < 0 { - return _responses.BadRequest("Query parameter 'from_ts' must be a non-negative integer") + return _responses.BadRequest(errors.New("Query parameter 'from_ts' must be a non-negative integer")) } } @@ -280,15 +282,15 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["until_ts"]) > 0 { untilTS, err = strconv.ParseInt(qs.Get("until_ts"), 10, 64) if err != nil || untilTS < 0 { - return _responses.BadRequest("Query parameter 'until_ts' must be a non-negative integer") + return _responses.BadRequest(errors.New("Query parameter 'until_ts' must be a non-negative integer")) } else if untilTS <= fromTS { - return _responses.BadRequest("Query parameter 'until_ts' must be greater than 'from_ts'") + return _responses.BadRequest(errors.New("Query parameter 'until_ts' must be greater than 'from_ts'")) } } searchTerm := qs.Get("search_term") if searchTerm == "" && len(qs["search_term"]) > 0 { - return _responses.BadRequest("Query parameter 'search_term' cannot be an empty string") + return _responses.BadRequest(errors.New("Query parameter 'search_term' cannot be an empty string")) } isAscendingOrder := true @@ -296,7 +298,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if direction == "b" && len(qs["dir"]) > 0 { isAscendingOrder = false } else { - return _responses.BadRequest("Query parameter 'dir' must be one of ['f', 'b']") + return _responses.BadRequest(errors.New("Query parameter 'dir' must be one of ['f', 'b']")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -313,11 +315,10 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ db := database.GetInstance().MetadataView.Prepare(rctx) stats, totalCount, err := db.UnoptimizedSynapseUserStatsPage(serverName, orderBy, start, limit, fromTS, untilTS, searchTerm, isAscendingOrder) - if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get users' usage stats on specified server") + return _responses.InternalServerError(errors.New("Failed to get users' usage stats on specified server")) } result := &synapse.SynUserStatsResponse{ diff --git a/api/r0/download.go b/api/r0/download.go index ab13d4a8..a662cac1 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -27,14 +27,14 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. timeoutMs := r.URL.Query().Get("timeout_ms") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest("allow_remote flag does not appear to be a boolean") + return _responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -43,14 +43,14 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest("allow_redirect flag does not appear to be a boolean") + return _responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } blockFor, err := util.CalcBlockForDuration(timeoutMs) if err != nil { - return _responses.BadRequest("timeout_ms does not appear to be an integer") + return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -91,7 +91,7 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } if filename == "" { diff --git a/api/r0/identicon.go b/api/r0/identicon.go index eff16d22..64c6e527 100644 --- a/api/r0/identicon.go +++ b/api/r0/identicon.go @@ -2,6 +2,7 @@ package r0 import ( "crypto/md5" + "fmt" "image/color" "io" "net/http" @@ -32,14 +33,14 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if widthStr != "" { width, err = strconv.Atoi(widthStr) if err != nil { - return _responses.InternalServerError("Error parsing width: " + err.Error()) + return _responses.InternalServerError(fmt.Errorf("Error parsing width: %w", err)) } height = width } if heightStr != "" { height, err = strconv.Atoi(heightStr) if err != nil { - return _responses.InternalServerError("Error parsing height: " + err.Error()) + return _responses.InternalServerError(fmt.Errorf("Error parsing height: %w", err)) } } diff --git a/api/r0/logout.go b/api/r0/logout.go index 7a15ef47..18a5e6b8 100644 --- a/api/r0/logout.go +++ b/api/r0/logout.go @@ -1,12 +1,13 @@ package r0 import ( + "errors" + "net/http" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" - "net/http" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -16,7 +17,7 @@ func Logout(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInf if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unable to logout") + return _responses.InternalServerError(errors.New("unable to logout")) } return _responses.EmptyResponse{} } @@ -26,7 +27,7 @@ func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unable to logout") + return _responses.InternalServerError(errors.New("unable to logout")) } return _responses.EmptyResponse{} } diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go index e6fe8f8a..a061f8d6 100644 --- a/api/r0/preview_url.go +++ b/api/r0/preview_url.go @@ -45,17 +45,17 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use ts, err = strconv.ParseInt(tsStr, 10, 64) if err != nil { rctx.Log.Error("Error parsing ts: ", err) - return _responses.BadRequest(err.Error()) + return _responses.BadRequest(err) } } // Validate the URL if urlStr == "" { - return _responses.BadRequest("No url provided") + return _responses.BadRequest(errors.New("No url provided")) } //goland:noinspection HttpUrlsUsage if strings.Index(urlStr, "http://") != 0 && strings.Index(urlStr, "https://") != 0 { - return _responses.BadRequest("Scheme not accepted") + return _responses.BadRequest(errors.New("Scheme not accepted")) } languageHeader := rctx.Config.UrlPreviews.DefaultLanguage @@ -79,12 +79,12 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { if errors.Is(err, common.ErrMediaNotFound) || errors.Is(err, common.ErrHostNotFound) { return _responses.NotFoundError() - } else if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) { - return _responses.BadRequest(err.Error()) - } else { - sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") } + if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) { + return _responses.BadRequest(err) + } + sentry.CaptureException(err) + return _responses.InternalServerError(errors.New("Unexpected Error")) } return &MatrixOpenGraph{ diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 50c88ec4..9c4994b5 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -28,14 +28,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta timeoutMs := r.URL.Query().Get("timeout_ms") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest("allow_remote flag does not appear to be a boolean") + return _responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -44,14 +44,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest("allow_redirect flag does not appear to be a boolean") + return _responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } blockFor, err := util.CalcBlockForDuration(timeoutMs) if err != nil { - return _responses.BadRequest("timeout_ms does not appear to be an integer") + return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -75,7 +75,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } if widthStr == "" || heightStr == "" { - return _responses.BadRequest("Width and height are required") + return _responses.BadRequest(errors.New("Width and height are required")) } width := 0 @@ -85,21 +85,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if widthStr != "" { parsedWidth, err := strconv.Atoi(widthStr) if err != nil { - return _responses.BadRequest("Width does not appear to be an integer") + return _responses.BadRequest(errors.New("Width does not appear to be an integer")) } width = parsedWidth } if heightStr != "" { parsedHeight, err := strconv.Atoi(heightStr) if err != nil { - return _responses.BadRequest("Height does not appear to be an integer") + return _responses.BadRequest(errors.New("Height does not appear to be an integer")) } height = parsedHeight } if animatedStr != "" { parsedFlag, err := strconv.ParseBool(animatedStr) if err != nil { - return _responses.BadRequest("Animated flag does not appear to be a boolean") + return _responses.BadRequest(errors.New("Animated flag does not appear to be a boolean")) } animated = parsedFlag } @@ -115,7 +115,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta }) if width <= 0 || height <= 0 { - return _responses.BadRequest("Width and height must be greater than zero") + return _responses.BadRequest(errors.New("Width and height must be greater than zero")) } thumbnail, stream, err := pipeline_thumbnail.Execute(rctx, server, mediaId, pipeline_thumbnail.ThumbnailOpts{ @@ -156,7 +156,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error("Unexpected error locating media record: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } else { return &_responses.DownloadResponse{ ContentType: record.ContentType, @@ -171,7 +171,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } return &_responses.DownloadResponse{ diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go index 30800652..24480609 100644 --- a/api/r0/upload_async.go +++ b/api/r0/upload_async.go @@ -70,10 +70,10 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apime } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ - //ContentUri: util.MxcUri(media.Origin, media.MediaId), // This endpoint doesn't return a URI + // ContentUri: util.MxcUri(media.Origin, media.MediaId), // This endpoint doesn't return a URI } } diff --git a/api/r0/upload_sync.go b/api/r0/upload_sync.go index 87f81beb..2af0732e 100644 --- a/api/r0/upload_sync.go +++ b/api/r0/upload_sync.go @@ -46,7 +46,7 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimet } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ diff --git a/api/router.go b/api/router.go index 2f84be5b..d9738013 100644 --- a/api/router.go +++ b/api/router.go @@ -29,25 +29,25 @@ func buildPrimaryRouter() *httprouter.Router { func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusMethodNotAllowed) - if b, err := json.Marshal(_responses.MethodNotAllowed()); err != nil { - panic(errors.New("error preparing MethodNotAllowed: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending MethodNotAllowed: " + err.Error())) - } + reponse, err := json.Marshal(_responses.MethodNotAllowed()) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %w", err)) + logrus.Errorf("error preparing MethodNotAllowed: %v", err) + return } + w.Write(reponse) } func notFoundFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) - if b, err := json.Marshal(_responses.NotFoundError()); err != nil { - panic(errors.New("error preparing NotFound: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending NotFound: " + err.Error())) - } + reponse, err := json.Marshal(_responses.NotFoundError()) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing NotFound: %w", err)) + logrus.Errorf("error preparing NotFound: %v", err) + return } + w.Write(reponse) } func finishCorsFn(w http.ResponseWriter, r *http.Request) { @@ -66,11 +66,12 @@ func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - if b, err := json.Marshal(_responses.InternalServerError("unexpected error")); err != nil { - panic(errors.New("error preparing InternalServerError: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending InternalServerError: " + err.Error())) - } + + reponse, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %w", err)) + logrus.Errorf("error preparing InternalServerError: %v", err) + return } + w.Write(reponse) } diff --git a/api/unstable/info.go b/api/unstable/info.go index eb245991..d42a5d03 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -54,14 +54,14 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User allowRemote := r.URL.Query().Get("allow_remote") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError("allow_remote flag does not appear to be a boolean") + return _responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -100,7 +100,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } response := &MediaInfoResponse{ @@ -137,7 +137,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if err != nil { rctx.Log.Error("Unexpected error locating media thumbnails: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } if len(thumbs) > 0 { diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go index f10c1adc..63b6cfda 100644 --- a/api/unstable/local_copy.go +++ b/api/unstable/local_copy.go @@ -28,14 +28,14 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User rctx.Log.Warn("This endpoint is deprecated. See https://github.com/t2bot/matrix-media-repo/issues/422") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + return _responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError("allow_remote flag does not appear to be a boolean") + return _responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -52,7 +52,7 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } if r.Host == server { - return _responses.BadRequest("Attempted to clone media to the same origin") + return _responses.BadRequest(errors.New("Attempted to clone media to the same origin")) } // TODO: There's a lot of room for improvement here. Instead of re-uploading media, we should just update the DB. @@ -80,7 +80,7 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } record, err = pipeline_upload.Execute(rctx, server, mediaId, stream, record.ContentType, record.UploadName, user.UserId, datastores.LocalMediaKind) @@ -91,7 +91,7 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return _responses.InternalServerError(errors.New("Unexpected Error")) } return &r0.MediaUploadedResponse{ diff --git a/api/v1/create.go b/api/v1/create.go index d1f40932..aecf7448 100644 --- a/api/v1/create.go +++ b/api/v1/create.go @@ -1,6 +1,7 @@ package v1 import ( + "errors" "net/http" "github.com/getsentry/sentry-go" @@ -21,7 +22,7 @@ func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Us if err != nil { rctx.Log.Error("Unexpected error creating media ID:", err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return _responses.InternalServerError(errors.New("unexpected error")) } return &MediaCreatedResponse{ diff --git a/database/db.go b/database/db.go index 5931a481..8b6f4d98 100644 --- a/database/db.go +++ b/database/db.go @@ -2,7 +2,7 @@ package database import ( "database/sql" - "errors" + "fmt" "sync" "github.com/DavidHuie/gomigrate" @@ -70,7 +70,7 @@ func openDatabase(connectionString string, maxConns int, maxIdleConns int) error var err error if d.conn, err = sql.Open("postgres", connectionString); err != nil { - return errors.New("error connecting to db: " + err.Error()) + return fmt.Errorf("error connecting to db: %w", err) } d.conn.SetMaxOpenConns(maxConns) d.conn.SetMaxIdleConns(maxIdleConns) @@ -78,51 +78,51 @@ func openDatabase(connectionString string, maxConns int, maxIdleConns int) error // Run migrations var migrator *gomigrate.Migrator if migrator, err = gomigrate.NewMigratorWithLogger(d.conn, gomigrate.Postgres{}, config.Runtime.MigrationsPath, &logging.SendToDebugLogger{}); err != nil { - return errors.New("error setting up migrator: " + err.Error()) + return fmt.Errorf("error setting up migrator: %w", err) } if err = migrator.Migrate(); err != nil { - return errors.New("error running migrations: " + err.Error()) + return fmt.Errorf("error running migrations: %w", err) } // Prepare the table accessors if d.Media, err = prepareMediaTables(d.conn); err != nil { - return errors.New("failed to create media table accessor: " + err.Error()) + return fmt.Errorf("failed to create media table accessor: %w", err) } if d.ExpiringMedia, err = prepareExpiringMediaTables(d.conn); err != nil { - return errors.New("failed to create expiring media table accessor: " + err.Error()) + return fmt.Errorf("failed to create expiring media table accessor: %w", err) } if d.UserStats, err = prepareUserStatsTables(d.conn); err != nil { - return errors.New("failed to create user stats table accessor: " + err.Error()) + return fmt.Errorf("failed to create user stats table accessor: %w", err) } if d.ReservedMedia, err = prepareReservedMediaTables(d.conn); err != nil { - return errors.New("failed to create reserved media table accessor: " + err.Error()) + return fmt.Errorf("failed to create reserved media table accessor: %w", err) } if d.MetadataView, err = prepareMetadataVirtualTables(d.conn); err != nil { - return errors.New("failed to create metadata virtual table accessor: " + err.Error()) + return fmt.Errorf("failed to create metadata virtual table accessor: %w", err) } if d.HeldMedia, err = prepareHeldMediaTables(d.conn); err != nil { - return errors.New("failed to create held media table accessor: " + err.Error()) + return fmt.Errorf("failed to create held media table accessor: %w", err) } if d.Thumbnails, err = prepareThumbnailsTables(d.conn); err != nil { - return errors.New("failed to create thumbnails table accessor: " + err.Error()) + return fmt.Errorf("failed to create thumbnails table accessor: %w", err) } if d.LastAccess, err = prepareLastAccessTables(d.conn); err != nil { - return errors.New("failed to create last access table accessor: " + err.Error()) + return fmt.Errorf("failed to create last access table accessor: %w", err) } if d.UrlPreviews, err = prepareUrlPreviewsTables(d.conn); err != nil { - return errors.New("failed to create url previews table accessor: " + err.Error()) + return fmt.Errorf("failed to create url previews table accessor: %w", err) } if d.MediaAttributes, err = prepareMediaAttributesTables(d.conn); err != nil { - return errors.New("failed to create media attributes table accessor: " + err.Error()) + return fmt.Errorf("failed to create media attributes table accessor: %w", err) } if d.Tasks, err = prepareTasksTables(d.conn); err != nil { - return errors.New("failed to create tasks table accessor: " + err.Error()) + return fmt.Errorf("failed to create tasks table accessor: %w", err) } if d.Exports, err = prepareExportsTables(d.conn); err != nil { - return errors.New("failed to create exports table accessor: " + err.Error()) + return fmt.Errorf("failed to create exports table accessor: %w", err) } if d.ExportParts, err = prepareExportPartsTables(d.conn); err != nil { - return errors.New("failed to create export parts table accessor: " + err.Error()) + return fmt.Errorf("failed to create export parts table accessor: %w", err) } instance = d diff --git a/database/table_expiring_media.go b/database/table_expiring_media.go index c33579af..0c51dc7a 100644 --- a/database/table_expiring_media.go +++ b/database/table_expiring_media.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -40,19 +41,19 @@ type expiringMediaTableWithContext struct { func prepareExpiringMediaTables(db *sql.DB) (*expiringMediaTableStatements, error) { var err error - var stmts = &expiringMediaTableStatements{} + stmts := &expiringMediaTableStatements{} if stmts.insertExpiringMedia, err = db.Prepare(insertExpiringMedia); err != nil { - return nil, errors.New("error preparing insertExpiringMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExpiringMedia: %w", err) } if stmts.selectExpiringMediaByUserCount, err = db.Prepare(selectExpiringMediaByUserCount); err != nil { - return nil, errors.New("error preparing selectExpiringMediaByUserCount: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExpiringMediaByUserCount: %w", err) } if stmts.selectExpiringMediaById, err = db.Prepare(selectExpiringMediaById); err != nil { - return nil, errors.New("error preparing selectExpiringMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExpiringMediaById: %w", err) } if stmts.deleteExpiringMediaById, err = db.Prepare(deleteExpiringMediaById); err != nil { - return nil, errors.New("error preparing deleteExpiringMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExpiringMediaById: %w", err) } return stmts, nil diff --git a/database/table_export_parts.go b/database/table_export_parts.go index 71d882ce..05e36971 100644 --- a/database/table_export_parts.go +++ b/database/table_export_parts.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -35,19 +36,19 @@ type exportPartsTableWithContext struct { func prepareExportPartsTables(db *sql.DB) (*exportPartsTableStatements, error) { var err error - var stmts = &exportPartsTableStatements{} + stmts := &exportPartsTableStatements{} if stmts.insertExportPart, err = db.Prepare(insertExportPart); err != nil { - return nil, errors.New("error preparing insertExportPart: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExportPart: %w", err) } if stmts.deleteExportPartsById, err = db.Prepare(deleteExportPartsById); err != nil { - return nil, errors.New("error preparing deleteExportPartsById: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExportPartsById: %w", err) } if stmts.selectExportPartsById, err = db.Prepare(selectExportPartsById); err != nil { - return nil, errors.New("error preparing selectExportPartsById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportPartsById: %w", err) } if stmts.selectExportPartById, err = db.Prepare(selectExportPartById); err != nil { - return nil, errors.New("error preparing selectExportPartById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportPartById: %w", err) } return stmts, nil diff --git a/database/table_exports.go b/database/table_exports.go index 6495475a..25f06054 100644 --- a/database/table_exports.go +++ b/database/table_exports.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -24,16 +25,16 @@ type exportsTableWithContext struct { func prepareExportsTables(db *sql.DB) (*exportsTableStatements, error) { var err error - var stmts = &exportsTableStatements{} + stmts := &exportsTableStatements{} if stmts.insertExport, err = db.Prepare(insertExport); err != nil { - return nil, errors.New("error preparing insertExport: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExport: %w", err) } if stmts.selectExportEntity, err = db.Prepare(selectExportEntity); err != nil { - return nil, errors.New("error preparing selectExportEntity: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportEntity: %w", err) } if stmts.deleteExport, err = db.Prepare(deleteExport); err != nil { - return nil, errors.New("error preparing deleteExport: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExport: %w", err) } return stmts, nil diff --git a/database/table_last_access.go b/database/table_last_access.go index eca9826a..e38cc975 100644 --- a/database/table_last_access.go +++ b/database/table_last_access.go @@ -2,7 +2,7 @@ package database import ( "database/sql" - "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -25,10 +25,10 @@ type lastAccessTableWithContext struct { func prepareLastAccessTables(db *sql.DB) (*lastAccessTableStatements, error) { var err error - var stmts = &lastAccessTableStatements{} + stmts := &lastAccessTableStatements{} if stmts.upsertLastAccess, err = db.Prepare(upsertLastAccess); err != nil { - return nil, errors.New("error preparing upsertLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing upsertLastAccess: %w", err) } return stmts, nil diff --git a/database/table_media.go b/database/table_media.go index e54f9368..2d21a16d 100644 --- a/database/table_media.go +++ b/database/table_media.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/lib/pq" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -80,67 +81,67 @@ type MediaTableWithContext struct { func prepareMediaTables(db *sql.DB) (*mediaTableStatements, error) { var err error - var stmts = &mediaTableStatements{} + stmts := &mediaTableStatements{} if stmts.selectDistinctMediaDatastoreIds, err = db.Prepare(selectDistinctMediaDatastoreIds); err != nil { - return nil, errors.New("error preparing selectDistinctMediaDatastoreIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectDistinctMediaDatastoreIds: %w", err) } if stmts.selectMediaIsQuarantinedByHash, err = db.Prepare(selectMediaIsQuarantinedByHash); err != nil { - return nil, errors.New("error preparing selectMediaIsQuarantinedByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaIsQuarantinedByHash: %w", err) } if stmts.selectMediaByHash, err = db.Prepare(selectMediaByHash); err != nil { - return nil, errors.New("error preparing selectMediaByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByHash: %w", err) } if stmts.insertMedia, err = db.Prepare(insertMedia); err != nil { - return nil, errors.New("error preparing insertMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertMedia: %w", err) } if stmts.selectMediaExists, err = db.Prepare(selectMediaExists); err != nil { - return nil, errors.New("error preparing selectMediaExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaExists: %w", err) } if stmts.selectMediaById, err = db.Prepare(selectMediaById); err != nil { - return nil, errors.New("error preparing selectMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaById: %w", err) } if stmts.selectMediaByUserId, err = db.Prepare(selectMediaByUserId); err != nil { - return nil, errors.New("error preparing selectMediaByUserId: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByUserId: %w", err) } if stmts.selectOldMediaByUserId, err = db.Prepare(selectOldMediaByUserId); err != nil { - return nil, errors.New("error preparing selectOldMediaByUserId: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaByUserId: %w", err) } if stmts.selectMediaByOrigin, err = db.Prepare(selectMediaByOrigin); err != nil { - return nil, errors.New("error preparing selectMediaByOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOrigin: %w", err) } if stmts.selectOldMediaByOrigin, err = db.Prepare(selectOldMediaByOrigin); err != nil { - return nil, errors.New("error preparing selectOldMediaByOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaByOrigin: %w", err) } if stmts.selectMediaByLocationExists, err = db.Prepare(selectMediaByLocationExists); err != nil { - return nil, errors.New("error preparing selectMediaByLocationExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByLocationExists: %w", err) } if stmts.selectMediaByUserCount, err = db.Prepare(selectMediaByUserCount); err != nil { - return nil, errors.New("error preparing selectMediaByUserCount: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByUserCount: %w", err) } if stmts.selectMediaByOriginAndUserIds, err = db.Prepare(selectMediaByOriginAndUserIds); err != nil { - return nil, errors.New("error preparing selectMediaByOriginAndUserIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOriginAndUserIds: %w", err) } if stmts.selectMediaByOriginAndIds, err = db.Prepare(selectMediaByOriginAndIds); err != nil { - return nil, errors.New("error preparing selectMediaByOriginAndIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOriginAndIds: %w", err) } if stmts.selectOldMediaExcludingDomains, err = db.Prepare(selectOldMediaExcludingDomains); err != nil { - return nil, errors.New("error preparing selectOldMediaExcludingDomains: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaExcludingDomains: %w", err) } if stmts.deleteMedia, err = db.Prepare(deleteMedia); err != nil { - return nil, errors.New("error preparing deleteMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteMedia: %w", err) } if stmts.updateMediaLocation, err = db.Prepare(updateMediaLocation); err != nil { - return nil, errors.New("error preparing updateMediaLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing updateMediaLocation: %w", err) } if stmts.selectMediaByLocation, err = db.Prepare(selectMediaByLocation); err != nil { - return nil, errors.New("error preparing selectMediaByLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByLocation: %w", err) } if stmts.selectMediaByQuarantine, err = db.Prepare(selectMediaByQuarantine); err != nil { - return nil, errors.New("error preparing selectMediaByQuarantine: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByQuarantine: %w", err) } if stmts.selectMediaByQuarantineAndOrigin, err = db.Prepare(selectMediaByQuarantineAndOrigin); err != nil { - return nil, errors.New("error preparing selectMediaByQuarantineAndOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByQuarantineAndOrigin: %w", err) } return stmts, nil diff --git a/database/table_media_attributes.go b/database/table_media_attributes.go index 7532ce73..d5838a0c 100644 --- a/database/table_media_attributes.go +++ b/database/table_media_attributes.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -39,13 +40,13 @@ type mediaAttributesTableWithContext struct { func prepareMediaAttributesTables(db *sql.DB) (*mediaAttributesTableStatements, error) { var err error - var stmts = &mediaAttributesTableStatements{} + stmts := &mediaAttributesTableStatements{} if stmts.selectMediaAttributes, err = db.Prepare(selectMediaAttributes); err != nil { - return nil, errors.New("error preparing selectMediaAttributes: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaAttributes: %w", err) } if stmts.upsertMediaPurpose, err = db.Prepare(upsertMediaPurpose); err != nil { - return nil, errors.New("error preparing upsertMediaPurpose: " + err.Error()) + return nil, fmt.Errorf("error preparing upsertMediaPurpose: %w", err) } return stmts, nil diff --git a/database/table_media_hold.go b/database/table_media_hold.go index 05bb3a5c..8af566bb 100644 --- a/database/table_media_hold.go +++ b/database/table_media_hold.go @@ -2,7 +2,7 @@ package database import ( "database/sql" - "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -35,13 +35,13 @@ type heldMediaTableWithContext struct { func prepareHeldMediaTables(db *sql.DB) (*heldMediaTableStatements, error) { var err error - var stmts = &heldMediaTableStatements{} + stmts := &heldMediaTableStatements{} if stmts.insertHeldMedia, err = db.Prepare(insertHeldMedia); err != nil { - return nil, errors.New("error preparing insertHeldMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertHeldMedia: %w", err) } if stmts.deleteHeldMedia, err = db.Prepare(deleteHeldMedia); err != nil { - return nil, errors.New("error preparing deleteHeldMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteHeldMedia: %w", err) } return stmts, nil diff --git a/database/table_reserved_media.go b/database/table_reserved_media.go index 776d0cf2..4c351cd4 100644 --- a/database/table_reserved_media.go +++ b/database/table_reserved_media.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -28,13 +29,13 @@ type reservedMediaTableWithContext struct { func prepareReservedMediaTables(db *sql.DB) (*reservedMediaTableStatements, error) { var err error - var stmts = &reservedMediaTableStatements{} + stmts := &reservedMediaTableStatements{} if stmts.insertReservedMediaNoConflict, err = db.Prepare(insertReservedMediaNoConflict); err != nil { - return nil, errors.New("error preparing insertReservedMediaNoConflict: " + err.Error()) + return nil, fmt.Errorf("error preparing insertReservedMediaNoConflict: %w", err) } if stmts.selectReservedMediaExists, err = db.Prepare(selectReservedMediaExists); err != nil { - return nil, errors.New("error preparing selectReservedMediaExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectReservedMediaExists: %w", err) } return stmts, nil diff --git a/database/table_tasks.go b/database/table_tasks.go index 8b2b5f67..742b5472 100644 --- a/database/table_tasks.go +++ b/database/table_tasks.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -39,25 +40,25 @@ type tasksTableWithContext struct { func prepareTasksTables(db *sql.DB) (*tasksTableStatements, error) { var err error - var stmts = &tasksTableStatements{} + stmts := &tasksTableStatements{} if stmts.selectTask, err = db.Prepare(selectTask); err != nil { - return nil, errors.New("error preparing selectTask: " + err.Error()) + return nil, fmt.Errorf("error preparing selectTask: %w", err) } if stmts.insertTask, err = db.Prepare(insertTask); err != nil { - return nil, errors.New("error preparing insertTask: " + err.Error()) + return nil, fmt.Errorf("error preparing insertTask: %w", err) } if stmts.selectAllTasks, err = db.Prepare(selectAllTasks); err != nil { - return nil, errors.New("error preparing selectAllTasks: " + err.Error()) + return nil, fmt.Errorf("error preparing selectAllTasks: %w", err) } if stmts.selectIncompleteTasks, err = db.Prepare(selectIncompleteTasks); err != nil { - return nil, errors.New("error preparing selectIncompleteTasks: " + err.Error()) + return nil, fmt.Errorf("error preparing selectIncompleteTasks: %w", err) } if stmts.updateTaskEndTime, err = db.Prepare(updateTaskEndTime); err != nil { - return nil, errors.New("error preparing updateTaskEndTime: " + err.Error()) + return nil, fmt.Errorf("error preparing updateTaskEndTime: %w", err) } if stmts.updateTaskError, err = db.Prepare(updateTaskError); err != nil { - return nil, errors.New("error preparing updateTaskError: " + err.Error()) + return nil, fmt.Errorf("error preparing updateTaskError: %w", err) } return stmts, nil diff --git a/database/table_thumbnails.go b/database/table_thumbnails.go index 5358103b..c0c936a7 100644 --- a/database/table_thumbnails.go +++ b/database/table_thumbnails.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -52,31 +53,31 @@ type thumbnailsTableWithContext struct { func prepareThumbnailsTables(db *sql.DB) (*thumbnailsTableStatements, error) { var err error - var stmts = &thumbnailsTableStatements{} + stmts := &thumbnailsTableStatements{} if stmts.selectThumbnailByParams, err = db.Prepare(selectThumbnailByParams); err != nil { - return nil, errors.New("error preparing selectThumbnailByParams: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailByParams: %w", err) } if stmts.insertThumbnail, err = db.Prepare(insertThumbnail); err != nil { - return nil, errors.New("error preparing insertThumbnail: " + err.Error()) + return nil, fmt.Errorf("error preparing insertThumbnail: %w", err) } if stmts.selectThumbnailByLocationExists, err = db.Prepare(selectThumbnailByLocationExists); err != nil { - return nil, errors.New("error preparing selectThumbnailByLocationExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailByLocationExists: %w", err) } if stmts.selectThumbnailsForMedia, err = db.Prepare(selectThumbnailsForMedia); err != nil { - return nil, errors.New("error preparing selectThumbnailsForMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsForMedia: %w", err) } if stmts.selectOldThumbnails, err = db.Prepare(selectOldThumbnails); err != nil { - return nil, errors.New("error preparing selectOldThumbnails: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldThumbnails: %w", err) } if stmts.deleteThumbnail, err = db.Prepare(deleteThumbnail); err != nil { - return nil, errors.New("error preparing deleteThumbnail: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteThumbnail: %w", err) } if stmts.updateThumbnailLocation, err = db.Prepare(updateThumbnailLocation); err != nil { - return nil, errors.New("error preparing updateThumbnailLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing updateThumbnailLocation: %w", err) } if stmts.selectThumbnailsByLocation, err = db.Prepare(selectThumbnailsByLocation); err != nil { - return nil, errors.New("error preparing selectThumbnailsByLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsByLocation: %w", err) } return stmts, nil diff --git a/database/table_url_previews.go b/database/table_url_previews.go index 55c6bde5..0ad62b58 100644 --- a/database/table_url_previews.go +++ b/database/table_url_previews.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -42,16 +43,16 @@ type urlPreviewsTableWithContext struct { func prepareUrlPreviewsTables(db *sql.DB) (*urlPreviewsTableStatements, error) { var err error - var stmts = &urlPreviewsTableStatements{} + stmts := &urlPreviewsTableStatements{} if stmts.selectUrlPreview, err = db.Prepare(selectUrlPreview); err != nil { - return nil, errors.New("error preparing selectUrlPreview: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUrlPreview: %w", err) } if stmts.insertUrlPreview, err = db.Prepare(insertUrlPreview); err != nil { - return nil, errors.New("error preparing insertUrlPreview: " + err.Error()) + return nil, fmt.Errorf("error preparing insertUrlPreview: %w", err) } if stmts.deleteOldUrlPreviews, err = db.Prepare(deleteOldUrlPreviews); err != nil { - return nil, errors.New("error preparing deleteOldUrlPreviews: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteOldUrlPreviews: %w", err) } return stmts, nil diff --git a/database/table_user_stats.go b/database/table_user_stats.go index ad36acd7..be1084e8 100644 --- a/database/table_user_stats.go +++ b/database/table_user_stats.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -25,10 +26,10 @@ type userStatsTableWithContext struct { func prepareUserStatsTables(db *sql.DB) (*userStatsTableStatements, error) { var err error - var stmts = &userStatsTableStatements{} + stmts := &userStatsTableStatements{} if stmts.selectUserStatsUploadedBytes, err = db.Prepare(selectUserStatsUploadedBytes); err != nil { - return nil, errors.New("error preparing selectUserStatsUploadedBytes: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUserStatsUploadedBytes: %w", err) } return stmts, nil diff --git a/database/virtualtable_metadata.go b/database/virtualtable_metadata.go index f99d256f..71b4846a 100644 --- a/database/virtualtable_metadata.go +++ b/database/virtualtable_metadata.go @@ -64,30 +64,30 @@ type metadataVirtualTableWithContext struct { func prepareMetadataVirtualTables(db *sql.DB) (*metadataVirtualTableStatements, error) { var err error - var stmts = &metadataVirtualTableStatements{ + stmts := &metadataVirtualTableStatements{ db: db, } if stmts.selectEstimatedDatastoreSize, err = db.Prepare(selectEstimatedDatastoreSize); err != nil { - return nil, errors.New("error preparing selectEstimatedDatastoreSize: " + err.Error()) + return nil, fmt.Errorf("error preparing selectEstimatedDatastoreSize: %w", err) } if stmts.selectUploadSizesForServer, err = db.Prepare(selectUploadSizesForServer); err != nil { - return nil, errors.New("error preparing selectUploadSizesForServer: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUploadSizesForServer: %w", err) } if stmts.selectUploadCountsForServer, err = db.Prepare(selectUploadCountsForServer); err != nil { - return nil, errors.New("error preparing selectUploadCountsForServer: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUploadCountsForServer: %w", err) } if stmts.selectMediaForDatastoreWithLastAccess, err = db.Prepare(selectMediaForDatastoreWithLastAccess); err != nil { - return nil, errors.New("error preparing selectMediaForDatastoreWithLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaForDatastoreWithLastAccess: %w", err) } if stmts.selectThumbnailsForDatastoreWithLastAccess, err = db.Prepare(selectThumbnailsForDatastoreWithLastAccess); err != nil { - return nil, errors.New("error preparing selectThumbnailsForDatastoreWithLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsForDatastoreWithLastAccess: %w", err) } if stmts.updateQuarantineByHash, err = db.Prepare(updateQuarantineByHash); err != nil { - return nil, errors.New("error preparing updateQuarantineByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing updateQuarantineByHash: %w", err) } if stmts.updateQuarantineByHashAndOrigin, err = db.Prepare(updateQuarantineByHashAndOrigin); err != nil { - return nil, errors.New("error preparing updateQuarantineByHashAndOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing updateQuarantineByHashAndOrigin: %w", err) } return stmts, nil diff --git a/datastores/buffer.go b/datastores/buffer.go index 4a0d71dd..37b629cf 100644 --- a/datastores/buffer.go +++ b/datastores/buffer.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "os" @@ -21,7 +22,7 @@ func BufferTemp(datastore config.DatastoreConfig, contents io.ReadCloser) (strin } else if datastore.Type == "file" { fpath, err = os.MkdirTemp(os.TempDir(), "mmr") if err != nil { - return "", 0, nil, errors.New("error generating temporary directory: " + err.Error()) + return "", 0, nil, fmt.Errorf("error generating temporary directory: %w", err) } } else { return "", 0, nil, errors.New("unknown datastore type - contact developer") @@ -32,14 +33,14 @@ func BufferTemp(datastore config.DatastoreConfig, contents io.ReadCloser) (strin logrus.Warnf("Datastore %s does not have a valid temporary path configured. This will lead to increased memory usage.", datastore.Id) target = &bytes.Buffer{} } else { - err = os.Mkdir(fpath, os.ModeDir|0700) + err = os.Mkdir(fpath, os.ModeDir|0o700) if err != nil && !os.IsExist(err) { - return "", 0, nil, errors.New("error creating temp path: " + err.Error()) + return "", 0, nil, fmt.Errorf("error creating temp path: %w", err) } var file *os.File file, err = os.CreateTemp(fpath, "mmr") if err != nil { - return "", 0, nil, errors.New("error generating temporary file: " + err.Error()) + return "", 0, nil, fmt.Errorf("error generating temporary file: %w", err) } target = file } diff --git a/matrix/server_discovery.go b/matrix/server_discovery.go index 02e5ce37..e8a0b808 100644 --- a/matrix/server_discovery.go +++ b/matrix/server_discovery.go @@ -2,6 +2,7 @@ package matrix import ( "encoding/json" + "errors" "fmt" "net" "net/http" @@ -15,8 +16,10 @@ import ( "github.com/sirupsen/logrus" ) -var apiUrlCacheInstance *cache.Cache -var apiUrlSingletonLock = &sync.Once{} +var ( + apiUrlCacheInstance *cache.Cache + apiUrlSingletonLock = &sync.Once{} +) type cachedServer struct { url string @@ -45,14 +48,17 @@ func GetServerApiUrl(hostname string) (string, string, error) { return server.url, server.hostname, nil } + addrErr := &net.AddrError{} h, p, err := net.SplitHostPort(hostname) defPort := false - if err != nil && strings.HasSuffix(err.Error(), "missing port in address") { + switch { + case errors.As(err, &addrErr) && addrErr.Err == "missing port in address": h, p, err = net.SplitHostPort(hostname + ":8448") - defPort = true - } - if err != nil { - return "", "", err + if err != nil { + return "", "", fmt.Errorf("failed to parse hostname: %w", err) + } + case err != nil: + return "", "", fmt.Errorf("failed to parse hostname: %w", err) } // Step 1 of the discovery process: if the hostname is an IP, use that with explicit or default port @@ -71,7 +77,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) server := cachedServer{url, h} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (explicit port)") + logrus.Debugf("Server API URL for %s is %s (explicit port)", hostname, url) return url, h, nil } @@ -101,7 +107,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(wkHost, wkPort)) server := cachedServer{url, wk.ServerAddr} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; IP address)") + logrus.Debugf("Server API URL for %s is %s (WK; IP address)", hostname, url) return url, wk.ServerAddr, nil } @@ -112,7 +118,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", wkHost) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; explicit port)") + logrus.Debugf("Server API URL for %s is %s (WK; explicit port)", hostname, url) return url, wkHost, nil } @@ -129,7 +135,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; SRV)") + logrus.Debugf("Server API URL for %s is %s (WK; SRV)", hostname, url) return url, wkHost, nil } @@ -147,7 +153,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; SRV-Deprecated)") + logrus.Debugf("Server API URL for %s is %s (WK; SRV-Deprecated)", hostname, url) return url, wkHost, nil } @@ -156,7 +162,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(wkHost, wkPort)) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; fallback)") + logrus.Debugf("Server API URL for %s is %s (WK; fallback)", hostname, url) return url, wkHost, nil } } @@ -179,7 +185,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, h} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (SRV)") + logrus.Debugf("Server API URL for %s is %s (SRV)", hostname, url) return url, h, nil } @@ -196,7 +202,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, h} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (SRV-Deprecated)") + logrus.Debugf("Server API URL for %s is %s (SRV-Deprecated)", hostname, url) return url, h, nil } @@ -205,6 +211,6 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) server := cachedServer{url, h} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (fallback)") + logrus.Debugf("Server API URL for %s is %s (fallback)", hostname, url) return url, h, nil } diff --git a/pipelines/_steps/meta/flag_access.go b/pipelines/_steps/meta/flag_access.go index 18f9ed31..465e0913 100644 --- a/pipelines/_steps/meta/flag_access.go +++ b/pipelines/_steps/meta/flag_access.go @@ -13,7 +13,7 @@ func FlagAccess(ctx rcontext.RequestContext, sha256hash string, uploadTime int64 metrics.MediaAgeAccessed.Observe(float64(util.NowMillis()-uploadTime) / 1000.0) } if err := database.GetInstance().LastAccess.Prepare(ctx).Upsert(sha256hash, util.NowMillis()); err != nil { - ctx.Log.Warnf("Non-fatal error while updating last access for '%s': %s", sha256hash, err.Error()) + ctx.Log.Warnf("Non-fatal error while updating last access for '%s': %v", sha256hash, err) sentry.CaptureException(err) } } diff --git a/pipelines/_steps/upload/lock.go b/pipelines/_steps/upload/lock.go index 6a499b0d..89721ad0 100644 --- a/pipelines/_steps/upload/lock.go +++ b/pipelines/_steps/upload/lock.go @@ -3,6 +3,7 @@ package upload import ( "context" "errors" + "fmt" "time" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -22,7 +23,7 @@ func LockForUpload(ctx rcontext.RequestContext, hash string) (func() error, erro } if err := mutex.LockContext(ctx.Context); err != nil { if time.Now().After(attemptDoneAt) { - return nil, errors.New("failed to acquire upload lock: " + err.Error()) + return nil, fmt.Errorf("failed to acquire upload lock: %w", err) } else { ctx.Log.Warn("failed to acquire upload lock: ", err) } diff --git a/plugins/manager.go b/plugins/manager.go index 7ae7a68e..3df22602 100644 --- a/plugins/manager.go +++ b/plugins/manager.go @@ -21,7 +21,7 @@ func ReloadPlugins() { logrus.Info("Loading plugin: ", pl.Executable) mmr, err := newPlugin(pl.Executable, pl.Config) if err != nil { - logrus.Errorf("failed to load plugin %s: %s", pl.Executable, err.Error()) + logrus.Errorf("failed to load plugin %s: %v", pl.Executable, err) continue } @@ -46,7 +46,7 @@ func CheckForSpam(r io.Reader, filename string, contentType string, userId strin for _, pl := range existingPlugins { as, err := pl.Antispam() if err != nil { - logrus.Warnf("error loading antispam plugin: %s", err.Error()) + logrus.Warnf("error loading antispam plugin: %v", err) continue } diff --git a/tasks/exec.go b/tasks/exec.go index 31ac14c2..55f42bc7 100644 --- a/tasks/exec.go +++ b/tasks/exec.go @@ -82,7 +82,7 @@ func beginTask(task *database.DbTask) { sentry.CaptureMessage(m) } }); err != nil { - m := fmt.Sprintf("Error trying to schedule task %s (ID: %d): %s", task.Name, task.TaskId, err.Error()) + m := fmt.Sprintf("Error trying to schedule task %s (ID: %d): %v", task.Name, task.TaskId, err) runnerCtx.Log.Warn(m) sentry.CaptureMessage(m) time.AfterFunc(15*time.Second, func() { diff --git a/test/test_internals/deps.go b/test/test_internals/deps.go index 18ba27dd..61b54290 100644 --- a/test/test_internals/deps.go +++ b/test/test_internals/deps.go @@ -167,15 +167,15 @@ func (c *ContainerDeps) Teardown() { hs.Teardown() } if err := c.redisContainer.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down redis container: %s", err.Error()) + log.Fatalf("Error shutting down redis container: %v", err) } if err := c.pgContainer.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down mmr-postgres container: %s", err.Error()) + log.Fatalf("Error shutting down mmr-postgres container: %v", err) } c.minioDep.Teardown() c.depNet.Teardown() if err := os.Remove(c.mmrExtConfigPath); err != nil && !os.IsNotExist(err) { - log.Fatalf("Error cleaning up MMR-External config file '%s': %s", c.mmrExtConfigPath, err.Error()) + log.Fatalf("Error cleaning up MMR-External config file '%s': %v", c.mmrExtConfigPath, err) } } diff --git a/test/test_internals/deps_minio.go b/test/test_internals/deps_minio.go index 07369e70..1601cc52 100644 --- a/test/test_internals/deps_minio.go +++ b/test/test_internals/deps_minio.go @@ -125,6 +125,6 @@ func MakeMinio(depNet *NetworkDep) (*MinioDep, error) { func (c *MinioDep) Teardown() { if err := c.container.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down minio: %s", err.Error()) + log.Fatalf("Error shutting down minio: %v", err) } } diff --git a/test/test_internals/deps_mmr.go b/test/test_internals/deps_mmr.go index a3d3b9af..1ea7354e 100644 --- a/test/test_internals/deps_mmr.go +++ b/test/test_internals/deps_mmr.go @@ -137,10 +137,10 @@ func makeMmrInstances(ctx context.Context, count int, depNet *NetworkDep, tmplAr func (c *mmrContainer) Teardown() { if err := c.container.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down MMR machine %d: %s", c.MachineId, err.Error()) + log.Fatalf("Error shutting down MMR machine %d: %v", c.MachineId, err) } if err := os.Remove(c.tmpConfigPath); err != nil && !os.IsNotExist(err) { - log.Fatalf("Error cleaning up MMR config file '%s': %s", c.tmpConfigPath, err.Error()) + log.Fatalf("Error cleaning up MMR config file '%s': %v", c.tmpConfigPath, err) } } diff --git a/test/test_internals/deps_network.go b/test/test_internals/deps_network.go index 3887cfa1..7daba444 100644 --- a/test/test_internals/deps_network.go +++ b/test/test_internals/deps_network.go @@ -57,6 +57,6 @@ func (n *NetworkDep) ApplyToContainer() testcontainers.ContainerCustomizer { func (n *NetworkDep) Teardown() { if err := n.dockerNet.Remove(n.ctx); err != nil { - log.Fatalf("Error cleaning up docker network '%s': %s", n.NetId, err.Error()) + log.Fatalf("Error cleaning up docker network '%s': %v", n.NetId, err) } } diff --git a/thumbnailing/i/apng.go b/thumbnailing/i/apng.go index 6bc0f730..e27d4dd2 100644 --- a/thumbnailing/i/apng.go +++ b/thumbnailing/i/apng.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "image" "image/draw" "io" @@ -43,7 +43,7 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width p, err := apng.DecodeAll(b) if err != nil { - return nil, errors.New("apng: error decoding image: " + err.Error()) + return nil, fmt.Errorf("apng: error decoding image: %w", err) } // prepare a blank frame to use as swap space @@ -72,7 +72,7 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width // Do the thumbnailing on the copied frame frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) if err != nil { - return nil, errors.New("apng: error generating thumbnail frame: " + err.Error()) + return nil, fmt.Errorf("apng: error generating thumbnail frame: %w", err) } if frameThumb == nil { tmpImg := image.NewRGBA(frameImg.Bounds()) @@ -94,7 +94,7 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width go func(pw *io.PipeWriter, p apng.APNG) { err = apng.Encode(pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("apng: error encoding final thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("apng: error encoding final thumbnail: %w", err)) } else { _ = pw.Close() } diff --git a/thumbnailing/i/bmp.go b/thumbnailing/i/bmp.go index eb45514c..83d8794e 100644 --- a/thumbnailing/i/bmp.go +++ b/thumbnailing/i/bmp.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "slices" @@ -35,7 +35,7 @@ func (d bmpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d bmpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { src, err := bmp.Decode(b) if err != nil { - return nil, errors.New("bmp: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("bmp: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/i/flac.go b/thumbnailing/i/flac.go index 56a92dc3..8177c464 100644 --- a/thumbnailing/i/flac.go +++ b/thumbnailing/i/flac.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "github.com/faiface/beep" @@ -11,8 +11,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type flacGenerator struct { -} +type flacGenerator struct{} func (d flacGenerator) supportedContentTypes() []string { return []string{"audio/flac"} @@ -29,7 +28,7 @@ func (d flacGenerator) matches(img io.Reader, contentType string) bool { func (d flacGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := flac.Decode(b) if err != nil { - return audio, format, errors.New("flac: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("flac: error decoding audio: %w", err) } return audio, format, nil } @@ -41,7 +40,7 @@ func (d flacGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { tags, rc, err := u.GetID3Tags(r) if err != nil { - return nil, errors.New("flac: error getting tags: " + err.Error()) + return nil, fmt.Errorf("flac: error getting tags: %w", err) } //goland:noinspection GoUnhandledErrorResult defer rc.Close() diff --git a/thumbnailing/i/gif.go b/thumbnailing/i/gif.go index 676d0cba..a36419f5 100644 --- a/thumbnailing/i/gif.go +++ b/thumbnailing/i/gif.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "image" "image/draw" "image/gif" @@ -35,7 +35,7 @@ func (d gifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { g, err := gif.DecodeAll(b) if err != nil { - return nil, errors.New("gif: error decoding image: " + err.Error()) + return nil, fmt.Errorf("gif: error decoding image: %w", err) } // Prepare a blank frame to use as swap space @@ -58,7 +58,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i // Do the thumbnailing on the copied frame frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) if err != nil { - return nil, errors.New("gif: error generating thumbnail frame: " + err.Error()) + return nil, fmt.Errorf("gif: error generating thumbnail frame: %w", err) } if frameThumb == nil { tmpImg := image.NewRGBA(frameImg.Bounds()) @@ -80,7 +80,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i go func(pw *io.PipeWriter, p *image.Paletted) { err = u.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("gif: error encoding still frame thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("gif: error encoding still frame thumbnail: %w", err)) } else { _ = pw.Close() } @@ -114,7 +114,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i go func(pw *io.PipeWriter, g *gif.GIF) { err = gif.EncodeAll(pw, g) if err != nil { - _ = pw.CloseWithError(errors.New("gif: error encoding final thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("gif: error encoding final thumbnail: %w", err)) } else { _ = pw.Close() } diff --git a/thumbnailing/i/heif.go b/thumbnailing/i/heif.go index c6f2fb9a..b57d4701 100644 --- a/thumbnailing/i/heif.go +++ b/thumbnailing/i/heif.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "image" "io" "slices" @@ -11,8 +11,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) -type heifGenerator struct { -} +type heifGenerator struct{} func (d heifGenerator) supportedContentTypes() []string { return []string{"image/heif", "image/heic"} @@ -37,7 +36,7 @@ func (d heifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx func (d heifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { src, _, err := image.Decode(b) if err != nil { - return nil, errors.New("heif: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("heif: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/i/jpegxl.go b/thumbnailing/i/jpegxl.go index 72f145f4..0ec30c2f 100644 --- a/thumbnailing/i/jpegxl.go +++ b/thumbnailing/i/jpegxl.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "os" "os/exec" @@ -33,7 +33,7 @@ func (d jpegxlGenerator) GetOriginDimensions(b io.Reader, contentType string, ct func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-jpegxl") if err != nil { - return nil, errors.New("jpegxl: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.jpegxl") @@ -43,22 +43,22 @@ func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, widt defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("jpegxl: error creating temp jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error creating temp jpegxl file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("jpegxl: error writing temp jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error writing temp jpegxl file: %w", err) } err = exec.Command("convert", tempFile1, tempFile2).Run() if err != nil { - return nil, errors.New("jpegxl: error converting jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error converting jpegxl file: %w", err) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("jpegxl: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/jpg.go b/thumbnailing/i/jpg.go index de20b188..05253e4a 100644 --- a/thumbnailing/i/jpg.go +++ b/thumbnailing/i/jpg.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "image" _ "image/jpeg" "io" @@ -39,12 +39,12 @@ func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i src, err := imaging.Decode(b) if err != nil { - return nil, errors.New("jpg: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("jpg: error decoding thumbnail: %w", err) } thumb, err := u.MakeThumbnail(src, method, width, height) if err != nil { - return nil, errors.New("jpg: error making thumbnail: " + err.Error()) + return nil, fmt.Errorf("jpg: error making thumbnail: %w", err) } thumb = u.ApplyOrientation(thumb, orientation) @@ -53,7 +53,7 @@ func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i go func(pw *io.PipeWriter, p image.Image) { err = u.Encode(ctx, pw, p, u.JpegSource) if err != nil { - _ = pw.CloseWithError(errors.New("jpg: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("jpg: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } diff --git a/thumbnailing/i/mp3.go b/thumbnailing/i/mp3.go index 19641313..65b13580 100644 --- a/thumbnailing/i/mp3.go +++ b/thumbnailing/i/mp3.go @@ -2,7 +2,7 @@ package i import ( "bytes" - "errors" + "fmt" "image" "image/color" "image/draw" @@ -40,7 +40,7 @@ func (d mp3Generator) matches(img io.Reader, contentType string) bool { func (d mp3Generator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := mp3.Decode(readers.MakeCloser(b)) if err != nil { - return audio, format, errors.New("mp3: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("mp3: error decoding audio: %w", err) } return audio, format, nil } @@ -52,7 +52,7 @@ func (d mp3Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { - return nil, errors.New("mp3: error getting tags: " + err.Error()) + return nil, fmt.Errorf("mp3: error getting tags: %w", err) } //goland:noinspection GoUnhandledErrorResult defer rc.Close() @@ -124,7 +124,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee r := image.Rect(dx, dy, ddx, ddy) if artworkImg == nil { - f, _ := os.OpenFile(path.Join(config.Runtime.AssetsPath, "default-artwork.png"), os.O_RDONLY, 0640) + f, _ := os.OpenFile(path.Join(config.Runtime.AssetsPath, "default-artwork.png"), os.O_RDONLY, 0o640) if f != nil { defer f.Close() tmp, _, _ := image.Decode(f) @@ -149,7 +149,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee waveformX := padding + r.Max.X info, err := d.GetDataFromStream(audio, format, (int)(math.Max((float64)(width-waveformX-padding), 1))) if err != nil { - return nil, errors.New("beep-visual: error sampling audio: " + err.Error()) + return nil, fmt.Errorf("beep-visual: error sampling audio: %w", err) } // Average out all the samples @@ -200,7 +200,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee go func(pw *io.PipeWriter, p image.Image) { err = u.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("beep-visual: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("beep-visual: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } diff --git a/thumbnailing/i/mp4.go b/thumbnailing/i/mp4.go index cd59d66e..b0617037 100644 --- a/thumbnailing/i/mp4.go +++ b/thumbnailing/i/mp4.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "os" "os/exec" @@ -33,7 +33,7 @@ func (d mp4Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-mp4") if err != nil { - return nil, errors.New("mp4: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("mp4: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.mp4") @@ -45,20 +45,20 @@ func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width i f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("mp4: error creating temp video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error creating temp video file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("mp4: error writing temp video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error writing temp video file: %w", err) } err = exec.Command("ffmpeg", "-i", tempFile1, "-vf", "select=eq(n\\,0)", tempFile2).Run() if err != nil { - return nil, errors.New("mp4: error converting video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error converting video file: %w", err) } f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("mp4: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("mp4: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/ogg.go b/thumbnailing/i/ogg.go index 0eca7fab..a301c7c0 100644 --- a/thumbnailing/i/ogg.go +++ b/thumbnailing/i/ogg.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "github.com/faiface/beep" @@ -12,8 +12,7 @@ import ( "github.com/t2bot/matrix-media-repo/util/readers" ) -type oggGenerator struct { -} +type oggGenerator struct{} func (d oggGenerator) supportedContentTypes() []string { return []string{"audio/ogg"} @@ -30,7 +29,7 @@ func (d oggGenerator) matches(img io.Reader, contentType string) bool { func (d oggGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := vorbis.Decode(readers.MakeCloser(b)) if err != nil { - return audio, format, errors.New("ogg: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("ogg: error decoding audio: %w", err) } return audio, format, nil } @@ -42,7 +41,7 @@ func (d oggGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { - return nil, errors.New("ogg: error getting tags: " + err.Error()) + return nil, fmt.Errorf("ogg: error getting tags: %v", err) } //goland:noinspection GoUnhandledErrorResult defer rc.Close() diff --git a/thumbnailing/i/png.go b/thumbnailing/i/png.go index f546ecf6..f73f2b10 100644 --- a/thumbnailing/i/png.go +++ b/thumbnailing/i/png.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "image" _ "image/png" "io" @@ -38,7 +38,7 @@ func (d pngGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { src, err := imaging.Decode(b) if err != nil { - return nil, errors.New("png: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("png: error decoding thumbnail: %w", err) } return d.GenerateThumbnailOf(src, width, height, method, ctx) @@ -54,7 +54,7 @@ func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int go func(pw *io.PipeWriter, p image.Image) { err = u.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("png: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("png: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } diff --git a/thumbnailing/i/svg.go b/thumbnailing/i/svg.go index fe232a36..22cfc207 100644 --- a/thumbnailing/i/svg.go +++ b/thumbnailing/i/svg.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "os" "os/exec" @@ -33,7 +33,7 @@ func (d svgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-svg") if err != nil { - return nil, errors.New("svg: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("svg: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.svg") @@ -43,22 +43,22 @@ func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("svg: error creating temp svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error creating temp svg file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("svg: error writing temp svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error writing temp svg file: %w", err) } err = exec.Command("convert", tempFile1, tempFile2).Run() if err != nil { - return nil, errors.New("svg: error converting svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error converting svg file: %w", err) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("svg: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("svg: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/tiff.go b/thumbnailing/i/tiff.go index ddd511f7..d381243b 100644 --- a/thumbnailing/i/tiff.go +++ b/thumbnailing/i/tiff.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -35,7 +35,7 @@ func (d tiffGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx func (d tiffGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { src, err := tiff.Decode(b) if err != nil { - return nil, errors.New("tiff: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("tiff: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/i/wav.go b/thumbnailing/i/wav.go index a1e497d7..caff42e9 100644 --- a/thumbnailing/i/wav.go +++ b/thumbnailing/i/wav.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "github.com/faiface/beep" @@ -29,7 +29,7 @@ func (d wavGenerator) matches(img io.Reader, contentType string) bool { func (d wavGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := wav.Decode(b) if err != nil { - return audio, format, errors.New("wav: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("wav: error decoding audio: %w", err) } return audio, format, nil } @@ -41,7 +41,7 @@ func (d wavGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { - return nil, errors.New("wav: error getting tags: " + err.Error()) + return nil, fmt.Errorf("wav: error getting tags: %w", err) } //goland:noinspection GoUnhandledErrorResult defer rc.Close() diff --git a/thumbnailing/i/webp.go b/thumbnailing/i/webp.go index d36b451f..21dec2c8 100644 --- a/thumbnailing/i/webp.go +++ b/thumbnailing/i/webp.go @@ -1,7 +1,7 @@ package i import ( - "errors" + "fmt" "io" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -35,7 +35,7 @@ func (d webpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx func (d webpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { src, err := webp.Decode(b) if err != nil { - return nil, errors.New("webp: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("webp: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index 30ad858a..2d46ed8b 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -2,6 +2,7 @@ package thumbnailing import ( "errors" + "fmt" "io" "reflect" "slices" @@ -43,7 +44,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h buffered := readers.NewBufferReadsReader(reconstructed) dimensional, w, h, err := generator.GetOriginDimensions(buffered, contentType, ctx) if err != nil { - return nil, errors.New("error getting dimensions: " + err.Error()) + return nil, fmt.Errorf("error getting dimensions: %w", err) } if dimensional { if (w * h) >= ctx.Config.Thumbnails.MaxPixels { diff --git a/thumbnailing/u/exif.go b/thumbnailing/u/exif.go index 636831ed..5123911c 100644 --- a/thumbnailing/u/exif.go +++ b/thumbnailing/u/exif.go @@ -16,16 +16,16 @@ type ExifOrientation struct { func GetExifOrientation(img io.Reader) (*ExifOrientation, error) { rawExif, err := exif.SearchAndExtractExifWithReader(img) - if err != nil { - if errors.Is(err, exif.ErrNoExif) { - return nil, nil - } - return nil, errors.New("exif: error reading possible exif data: " + err.Error()) + switch { + case errors.Is(err, exif.ErrNoExif): + return nil, nil + case err != nil: + return nil, fmt.Errorf("exif: error reading possible exif data: %w", err) } tags, _, err := exif.GetFlatExifData(rawExif, nil) if err != nil { - return nil, errors.New("exif: error parsing exif data: " + err.Error()) + return nil, fmt.Errorf("exif: error parsing exif data: %w", err) } var tag exif.ExifTag diff --git a/thumbnailing/u/framing.go b/thumbnailing/u/framing.go index 80f5f9f2..4ffcb7d8 100644 --- a/thumbnailing/u/framing.go +++ b/thumbnailing/u/framing.go @@ -26,7 +26,7 @@ func ExtractExifOrientation(r io.Reader) *ExifOrientation { orientation, err := GetExifOrientation(r) if err != nil { // assume no orientation if there was an error reading the exif header - logrus.Warn("Non-fatal error reading exif headers:", err.Error()) + logrus.Warnf("Non-fatal error reading exif headers: %v", err) sentry.CaptureException(err) orientation = nil } diff --git a/thumbnailing/u/metadata.go b/thumbnailing/u/metadata.go index 5607ded4..6a2eedb9 100644 --- a/thumbnailing/u/metadata.go +++ b/thumbnailing/u/metadata.go @@ -1,7 +1,7 @@ package u import ( - "errors" + "fmt" "io" "os" @@ -18,8 +18,8 @@ func GetID3Tags(b io.Reader) (tag.Metadata, io.ReadSeekCloser, error) { tryCleanup := func() { if f != nil { if err = os.Remove(f.Name()); err != nil && !os.IsNotExist(err) { - logrus.Warnf("Error deleting temp file '%s': %s", f.Name(), err.Error()) - sentry.CaptureException(errors.New("id3: error deleting temp file: " + err.Error())) + logrus.Warnf("Error deleting temp file '%s': %v", f.Name(), err) + sentry.CaptureException(fmt.Errorf("id3: error deleting temp file: %v", err)) } } } diff --git a/thumbnailing/u/sample.go b/thumbnailing/u/sample.go index 4e055add..c6854575 100644 --- a/thumbnailing/u/sample.go +++ b/thumbnailing/u/sample.go @@ -2,6 +2,7 @@ package u import ( "errors" + "fmt" "math" "github.com/faiface/beep" @@ -16,7 +17,7 @@ func FastSampleAudio(stream beep.StreamSeekCloser, numSamples int) ([][2]float64 if stream.Position() != pos { err := stream.Seek(pos) if err != nil { - return nil, errors.New("fast-sample: could not seek: " + err.Error()) + return nil, fmt.Errorf("fast-sample: could not seek: %w", err) } } From 6dd81b16a85711ded6053ac946927019e4914a48 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Tue, 27 Feb 2024 08:15:20 +0100 Subject: [PATCH 03/22] fix send on closed channel --- pipelines/_steps/url_preview/upload_image.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pipelines/_steps/url_preview/upload_image.go b/pipelines/_steps/url_preview/upload_image.go index 4feafadf..54182107 100644 --- a/pipelines/_steps/url_preview/upload_image.go +++ b/pipelines/_steps/url_preview/upload_image.go @@ -21,8 +21,7 @@ func UploadImage(ctx rcontext.RequestContext, image *m.PreviewImage, onHost stri defer image.Data.Close() pr, pw := io.Pipe() tee := io.TeeReader(image.Data, pw) - mediaChan := make(chan *database.DbMedia) - defer close(mediaChan) + mediaChan := make(chan *database.DbMedia, 1) go func() { media, err := pipeline_upload.Execute(ctx, onHost, "", io.NopCloser(tee), image.ContentType, image.Filename, userId, datastores.LocalMediaKind) if err != nil { @@ -30,12 +29,8 @@ func UploadImage(ctx rcontext.RequestContext, image *m.PreviewImage, onHost stri } else { _ = pw.Close() } - go func() { - defer func() { - recover() // consume write-to-closed-channel errors - }() - mediaChan <- media - }() + mediaChan <- media + close(mediaChan) }() w := 0 From 3e03f079dc8346f861cba98f842dfae3ae8fc7f6 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Tue, 27 Feb 2024 09:32:01 +0100 Subject: [PATCH 04/22] fix panic being used as a way to communicate errors --- api/_routers/03-host_detection.go | 14 +++++++------- api/_routers/97-require-repo-admin.go | 5 +++-- api/_routers/98-use-rcontext.go | 9 ++++++--- api/router.go | 18 +++++++++--------- api/webserver.go | 6 ------ 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/api/_routers/03-host_detection.go b/api/_routers/03-host_detection.go index 7e908559..c4c1040c 100644 --- a/api/_routers/03-host_detection.go +++ b/api/_routers/03-host_detection.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net" "net/http" "strings" @@ -61,13 +60,14 @@ func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { }).Inc() logger.Warnf("The server name provided ('%s') in the Host header is not configured, or the request was made directly to the media repo. Please specify a Host header and check your reverse proxy configuration. The request is being rejected.", r.Host) w.WriteHeader(http.StatusBadGateway) - if b, err := json.Marshal(_responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { - panic(fmt.Errorf("error preparing BadGatewayError: %v", err)) - } else { - if _, err = w.Write(b); err != nil { - panic(fmt.Errorf("error sending BadGatewayError: %v", err)) - } + + var b []byte + if b, err = json.Marshal(_responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { + logger.Errorf("Error preparing BadGateway: %v", err) + sentry.CaptureException(err) + return } + w.Write(b) return // don't call next handler } diff --git a/api/_routers/97-require-repo-admin.go b/api/_routers/97-require-repo-admin.go index cac1c38a..25b93baa 100644 --- a/api/_routers/97-require-repo-admin.go +++ b/api/_routers/97-require-repo-admin.go @@ -1,9 +1,9 @@ package _routers import ( - "errors" "net/http" + "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -14,7 +14,8 @@ func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { if user.UserId == "" { - panic(errors.New("safety check failed: Repo admin access check received empty user ID")) + logrus.Error("safety check failed: Repo admin access check received empty user ID") + return _responses.AuthFailed() } if !user.IsShared && !util.IsGlobalAdmin(user.UserId) { diff --git a/api/_routers/98-use-rcontext.go b/api/_routers/98-use-rcontext.go index 3b0a5422..7a3045fe 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/_routers/98-use-rcontext.go @@ -225,7 +225,9 @@ beforeParseDownload: contentType = "application/json" b, err := json.Marshal(res) if err != nil { - panic(err) // blow up this request + sentry.CaptureException(err) + log.Errorf("Failed to marshal response: %v", err) + return } stream = io.NopCloser(bytes.NewReader(b)) expectedBytes = int64(len(b)) @@ -253,10 +255,11 @@ beforeParseDownload: defer stream.Close() written, err := io.Copy(w, stream) if err != nil { - panic(err) // blow up this request + log.Errorf("Failed to write response: %v", err) } if expectedBytes > 0 && written != expectedBytes { - panic(errors.New(fmt.Sprintf("mismatch transfer size: %d expected, %d sent", expectedBytes, written))) + sentry.CaptureException(fmt.Errorf("expected %d bytes, but only wrote %d bytes for %q", expectedBytes, written, r.URL.Path)) + log.Warnf("Expected %d bytes, but only wrote %d bytes", expectedBytes, written) } } diff --git a/api/router.go b/api/router.go index d9738013..48aaa885 100644 --- a/api/router.go +++ b/api/router.go @@ -29,25 +29,25 @@ func buildPrimaryRouter() *httprouter.Router { func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusMethodNotAllowed) - reponse, err := json.Marshal(_responses.MethodNotAllowed()) + b, err := json.Marshal(_responses.MethodNotAllowed()) if err != nil { - sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %w", err)) + sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %v", err)) logrus.Errorf("error preparing MethodNotAllowed: %v", err) return } - w.Write(reponse) + w.Write(b) } func notFoundFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) - reponse, err := json.Marshal(_responses.NotFoundError()) + b, err := json.Marshal(_responses.NotFoundError()) if err != nil { - sentry.CaptureException(fmt.Errorf("error preparing NotFound: %w", err)) + sentry.CaptureException(fmt.Errorf("error preparing NotFound: %v", err)) logrus.Errorf("error preparing NotFound: %v", err) return } - w.Write(reponse) + w.Write(b) } func finishCorsFn(w http.ResponseWriter, r *http.Request) { @@ -67,11 +67,11 @@ func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - reponse, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) + b, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) if err != nil { - sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %w", err)) + sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %v", err)) logrus.Errorf("error preparing InternalServerError: %v", err) return } - w.Write(reponse) + w.Write(b) } diff --git a/api/webserver.go b/api/webserver.go index 1877b9a7..aade5acb 100644 --- a/api/webserver.go +++ b/api/webserver.go @@ -25,12 +25,6 @@ var reload = false func Init() *sync.WaitGroup { address := net.JoinHostPort(config.Get().General.BindAddress, strconv.Itoa(config.Get().General.Port)) - //defer func() { - // if err := recover(); err != nil { - // logrus.Fatal(err) - // } - //}() - handler := buildRoutes() if config.Get().RateLimit.Enabled { From ef35ac7010a5289b6b2eb8abf8790642a473d967 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Tue, 27 Feb 2024 18:33:50 +0100 Subject: [PATCH 05/22] get rid of some unnecessary panics --- api/router.go | 4 ---- api/webserver.go | 2 +- cmd/homeserver_live_importers/import_dendrite/main.go | 3 ++- cmd/homeserver_live_importers/import_synapse/main.go | 3 ++- common/assets/process.go | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/router.go b/api/router.go index 48aaa885..f53f5176 100644 --- a/api/router.go +++ b/api/router.go @@ -2,16 +2,12 @@ package api import ( "encoding/json" - "errors" "fmt" "net/http" - "github.com/getsentry/sentry-go" "github.com/julienschmidt/httprouter" - "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" - "github.com/t2bot/matrix-media-repo/util" ) func buildPrimaryRouter() *httprouter.Router { diff --git a/api/webserver.go b/api/webserver.go index aade5acb..c5024ec0 100644 --- a/api/webserver.go +++ b/api/webserver.go @@ -80,7 +80,7 @@ func Stop() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := srv.Shutdown(ctx); err != nil { - panic(err) + logrus.Fatalf("Could not gracefully shutdown the server: %v", err) } } } diff --git a/cmd/homeserver_live_importers/import_dendrite/main.go b/cmd/homeserver_live_importers/import_dendrite/main.go index b434c9fe..cae20c32 100644 --- a/cmd/homeserver_live_importers/import_dendrite/main.go +++ b/cmd/homeserver_live_importers/import_dendrite/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/dendrite" @@ -13,7 +14,7 @@ func main() { ctx.Log.Debug("Connecting to homeserver database...") hsDb, err := dendrite.OpenDatabase(cfg.ConnectionString, cfg.ServerName) if err != nil { - panic(err) + logrus.Fatalf("Failed to open database: %v", err) } _common.PsqlMatrixDownloadCopy[dendrite.LocalMedia](ctx, cfg, hsDb, func(record *dendrite.LocalMedia) (*_common.MediaMetadata, error) { diff --git a/cmd/homeserver_live_importers/import_synapse/main.go b/cmd/homeserver_live_importers/import_synapse/main.go index 03a9f442..8edbcd69 100644 --- a/cmd/homeserver_live_importers/import_synapse/main.go +++ b/cmd/homeserver_live_importers/import_synapse/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -13,7 +14,7 @@ func main() { ctx.Log.Debug("Connecting to homeserver database...") hsDb, err := synapse.OpenDatabase(cfg.ConnectionString) if err != nil { - panic(err) + logrus.Fatalf("Failed to open database: %v", err) } _common.PsqlMatrixDownloadCopy[synapse.LocalMedia](ctx, cfg, hsDb, func(record *synapse.LocalMedia) (*_common.MediaMetadata, error) { diff --git a/common/assets/process.go b/common/assets/process.go index 1ab0851c..10e294c6 100644 --- a/common/assets/process.go +++ b/common/assets/process.go @@ -24,7 +24,7 @@ func SetupMigrations(givenMigrationsPath string) { if !exists { tempMigrations, err = os.MkdirTemp(os.TempDir(), "media-repo-migrations") if err != nil { - panic(err) + logrus.Fatalf("Failed to create temporary migrations directory: %v", err) } logrus.Info("Migrations path doesn't exist - attempting to unpack from compiled data") extractPrefixTo("migrations", tempMigrations) From 9518366b9d677f97c8da1e220324c06db0ccd942 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Tue, 27 Feb 2024 15:55:02 +0100 Subject: [PATCH 06/22] improve variable naming --- api/_auth_cache/auth_cache.go | 20 ++++++++--------- api/_routers/00-install-params.go | 6 ++--- api/_routers/01-install_metadata.go | 28 ++++++++++++------------ api/branched_route.go | 20 ++++++++--------- api/custom/datastores.go | 18 +++++++-------- api/router.go | 16 +++++++++----- api/webserver.go | 12 +++++----- database/table_url_previews.go | 12 +++++----- matrix/server_discovery.go | 34 ++++++++++++++--------------- util/strings.go | 4 ++-- 10 files changed, 88 insertions(+), 82 deletions(-) diff --git a/api/_auth_cache/auth_cache.go b/api/_auth_cache/auth_cache.go index da1f2d5c..f3721496 100644 --- a/api/_auth_cache/auth_cache.go +++ b/api/_auth_cache/auth_cache.go @@ -103,25 +103,25 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId return checkTokenWithHomeserver(ctx, accessToken, appserviceUserId, true) } - for _, r := range ctx.Config.AccessTokens.Appservices { - if r.AppserviceToken != accessToken { + for _, appSrv := range ctx.Config.AccessTokens.Appservices { + if appSrv.AppserviceToken != accessToken { continue } - if r.SenderUserId != "" && (r.SenderUserId == appserviceUserId || appserviceUserId == "") { - ctx.Log.Debugf("Access token belongs to appservice (sender user ID): %s", r.Id) - cacheToken(ctx, accessToken, appserviceUserId, r.SenderUserId, nil) - return r.SenderUserId, nil + if appSrv.SenderUserId != "" && (appSrv.SenderUserId == appserviceUserId || appserviceUserId == "") { + ctx.Log.Debugf("Access token belongs to appservice (sender user ID): %s", appSrv.Id) + cacheToken(ctx, accessToken, appserviceUserId, appSrv.SenderUserId, nil) + return appSrv.SenderUserId, nil } - for _, n := range r.UserNamespaces { + for _, n := range appSrv.UserNamespaces { regex, ok := regexCache[n.Regex] if !ok { regex = regexp.MustCompile(n.Regex) regexCache[n.Regex] = regex } if regex.MatchString(appserviceUserId) { - ctx.Log.Debugf("Access token belongs to appservice: %s", r.Id) + ctx.Log.Debugf("Access token belongs to appservice: %s", appSrv.Id) cacheToken(ctx, accessToken, appserviceUserId, appserviceUserId, nil) return appserviceUserId, nil } @@ -133,13 +133,13 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId } func cacheToken(ctx rcontext.RequestContext, accessToken string, appserviceUserId string, userId string, err error) { - v := cachedToken{ + token := cachedToken{ userId: userId, err: err, } t := time.Duration(ctx.Config.AccessTokens.MaxCacheTimeSeconds) * time.Second rwLock.Lock() - tokenCache.Set(cacheKey(accessToken, appserviceUserId), v, t) + tokenCache.Set(cacheKey(accessToken, appserviceUserId), token, t) rwLock.Unlock() } diff --git a/api/_routers/00-install-params.go b/api/_routers/00-install-params.go index 2f58f178..b289ed63 100644 --- a/api/_routers/00-install-params.go +++ b/api/_routers/00-install-params.go @@ -12,11 +12,11 @@ var ServerNameRegex = regexp.MustCompile("[a-zA-Z0-9.:\\-_]+") // var NumericIdRegex = regexp.MustCompile("[0-9]+") func GetParam(name string, r *http.Request) string { - p := httprouter.ParamsFromContext(r.Context()) - if p == nil { + parameter := httprouter.ParamsFromContext(r.Context()) + if parameter == nil { return "" } - return p.ByName(name) + return parameter.ByName(name) } func ForceSetParam(name string, val string, r *http.Request) *http.Request { diff --git a/api/_routers/01-install_metadata.go b/api/_routers/01-install_metadata.go index db3fa1d0..dc6fa9d2 100644 --- a/api/_routers/01-install_metadata.go +++ b/api/_routers/01-install_metadata.go @@ -37,8 +37,8 @@ func NewInstallMetadataRouter(ignoreHost bool, actionName string, counter *Reque } } -func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { - requestId := i.counter.NextId() +func (router *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + requestId := router.counter.NextId() logger := logrus.WithFields(logrus.Fields{ "method": r.Method, "host": r.Host, @@ -54,44 +54,44 @@ func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request ctx := r.Context() ctx = context.WithValue(ctx, common.ContextRequestStartTime, util.NowMillis()) ctx = context.WithValue(ctx, common.ContextRequestId, requestId) - ctx = context.WithValue(ctx, common.ContextAction, i.actionName) - ctx = context.WithValue(ctx, common.ContextIgnoreHost, i.ignoreHost) + ctx = context.WithValue(ctx, common.ContextAction, router.actionName) + ctx = context.WithValue(ctx, common.ContextIgnoreHost, router.ignoreHost) ctx = context.WithValue(ctx, common.ContextLogger, logger) r = r.WithContext(ctx) - if i.next != nil { - i.next.ServeHTTP(w, r) + if router.next != nil { + router.next.ServeHTTP(w, r) } } func GetActionName(r *http.Request) string { - x, ok := r.Context().Value(common.ContextAction).(string) + action, ok := r.Context().Value(common.ContextAction).(string) if !ok { return "" } - return x + return action } func ShouldIgnoreHost(r *http.Request) bool { - x, ok := r.Context().Value(common.ContextIgnoreHost).(bool) + ignoreHost, ok := r.Context().Value(common.ContextIgnoreHost).(bool) if !ok { return false } - return x + return ignoreHost } func GetLogger(r *http.Request) *logrus.Entry { - x, ok := r.Context().Value(common.ContextLogger).(*logrus.Entry) + log, ok := r.Context().Value(common.ContextLogger).(*logrus.Entry) if !ok { return nil } - return x + return log } func GetRequestDuration(r *http.Request) float64 { - x, ok := r.Context().Value(common.ContextRequestStartTime).(int64) + duration, ok := r.Context().Value(common.ContextRequestStartTime).(int64) if !ok { return -1 } - return float64(util.NowMillis()-x) / 1000.0 + return float64(util.NowMillis()-duration) / 1000.0 } diff --git a/api/branched_route.go b/api/branched_route.go index 6dc9c98f..04090e30 100644 --- a/api/branched_route.go +++ b/api/branched_route.go @@ -19,10 +19,10 @@ type splitBranch struct { func branchedRoute(branches []branch) http.Handler { sbranches := make([]splitBranch, len(branches)) - for i, b := range branches { + for i, branch := range branches { sbranches[i] = splitBranch{ - segments: strings.Split(b.string, "/"), - handler: b.Handler, + segments: strings.Split(branch.string, "/"), + handler: branch.Handler, } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -31,17 +31,17 @@ func branchedRoute(branches []branch) http.Handler { catchAll = catchAll[1:] } params := strings.Split(catchAll, "/") - for _, b := range sbranches { - if b.segments[0][0] == ':' || b.segments[0] == params[0] { - if len(b.segments) != len(params) { + for _, branch := range sbranches { + if branch.segments[0][0] == ':' || branch.segments[0] == params[0] { + if len(branch.segments) != len(params) { continue } - for i, s := range b.segments { - if s[0] == ':' { - r = _routers.ForceSetParam(s[1:], params[i], r) + for i, segment := range branch.segments { + if segment[0] == ':' { + r = _routers.ForceSetParam(segment[1:], params[i], r) } } - b.handler.ServeHTTP(w, r) + branch.handler.ServeHTTP(w, r) return } } diff --git a/api/custom/datastores.go b/api/custom/datastores.go index 9e43907c..bdb7c538 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" @@ -16,7 +17,6 @@ import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DatastoreMigration struct { @@ -26,17 +26,17 @@ type DatastoreMigration struct { func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { response := make(map[string]interface{}) - for _, ds := range config.UniqueDatastores() { - uri, err := datastores.GetUri(ds) + for _, store := range config.UniqueDatastores() { + uri, err := datastores.GetUri(store) if err != nil { sentry.CaptureException(err) rctx.Log.Error("Error getting datastore URI: ", err) return _responses.InternalServerError(errors.New("unexpected error getting datastore information")) } - dsMap := make(map[string]interface{}) - dsMap["type"] = ds.Type - dsMap["uri"] = uri - response[ds.Id] = dsMap + dataStoreMap := make(map[string]interface{}) + dataStoreMap["type"] = store.Type + dataStoreMap["uri"] = uri + response[store.Id] = dataStoreMap } return &_responses.DoNotCacheResponse{Payload: response} @@ -44,7 +44,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta. func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixNano() / int64(time.Millisecond) var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) @@ -97,7 +97,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixNano() / int64(time.Millisecond) var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) diff --git a/api/router.go b/api/router.go index f53f5176..dbe3d5ee 100644 --- a/api/router.go +++ b/api/router.go @@ -2,12 +2,16 @@ package api import ( "encoding/json" + "errors" "fmt" "net/http" + "github.com/getsentry/sentry-go" "github.com/julienschmidt/httprouter" + "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/util" ) func buildPrimaryRouter() *httprouter.Router { @@ -25,25 +29,25 @@ func buildPrimaryRouter() *httprouter.Router { func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusMethodNotAllowed) - b, err := json.Marshal(_responses.MethodNotAllowed()) + reponse, err := json.Marshal(_responses.MethodNotAllowed()) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %v", err)) logrus.Errorf("error preparing MethodNotAllowed: %v", err) return } - w.Write(b) + w.Write(reponse) } func notFoundFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) - b, err := json.Marshal(_responses.NotFoundError()) + reponse, err := json.Marshal(_responses.NotFoundError()) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing NotFound: %v", err)) logrus.Errorf("error preparing NotFound: %v", err) return } - w.Write(b) + w.Write(reponse) } func finishCorsFn(w http.ResponseWriter, r *http.Request) { @@ -63,11 +67,11 @@ func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - b, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) + reponse, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %v", err)) logrus.Errorf("error preparing InternalServerError: %v", err) return } - w.Write(b) + w.Write(reponse) } diff --git a/api/webserver.go b/api/webserver.go index c5024ec0..25c761bc 100644 --- a/api/webserver.go +++ b/api/webserver.go @@ -18,9 +18,11 @@ import ( "github.com/t2bot/matrix-media-repo/common/config" ) -var srv *http.Server -var waitGroup = &sync.WaitGroup{} -var reload = false +var ( + srv *http.Server + waitGroup = &sync.WaitGroup{} + reload = false +) func Init() *sync.WaitGroup { address := net.JoinHostPort(config.Get().General.BindAddress, strconv.Itoa(config.Get().General.Port)) @@ -35,8 +37,8 @@ func Init() *sync.WaitGroup { limiter.SetBurst(config.Get().RateLimit.BurstCount) limiter.SetMax(config.Get().RateLimit.RequestsPerSecond) - b, _ := json.Marshal(_responses.RateLimitReached()) - limiter.SetMessage(string(b)) + reponse, _ := json.Marshal(_responses.RateLimitReached()) + limiter.SetMessage(string(reponse)) limiter.SetMessageContentType("application/json") handler = tollbooth.LimitHandler(limiter, handler) diff --git a/database/table_url_previews.go b/database/table_url_previews.go index 0ad62b58..cbad3edd 100644 --- a/database/table_url_previews.go +++ b/database/table_url_previews.go @@ -65,8 +65,8 @@ func (s *urlPreviewsTableStatements) Prepare(ctx rcontext.RequestContext) *urlPr } } -func (s *urlPreviewsTableWithContext) Get(url string, ts int64, languageHeader string) (*DbUrlPreview, error) { - row := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, ts, languageHeader) +func (s *urlPreviewsTableWithContext) Get(url string, timestamp int64, languageHeader string) (*DbUrlPreview, error) { + row := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, timestamp, languageHeader) val := &DbUrlPreview{} err := row.Scan(&val.Url, &val.ErrorCode, &val.BucketTs, &val.SiteUrl, &val.SiteName, &val.ResourceType, &val.Description, &val.Title, &val.ImageMxc, &val.ImageType, &val.ImageSize, &val.ImageWidth, &val.ImageHeight, &val.LanguageHeader) if errors.Is(err, sql.ErrNoRows) { @@ -75,8 +75,8 @@ func (s *urlPreviewsTableWithContext) Get(url string, ts int64, languageHeader s return val, err } -func (s *urlPreviewsTableWithContext) Insert(p *DbUrlPreview) error { - _, err := s.statements.insertUrlPreview.ExecContext(s.ctx, p.Url, p.ErrorCode, p.BucketTs, p.SiteUrl, p.SiteName, p.ResourceType, p.Description, p.Title, p.ImageMxc, p.ImageType, p.ImageSize, p.ImageWidth, p.ImageHeight, p.LanguageHeader) +func (s *urlPreviewsTableWithContext) Insert(preview *DbUrlPreview) error { + _, err := s.statements.insertUrlPreview.ExecContext(s.ctx, preview.Url, preview.ErrorCode, preview.BucketTs, preview.SiteUrl, preview.SiteName, preview.ResourceType, preview.Description, preview.Title, preview.ImageMxc, preview.ImageType, preview.ImageSize, preview.ImageWidth, preview.ImageHeight, preview.LanguageHeader) return err } @@ -89,7 +89,7 @@ func (s *urlPreviewsTableWithContext) InsertError(url string, errorCode string) }) } -func (s *urlPreviewsTableWithContext) DeleteOlderThan(ts int64) error { - _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, ts) +func (s *urlPreviewsTableWithContext) DeleteOlderThan(timestamp int64) error { + _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, timestamp) return err } diff --git a/matrix/server_discovery.go b/matrix/server_discovery.go index e8a0b808..2d85869f 100644 --- a/matrix/server_discovery.go +++ b/matrix/server_discovery.go @@ -49,11 +49,11 @@ func GetServerApiUrl(hostname string) (string, string, error) { } addrErr := &net.AddrError{} - h, p, err := net.SplitHostPort(hostname) + host, port, err := net.SplitHostPort(hostname) defPort := false switch { case errors.As(err, &addrErr) && addrErr.Err == "missing port in address": - h, p, err = net.SplitHostPort(hostname + ":8448") + host, port, err = net.SplitHostPort(hostname + ":8448") if err != nil { return "", "", fmt.Errorf("failed to parse hostname: %w", err) } @@ -62,9 +62,9 @@ func GetServerApiUrl(hostname string) (string, string, error) { } // Step 1 of the discovery process: if the hostname is an IP, use that with explicit or default port - logrus.Debug("Testing if " + h + " is an IP address") - if is.IP(h) { - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) + logrus.Debug("Testing if " + host + " is an IP address") + if is.IP(host) { + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) server := cachedServer{url, hostname} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debug("Server API URL for " + hostname + " is " + url + " (IP address)") @@ -74,17 +74,17 @@ func GetServerApiUrl(hostname string) (string, string, error) { // Step 2: if the hostname is not an IP address, and an explicit port is given, use that logrus.Debug("Testing if a default port was used. Using default = ", defPort) if !defPort { - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) - server := cachedServer{url, h} + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debugf("Server API URL for %s is %s (explicit port)", hostname, url) - return url, h, nil + return url, host, nil } // Step 3: if the hostname is not an IP address and no explicit port is given, do .well-known // Note that we have sprawling branches here because we need to fall through to step 4 if parsing fails - logrus.Debug("Doing .well-known lookup on " + h) - r, err := http.Get(fmt.Sprintf("https://%s/.well-known/matrix/server", h)) + logrus.Debug("Doing .well-known lookup on " + host) + r, err := http.Get(fmt.Sprintf("https://%s/.well-known/matrix/server", host)) if r != nil { defer r.Body.Close() } @@ -183,10 +183,10 @@ func GetServerApiUrl(hostname string) (string, string, error) { realAddr = realAddr[0 : len(realAddr)-1] } url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) - server := cachedServer{url, h} + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debugf("Server API URL for %s is %s (SRV)", hostname, url) - return url, h, nil + return url, host, nil } // Step 5: try resolving a hostname using DEPRECATED SRV records and use it @@ -200,17 +200,17 @@ func GetServerApiUrl(hostname string) (string, string, error) { realAddr = realAddr[0 : len(realAddr)-1] } url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) - server := cachedServer{url, h} + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debugf("Server API URL for %s is %s (SRV-Deprecated)", hostname, url) - return url, h, nil + return url, host, nil } // Step 6: use the target host as-is logrus.Debug("Using host as-is: ", hostname) - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) - server := cachedServer{url, h} + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debugf("Server API URL for %s is %s (fallback)", hostname, url) - return url, h, nil + return url, host, nil } diff --git a/util/strings.go b/util/strings.go index 9ecc3ead..f7e5d46f 100644 --- a/util/strings.go +++ b/util/strings.go @@ -5,8 +5,8 @@ import ( ) func HasAnyPrefix(val string, prefixes []string) bool { - for _, p := range prefixes { - if strings.HasPrefix(val, p) { + for _, prefix := range prefixes { + if strings.HasPrefix(val, prefix) { return true } } From 87964bb12479bb33053df0db80442e3888139ffb Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sat, 2 Mar 2024 22:42:07 +0100 Subject: [PATCH 07/22] introduce the concept of time into the codebase --- api/custom/datastores.go | 4 +-- api/custom/purge.go | 21 +++++++------ api/r0/download.go | 9 ++++-- api/r0/preview_url.go | 8 +++-- api/r0/thumbnail.go | 9 ++++-- archival/v2archive/writer.go | 6 ++-- database/table_expiring_media.go | 7 +++-- database/table_media.go | 5 ++-- database/table_media_hold.go | 8 ++--- database/table_thumbnails.go | 5 ++-- database/table_url_previews.go | 8 ++--- pipelines/_steps/meta/flag_access.go | 8 +++-- pipelines/pipeline_create/pipeline.go | 12 ++++---- pipelines/pipeline_preview/pipeline.go | 13 ++++---- pipelines/pipeline_upload/pipeline.go | 4 +-- tasks/exec.go | 3 +- tasks/schedule.go | 5 ++-- tasks/task_runner/00-internal.go | 5 ++-- tasks/task_runner/purge_held_media_ids.go | 7 +++-- tasks/task_runner/purge_previews.go | 7 +++-- tasks/task_runner/purge_remote_media.go | 8 +++-- tasks/task_runner/purge_thumbnails.go | 5 ++-- util/time.go | 36 ----------------------- 23 files changed, 98 insertions(+), 105 deletions(-) delete mode 100644 util/time.go diff --git a/api/custom/datastores.go b/api/custom/datastores.go index bdb7c538..e96b1b1c 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -44,7 +44,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta. func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := time.Now().UnixNano() / int64(time.Millisecond) + beforeTs := time.Now().UnixMilli() var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) @@ -97,7 +97,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := time.Now().UnixNano() / int64(time.Millisecond) + beforeTs := time.Now().UnixMilli() var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) diff --git a/api/custom/purge.go b/api/custom/purge.go index f5b8b997..ee8c273d 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" @@ -33,13 +34,14 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } + before := time.UnixMilli(beforeTs) rctx = rctx.LogWithFields(logrus.Fields{ - "beforeTs": beforeTs, + "beforeTs": before, }) // We don't bother clearing the cache because it's still probably useful there - removed, err := task_runner.PurgeRemoteMediaBefore(rctx, beforeTs) + removed, err := task_runner.PurgeRemoteMediaBefore(rctx, before) if err != nil { rctx.Log.Error("Error purging remote media: ", err) sentry.CaptureException(err) @@ -119,13 +121,14 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { var err error - beforeTs := util.NowMillis() + before := time.Now() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { - beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) + beforeTS, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } + before = time.UnixMilli(beforeTS) } includeLocal := false @@ -138,7 +141,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. } rctx = rctx.LogWithFields(logrus.Fields{ - "before_ts": beforeTs, + "before_ts": before, "include_local": includeLocal, }) @@ -148,7 +151,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. } mediaDb := database.GetInstance().Media.Prepare(rctx) - records, err := mediaDb.GetOldExcluding(domains, beforeTs) + records, err := mediaDb.GetOldExcluding(domains, before) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) @@ -177,7 +180,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) @@ -234,7 +237,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) @@ -307,7 +310,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) diff --git a/api/r0/download.go b/api/r0/download.go index a662cac1..9cc59709 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" @@ -48,10 +49,14 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. canRedirect = parsedFlag } - blockFor, err := util.CalcBlockForDuration(timeoutMs) + timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } + timeout := time.Duration(timeoutMS) * time.Millisecond + if timeout > time.Minute { + timeout = time.Minute + } rctx = rctx.LogWithFields(logrus.Fields{ "mediaId": mediaId, @@ -68,7 +73,7 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. media, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{ FetchRemoteIfNeeded: downloadRemote, - BlockForReadUntil: blockFor, + BlockForReadUntil: timeout, CanRedirect: canRedirect, }) if err != nil { diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go index a061f8d6..5476b4c2 100644 --- a/api/r0/preview_url.go +++ b/api/r0/preview_url.go @@ -5,6 +5,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" @@ -13,7 +14,6 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type MatrixOpenGraph struct { @@ -39,7 +39,8 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use // Parse the parameters urlStr := params.Get("url") tsStr := params.Get("ts") - ts := util.NowMillis() + + var ts int64 var err error if tsStr != "" { ts, err = strconv.ParseInt(tsStr, 10, 64) @@ -48,6 +49,9 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use return _responses.BadRequest(err) } } + if ts == 0 { + ts = time.Now().UnixMilli() + } // Validate the URL if urlStr == "" { diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 9c4994b5..31126f5b 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" @@ -49,10 +50,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta canRedirect = parsedFlag } - blockFor, err := util.CalcBlockForDuration(timeoutMs) + timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } + timeout := time.Duration(timeoutMS) * time.Millisecond + if timeout > time.Minute { + timeout = time.Minute + } rctx = rctx.LogWithFields(logrus.Fields{ "mediaId": mediaId, @@ -121,7 +126,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta thumbnail, stream, err := pipeline_thumbnail.Execute(rctx, server, mediaId, pipeline_thumbnail.ThumbnailOpts{ DownloadOpts: pipeline_download.DownloadOpts{ FetchRemoteIfNeeded: downloadRemote, - BlockForReadUntil: blockFor, + BlockForReadUntil: timeout, RecordOnly: false, // overridden CanRedirect: canRedirect, }, diff --git a/archival/v2archive/writer.go b/archival/v2archive/writer.go index 4eef0d8c..bb3c1e7d 100644 --- a/archival/v2archive/writer.go +++ b/archival/v2archive/writer.go @@ -150,7 +150,7 @@ func (w *ArchiveWriter) AppendMedia(file io.ReadCloser, info MediaInfo) (string, } internalName := fmt.Sprintf("%s__%s%s", info.Origin, info.MediaId, mime.Extension()) - createTime := util.FromMillis(info.CreationTs) + createTime := time.UnixMilli(info.CreationTs) size, sha256hash, err := w.putFile(br.GetRewoundReader(), internalName, createTime) if err != nil { @@ -210,7 +210,7 @@ func (w *ArchiveWriter) putFile(r io.Reader, name string, creationTime time.Time hasher := sha256.New() header := &tar.Header{ Name: name, - Mode: int64(0644), + Mode: int64(0o644), ModTime: creationTime, Size: i1, } @@ -242,7 +242,7 @@ func (w *ArchiveWriter) Finish() error { manifest := &Manifest{ Version: ManifestVersion, EntityId: w.entity, - CreatedTs: util.NowMillis(), + CreatedTs: time.Now().UnixMilli(), Media: w.mediaManifest, } pr, pw := io.Pipe() diff --git a/database/table_expiring_media.go b/database/table_expiring_media.go index 0c51dc7a..4fd218cd 100644 --- a/database/table_expiring_media.go +++ b/database/table_expiring_media.go @@ -4,9 +4,9 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbExpiringMedia struct { @@ -17,7 +17,8 @@ type DbExpiringMedia struct { } func (r *DbExpiringMedia) IsExpired() bool { - return r.ExpiresTs < util.NowMillis() + expiresTs := time.UnixMilli(r.ExpiresTs) + return expiresTs.Before(time.Now()) } const insertExpiringMedia = "INSERT INTO expiring_media (origin, media_id, user_id, expires_ts) VALUES ($1, $2, $3, $4);" @@ -72,7 +73,7 @@ func (s *expiringMediaTableWithContext) Insert(origin string, mediaId string, us } func (s *expiringMediaTableWithContext) ByUserCount(userId string) (int64, error) { - row := s.statements.selectExpiringMediaByUserCount.QueryRowContext(s.ctx, userId, util.NowMillis()) + row := s.statements.selectExpiringMediaByUserCount.QueryRowContext(s.ctx, userId, time.Now().UnixMilli()) val := int64(0) err := row.Scan(&val) if errors.Is(err, sql.ErrNoRows) { diff --git a/database/table_media.go b/database/table_media.go index 2d21a16d..f0dfd620 100644 --- a/database/table_media.go +++ b/database/table_media.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/lib/pq" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -234,8 +235,8 @@ func (s *MediaTableWithContext) GetByIds(origin string, mediaIds []string) ([]*D return s.scanRows(s.statements.selectMediaByOriginAndIds.QueryContext(s.ctx, origin, pq.Array(mediaIds))) } -func (s *MediaTableWithContext) GetOldExcluding(origins []string, beforeTs int64) ([]*DbMedia, error) { - return s.scanRows(s.statements.selectOldMediaExcludingDomains.QueryContext(s.ctx, pq.Array(origins), beforeTs)) +func (s *MediaTableWithContext) GetOldExcluding(origins []string, beforeTs time.Time) ([]*DbMedia, error) { + return s.scanRows(s.statements.selectOldMediaExcludingDomains.QueryContext(s.ctx, pq.Array(origins), beforeTs.UnixMilli())) } func (s *MediaTableWithContext) GetByLocation(datastoreId string, location string) ([]*DbMedia, error) { diff --git a/database/table_media_hold.go b/database/table_media_hold.go index 8af566bb..df094476 100644 --- a/database/table_media_hold.go +++ b/database/table_media_hold.go @@ -3,9 +3,9 @@ package database import ( "database/sql" "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbHeldMedia struct { @@ -55,11 +55,11 @@ func (s *heldMediaTableStatements) Prepare(ctx rcontext.RequestContext) *heldMed } func (s *heldMediaTableWithContext) TryInsert(origin string, mediaId string, reason HeldReason) error { - _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, util.NowMillis()) + _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, time.Now().UnixMilli()) return err } -func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThanTs int64) error { - _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThanTs) +func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThan time.Time) error { + _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThan.UnixMilli()) return err } diff --git a/database/table_thumbnails.go b/database/table_thumbnails.go index c0c936a7..9033e735 100644 --- a/database/table_thumbnails.go +++ b/database/table_thumbnails.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -124,8 +125,8 @@ func (s *thumbnailsTableWithContext) GetForMedia(origin string, mediaId string) return s.scanRows(s.statements.selectThumbnailsForMedia.QueryContext(s.ctx, origin, mediaId)) } -func (s *thumbnailsTableWithContext) GetOlderThan(ts int64) ([]*DbThumbnail, error) { - return s.scanRows(s.statements.selectOldThumbnails.QueryContext(s.ctx, ts)) +func (s *thumbnailsTableWithContext) GetOlderThan(ts time.Time) ([]*DbThumbnail, error) { + return s.scanRows(s.statements.selectOldThumbnails.QueryContext(s.ctx, ts.UnixMilli())) } func (s *thumbnailsTableWithContext) GetByLocation(datastoreId string, location string) ([]*DbThumbnail, error) { diff --git a/database/table_url_previews.go b/database/table_url_previews.go index cbad3edd..44c651e1 100644 --- a/database/table_url_previews.go +++ b/database/table_url_previews.go @@ -4,9 +4,9 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbUrlPreview struct { @@ -84,12 +84,12 @@ func (s *urlPreviewsTableWithContext) InsertError(url string, errorCode string) _ = s.Insert(&DbUrlPreview{ Url: url, ErrorCode: errorCode, - BucketTs: util.GetHourBucket(util.NowMillis()), + BucketTs: time.Now().UnixMilli(), // remainder of fields don't matter }) } -func (s *urlPreviewsTableWithContext) DeleteOlderThan(timestamp int64) error { - _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, timestamp) +func (s *urlPreviewsTableWithContext) DeleteOlderThan(ts time.Time) error { + _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, ts.UnixMilli()) return err } diff --git a/pipelines/_steps/meta/flag_access.go b/pipelines/_steps/meta/flag_access.go index 465e0913..8cc313d3 100644 --- a/pipelines/_steps/meta/flag_access.go +++ b/pipelines/_steps/meta/flag_access.go @@ -1,18 +1,20 @@ package meta import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/util" ) func FlagAccess(ctx rcontext.RequestContext, sha256hash string, uploadTime int64) { + uploaded := time.UnixMilli(uploadTime) if uploadTime > 0 { - metrics.MediaAgeAccessed.Observe(float64(util.NowMillis()-uploadTime) / 1000.0) + metrics.MediaAgeAccessed.Observe(time.Since(uploaded).Seconds()) } - if err := database.GetInstance().LastAccess.Prepare(ctx).Upsert(sha256hash, util.NowMillis()); err != nil { + if err := database.GetInstance().LastAccess.Prepare(ctx).Upsert(sha256hash, time.Now().UnixMilli()); err != nil { ctx.Log.Warnf("Non-fatal error while updating last access for '%s': %v", sha256hash, err) sentry.CaptureException(err) } diff --git a/pipelines/pipeline_create/pipeline.go b/pipelines/pipeline_create/pipeline.go index ea7d14ac..4946766b 100644 --- a/pipelines/pipeline_create/pipeline.go +++ b/pipelines/pipeline_create/pipeline.go @@ -1,11 +1,12 @@ package pipeline_create import ( + "time" + "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" - "github.com/t2bot/matrix-media-repo/util" ) const DefaultExpirationTime = 0 @@ -24,10 +25,11 @@ func Execute(ctx rcontext.RequestContext, origin string, userId string, expirati // Step 3: Insert record of expiration if expirationTime == DefaultExpirationTime { - expirationTime = ctx.Config.Uploads.MaxAgeSeconds * 1000 + expirationTime = ctx.Config.Uploads.MaxAgeSeconds } - expiresTs := util.NowMillis() + expirationTime - if err = database.GetInstance().ExpiringMedia.Prepare(ctx).Insert(origin, mediaId, userId, expiresTs); err != nil { + expires := time.Now().Add(time.Duration(expirationTime) * time.Second) + expiresTS := expires.UnixMilli() + if err = database.GetInstance().ExpiringMedia.Prepare(ctx).Insert(origin, mediaId, userId, expiresTS); err != nil { return nil, err } @@ -36,6 +38,6 @@ func Execute(ctx rcontext.RequestContext, origin string, userId string, expirati Origin: origin, MediaId: mediaId, UserId: userId, - ExpiresTs: expiresTs, + ExpiresTs: expiresTS, }, nil } diff --git a/pipelines/pipeline_preview/pipeline.go b/pipelines/pipeline_preview/pipeline.go index 040fb3ca..2de237f7 100644 --- a/pipelines/pipeline_preview/pipeline.go +++ b/pipelines/pipeline_preview/pipeline.go @@ -4,13 +4,13 @@ import ( "errors" "fmt" "net/url" + "time" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/pipelines/_steps/url_preview" "github.com/t2bot/matrix-media-repo/url_previewing/m" - "github.com/t2bot/matrix-media-repo/util" "golang.org/x/sync/singleflight" ) @@ -31,12 +31,13 @@ func Execute(ctx rcontext.RequestContext, onHost string, previewUrl string, user // Step 2: Fix timestamp bucket. If we're within 60 seconds of a bucket, just assume we're okay, so we don't // infinitely recurse into ourselves. - now := util.NowMillis() - atBucket := util.GetHourBucket(opts.Timestamp) // we should only be using this for the remainder of the function - nowBucket := util.GetHourBucket(now) - if (now-opts.Timestamp) > 60000 && atBucket != nowBucket { + requestTS := time.UnixMilli(opts.Timestamp) + atBucket := requestTS.UnixMilli() + nowBucket := time.Now().UnixMilli() + + if time.Now().After(requestTS.Add(60*time.Second)) && atBucket != nowBucket { return Execute(ctx, onHost, previewUrl, userId, PreviewOpts{ - Timestamp: now, + Timestamp: time.Now().Unix() * 1000, LanguageHeader: opts.LanguageHeader, }) } diff --git a/pipelines/pipeline_upload/pipeline.go b/pipelines/pipeline_upload/pipeline.go index 369c899e..82e7620e 100644 --- a/pipelines/pipeline_upload/pipeline.go +++ b/pipelines/pipeline_upload/pipeline.go @@ -3,6 +3,7 @@ package pipeline_upload import ( "errors" "io" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common" @@ -14,7 +15,6 @@ import ( "github.com/t2bot/matrix-media-repo/pipelines/_steps/meta" "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -117,7 +117,7 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, r io.Re ContentType: contentType, UserId: userId, SizeBytes: sizeBytes, - CreationTs: util.NowMillis(), + CreationTs: time.Now().UnixMilli(), Quarantined: false, Locatable: &database.Locatable{ Sha256Hash: sha256hash, diff --git a/tasks/exec.go b/tasks/exec.go index 55f42bc7..a842f86e 100644 --- a/tasks/exec.go +++ b/tasks/exec.go @@ -11,7 +11,6 @@ import ( "github.com/t2bot/matrix-media-repo/notifier" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/tasks/task_runner" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/ids" ) @@ -63,7 +62,7 @@ func beginTask(task *database.DbTask) { } runnerCtx := rcontext.Initial().LogWithFields(logrus.Fields{"task_id": task.TaskId}) - oneHourAgo := util.NowMillis() - (60 * 60 * 1000) + oneHourAgo := time.Now().Add(-1 * time.Hour).UnixMilli() if task.StartTs < oneHourAgo { runnerCtx.Log.Warn("Not starting task because it is more than 1 hour old.") return diff --git a/tasks/schedule.go b/tasks/schedule.go index 3f4b83b4..b3c0c578 100644 --- a/tasks/schedule.go +++ b/tasks/schedule.go @@ -11,7 +11,6 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/notifier" "github.com/t2bot/matrix-media-repo/tasks/task_runner" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/ids" ) @@ -34,7 +33,7 @@ const ExecutingMachineId = int64(0) type RecurringTaskFn func(ctx rcontext.RequestContext) -var localRand = rand.New(rand.NewSource(util.NowMillis())) +var localRand = rand.New(rand.NewSource(time.Now().UnixNano())) var recurDoneChs = make(map[RecurringTaskName]chan bool) var recurLock = new(sync.RWMutex) @@ -44,7 +43,7 @@ func scheduleTask(ctx rcontext.RequestContext, name TaskName, params interface{} return nil, err } db := database.GetInstance().Tasks.Prepare(ctx) - r, err := db.Insert(string(name), jsonParams, util.NowMillis()) + r, err := db.Insert(string(name), jsonParams, time.Now().UnixMilli()) if err != nil { return nil, err } diff --git a/tasks/task_runner/00-internal.go b/tasks/task_runner/00-internal.go index 7a39411b..e8c07abb 100644 --- a/tasks/task_runner/00-internal.go +++ b/tasks/task_runner/00-internal.go @@ -1,15 +1,16 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func markDone(ctx rcontext.RequestContext, task *database.DbTask) { taskDb := database.GetInstance().Tasks.Prepare(ctx) - if err := taskDb.SetEndTime(task.TaskId, util.NowMillis()); err != nil { + if err := taskDb.SetEndTime(task.TaskId, time.Now().UnixMilli()); err != nil { ctx.Log.Warn("Error updating task as complete: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_held_media_ids.go b/tasks/task_runner/purge_held_media_ids.go index 719f4f43..4c25cee4 100644 --- a/tasks/task_runner/purge_held_media_ids.go +++ b/tasks/task_runner/purge_held_media_ids.go @@ -1,19 +1,20 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func PurgeHeldMediaIds(ctx rcontext.RequestContext) { // dev note: don't use ctx for config lookup to avoid misreading it - beforeTs := util.NowMillis() - int64(7*24*60*60*1000) // 7 days + before := time.Now().AddDate(0, 0, -7) db := database.GetInstance().HeldMedia.Prepare(ctx) - if err := db.DeleteOlderThan(database.ForCreateHeldReason, beforeTs); err != nil { + if err := db.DeleteOlderThan(database.ForCreateHeldReason, before); err != nil { ctx.Log.Error("Error deleting held media IDs: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_previews.go b/tasks/task_runner/purge_previews.go index c7d3e305..7ecaea89 100644 --- a/tasks/task_runner/purge_previews.go +++ b/tasks/task_runner/purge_previews.go @@ -1,11 +1,12 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func PurgePreviews(ctx rcontext.RequestContext) { @@ -15,11 +16,11 @@ func PurgePreviews(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().UrlPreviews.ExpireDays*24*60*60*1000) + before := time.Now().AddDate(0, 0, -1*config.Get().UrlPreviews.ExpireDays) db := database.GetInstance().UrlPreviews.Prepare(ctx) // TODO: Fix https://github.com/t2bot/matrix-media-repo/issues/424 ("can't clean up preview media") - if err := db.DeleteOlderThan(beforeTs); err != nil { + if err := db.DeleteOlderThan(before); err != nil { ctx.Log.Error("Error deleting previews: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_remote_media.go b/tasks/task_runner/purge_remote_media.go index dc4a0f3b..1b4a7bf1 100644 --- a/tasks/task_runner/purge_remote_media.go +++ b/tasks/task_runner/purge_remote_media.go @@ -1,6 +1,8 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -15,8 +17,8 @@ func PurgeRemoteMedia(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().Downloads.ExpireDays*24*60*60*1000) - _, err := PurgeRemoteMediaBefore(ctx, beforeTs) + before := time.Now().AddDate(0, 0, -1*config.Get().Downloads.ExpireDays) + _, err := PurgeRemoteMediaBefore(ctx, before) if err != nil { ctx.Log.Error("Error purging media: ", err) sentry.CaptureException(err) @@ -24,7 +26,7 @@ func PurgeRemoteMedia(ctx rcontext.RequestContext) { } // PurgeRemoteMediaBefore returns (count affected, error) -func PurgeRemoteMediaBefore(ctx rcontext.RequestContext, beforeTs int64) (int, error) { +func PurgeRemoteMediaBefore(ctx rcontext.RequestContext, beforeTs time.Time) (int, error) { mediaDb := database.GetInstance().Media.Prepare(ctx) origins := util.GetOurDomains() diff --git a/tasks/task_runner/purge_thumbnails.go b/tasks/task_runner/purge_thumbnails.go index 79ea16ba..bd40b585 100644 --- a/tasks/task_runner/purge_thumbnails.go +++ b/tasks/task_runner/purge_thumbnails.go @@ -2,6 +2,7 @@ package task_runner import ( "fmt" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" @@ -18,9 +19,9 @@ func PurgeThumbnails(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().UrlPreviews.ExpireDays*24*60*60*1000) + before := time.Now().AddDate(0, 0, -1*config.Get().UrlPreviews.ExpireDays) thumbsDb := database.GetInstance().Thumbnails.Prepare(ctx) - old, err := thumbsDb.GetOlderThan(beforeTs) + old, err := thumbsDb.GetOlderThan(before) if err != nil { ctx.Log.Error("Error deleting thumbnails: ", err) sentry.CaptureException(err) diff --git a/util/time.go b/util/time.go deleted file mode 100644 index 2b8d5a30..00000000 --- a/util/time.go +++ /dev/null @@ -1,36 +0,0 @@ -package util - -import ( - "strconv" - "time" -) - -func NowMillis() int64 { - return time.Now().UnixNano() / 1000000 -} - -func FromMillis(m int64) time.Time { - return time.Unix(0, m*int64(time.Millisecond)) -} - -func CalcBlockForDuration(timeoutMs string) (time.Duration, error) { - blockFor := 20 * time.Second - if timeoutMs != "" { - parsed, err := strconv.Atoi(timeoutMs) - if err != nil { - return 0, err - } - if parsed > 0 { - // Limit to 60 seconds - if parsed > 60000 { - parsed = 60000 - } - blockFor = time.Duration(parsed) * time.Millisecond - } - } - return blockFor, nil -} - -func GetHourBucket(ts int64) int64 { - return (ts / 3600000) * 3600000 -} From 186f1c99777610c718a8a589c4e74d3b731c3144 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sat, 2 Mar 2024 18:37:23 +0100 Subject: [PATCH 08/22] rename thumbnailing/i to thumbnailing/preview --- api/_routers/01-install_metadata.go | 7 ++++--- api/unstable/info.go | 4 ++-- thumbnailing/{i => preview}/01-factories.go | 2 +- thumbnailing/{i => preview}/apng.go | 5 ++--- thumbnailing/{i => preview}/bmp.go | 2 +- thumbnailing/{i => preview}/flac.go | 2 +- thumbnailing/{i => preview}/gif.go | 5 ++--- thumbnailing/{i => preview}/heif.go | 2 +- thumbnailing/{i => preview}/jpegxl.go | 5 ++--- thumbnailing/{i => preview}/jpg.go | 2 +- thumbnailing/{i => preview}/mp3.go | 5 ++--- thumbnailing/{i => preview}/mp4.go | 2 +- thumbnailing/{i => preview}/ogg.go | 2 +- thumbnailing/{i => preview}/png.go | 5 ++--- thumbnailing/{i => preview}/svg.go | 5 ++--- thumbnailing/{i => preview}/tiff.go | 5 ++--- thumbnailing/{i => preview}/wav.go | 5 ++--- thumbnailing/{i => preview}/webp.go | 5 ++--- thumbnailing/thumbnail.go | 10 +++++----- 19 files changed, 36 insertions(+), 44 deletions(-) rename thumbnailing/{i => preview}/01-factories.go (98%) rename thumbnailing/{i => preview}/apng.go (98%) rename thumbnailing/{i => preview}/bmp.go (98%) rename thumbnailing/{i => preview}/flac.go (99%) rename thumbnailing/{i => preview}/gif.go (98%) rename thumbnailing/{i => preview}/heif.go (98%) rename thumbnailing/{i => preview}/jpegxl.go (97%) rename thumbnailing/{i => preview}/jpg.go (99%) rename thumbnailing/{i => preview}/mp3.go (99%) rename thumbnailing/{i => preview}/mp4.go (99%) rename thumbnailing/{i => preview}/ogg.go (99%) rename thumbnailing/{i => preview}/png.go (97%) rename thumbnailing/{i => preview}/svg.go (97%) rename thumbnailing/{i => preview}/tiff.go (96%) rename thumbnailing/{i => preview}/wav.go (97%) rename thumbnailing/{i => preview}/webp.go (96%) diff --git a/api/_routers/01-install_metadata.go b/api/_routers/01-install_metadata.go index dc6fa9d2..10ab4d04 100644 --- a/api/_routers/01-install_metadata.go +++ b/api/_routers/01-install_metadata.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "strconv" + "time" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common" @@ -52,7 +53,7 @@ func (router *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Re }) ctx := r.Context() - ctx = context.WithValue(ctx, common.ContextRequestStartTime, util.NowMillis()) + ctx = context.WithValue(ctx, common.ContextRequestStartTime, time.Now()) ctx = context.WithValue(ctx, common.ContextRequestId, requestId) ctx = context.WithValue(ctx, common.ContextAction, router.actionName) ctx = context.WithValue(ctx, common.ContextIgnoreHost, router.ignoreHost) @@ -89,9 +90,9 @@ func GetLogger(r *http.Request) *logrus.Entry { } func GetRequestDuration(r *http.Request) float64 { - duration, ok := r.Context().Value(common.ContextRequestStartTime).(int64) + duration, ok := r.Context().Value(common.ContextRequestStartTime).(time.Time) if !ok { return -1 } - return float64(util.NowMillis()-duration) / 1000.0 + return float64(time.Since(duration).Milliseconds()) } diff --git a/api/unstable/info.go b/api/unstable/info.go index d42a5d03..78f92307 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -18,7 +18,7 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/thumbnailing" - "github.com/t2bot/matrix-media-repo/thumbnailing/i" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/util" ) @@ -121,7 +121,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } else if strings.HasPrefix(response.ContentType, "audio/") { generator, reconstructed, err := thumbnailing.GetGenerator(stream, response.ContentType, false) if err == nil { - if audiogenerator, ok := generator.(i.AudioGenerator); ok { + if audiogenerator, ok := generator.(preview.AudioGenerator); ok { audioInfo, err := audiogenerator.GetAudioData(reconstructed, 768, rctx) if err == nil { response.KeySamples = audioInfo.KeySamples diff --git a/thumbnailing/i/01-factories.go b/thumbnailing/preview/01-factories.go similarity index 98% rename from thumbnailing/i/01-factories.go rename to thumbnailing/preview/01-factories.go index 5c56603e..db08b6ee 100644 --- a/thumbnailing/i/01-factories.go +++ b/thumbnailing/preview/01-factories.go @@ -1,4 +1,4 @@ -package i +package preview import ( "io" diff --git a/thumbnailing/i/apng.go b/thumbnailing/preview/apng.go similarity index 98% rename from thumbnailing/i/apng.go rename to thumbnailing/preview/apng.go index e27d4dd2..fc807d36 100644 --- a/thumbnailing/i/apng.go +++ b/thumbnailing/preview/apng.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -13,8 +13,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type apngGenerator struct { -} +type apngGenerator struct{} func (d apngGenerator) supportedContentTypes() []string { return []string{"image/png", "image/apng"} diff --git a/thumbnailing/i/bmp.go b/thumbnailing/preview/bmp.go similarity index 98% rename from thumbnailing/i/bmp.go rename to thumbnailing/preview/bmp.go index 83d8794e..7195e13f 100644 --- a/thumbnailing/i/bmp.go +++ b/thumbnailing/preview/bmp.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/flac.go b/thumbnailing/preview/flac.go similarity index 99% rename from thumbnailing/i/flac.go rename to thumbnailing/preview/flac.go index 8177c464..e7186727 100644 --- a/thumbnailing/i/flac.go +++ b/thumbnailing/preview/flac.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/gif.go b/thumbnailing/preview/gif.go similarity index 98% rename from thumbnailing/i/gif.go rename to thumbnailing/preview/gif.go index a36419f5..32c80a1d 100644 --- a/thumbnailing/i/gif.go +++ b/thumbnailing/preview/gif.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -13,8 +13,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type gifGenerator struct { -} +type gifGenerator struct{} func (d gifGenerator) supportedContentTypes() []string { return []string{"image/gif"} diff --git a/thumbnailing/i/heif.go b/thumbnailing/preview/heif.go similarity index 98% rename from thumbnailing/i/heif.go rename to thumbnailing/preview/heif.go index b57d4701..b5933f31 100644 --- a/thumbnailing/i/heif.go +++ b/thumbnailing/preview/heif.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/jpegxl.go b/thumbnailing/preview/jpegxl.go similarity index 97% rename from thumbnailing/i/jpegxl.go rename to thumbnailing/preview/jpegxl.go index 0ec30c2f..0bfe8e58 100644 --- a/thumbnailing/i/jpegxl.go +++ b/thumbnailing/preview/jpegxl.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -11,8 +11,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) -type jpegxlGenerator struct { -} +type jpegxlGenerator struct{} func (d jpegxlGenerator) supportedContentTypes() []string { return []string{"image/jxl"} diff --git a/thumbnailing/i/jpg.go b/thumbnailing/preview/jpg.go similarity index 99% rename from thumbnailing/i/jpg.go rename to thumbnailing/preview/jpg.go index 05253e4a..762c74f7 100644 --- a/thumbnailing/i/jpg.go +++ b/thumbnailing/preview/jpg.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/mp3.go b/thumbnailing/preview/mp3.go similarity index 99% rename from thumbnailing/i/mp3.go rename to thumbnailing/preview/mp3.go index 65b13580..95c64b67 100644 --- a/thumbnailing/i/mp3.go +++ b/thumbnailing/preview/mp3.go @@ -1,4 +1,4 @@ -package i +package preview import ( "bytes" @@ -22,8 +22,7 @@ import ( "github.com/t2bot/matrix-media-repo/util/readers" ) -type mp3Generator struct { -} +type mp3Generator struct{} func (d mp3Generator) supportedContentTypes() []string { return []string{"audio/mpeg"} diff --git a/thumbnailing/i/mp4.go b/thumbnailing/preview/mp4.go similarity index 99% rename from thumbnailing/i/mp4.go rename to thumbnailing/preview/mp4.go index b0617037..3796b7cf 100644 --- a/thumbnailing/i/mp4.go +++ b/thumbnailing/preview/mp4.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/ogg.go b/thumbnailing/preview/ogg.go similarity index 99% rename from thumbnailing/i/ogg.go rename to thumbnailing/preview/ogg.go index a301c7c0..e8005fd3 100644 --- a/thumbnailing/i/ogg.go +++ b/thumbnailing/preview/ogg.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" diff --git a/thumbnailing/i/png.go b/thumbnailing/preview/png.go similarity index 97% rename from thumbnailing/i/png.go rename to thumbnailing/preview/png.go index f73f2b10..8ce028a8 100644 --- a/thumbnailing/i/png.go +++ b/thumbnailing/preview/png.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -12,8 +12,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type pngGenerator struct { -} +type pngGenerator struct{} func (d pngGenerator) supportedContentTypes() []string { return []string{"image/png"} diff --git a/thumbnailing/i/svg.go b/thumbnailing/preview/svg.go similarity index 97% rename from thumbnailing/i/svg.go rename to thumbnailing/preview/svg.go index 22cfc207..56bf25c2 100644 --- a/thumbnailing/i/svg.go +++ b/thumbnailing/preview/svg.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -11,8 +11,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) -type svgGenerator struct { -} +type svgGenerator struct{} func (d svgGenerator) supportedContentTypes() []string { return []string{"image/svg+xml"} diff --git a/thumbnailing/i/tiff.go b/thumbnailing/preview/tiff.go similarity index 96% rename from thumbnailing/i/tiff.go rename to thumbnailing/preview/tiff.go index d381243b..1e6d3301 100644 --- a/thumbnailing/i/tiff.go +++ b/thumbnailing/preview/tiff.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -9,8 +9,7 @@ import ( "golang.org/x/image/tiff" ) -type tiffGenerator struct { -} +type tiffGenerator struct{} func (d tiffGenerator) supportedContentTypes() []string { return []string{"image/tiff"} diff --git a/thumbnailing/i/wav.go b/thumbnailing/preview/wav.go similarity index 97% rename from thumbnailing/i/wav.go rename to thumbnailing/preview/wav.go index caff42e9..91fddd35 100644 --- a/thumbnailing/i/wav.go +++ b/thumbnailing/preview/wav.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -11,8 +11,7 @@ import ( "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type wavGenerator struct { -} +type wavGenerator struct{} func (d wavGenerator) supportedContentTypes() []string { return []string{"audio/wav"} diff --git a/thumbnailing/i/webp.go b/thumbnailing/preview/webp.go similarity index 96% rename from thumbnailing/i/webp.go rename to thumbnailing/preview/webp.go index 21dec2c8..149d8632 100644 --- a/thumbnailing/i/webp.go +++ b/thumbnailing/preview/webp.go @@ -1,4 +1,4 @@ -package i +package preview import ( "fmt" @@ -9,8 +9,7 @@ import ( "golang.org/x/image/webp" ) -type webpGenerator struct { -} +type webpGenerator struct{} func (d webpGenerator) supportedContentTypes() []string { return []string{"image/webp"} diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index 2d46ed8b..eefa5b51 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -9,8 +9,8 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/i" "github.com/t2bot/matrix-media-repo/thumbnailing/m" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -18,7 +18,7 @@ import ( var ErrUnsupported = errors.New("unsupported thumbnail type") func IsSupported(contentType string) bool { - return slices.Contains(i.GetSupportedContentTypes(), contentType) + return slices.Contains(preview.GetSupportedContentTypes(), contentType) } func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { @@ -32,7 +32,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h return nil, ErrUnsupported } - generator, reconstructed := i.GetGenerator(imgStream, contentType, animated) + generator, reconstructed := preview.GetGenerator(imgStream, contentType, animated) if generator == nil { ctx.Log.Debugf("Unsupported thumbnail type at generator for '%s'", contentType) return nil, ErrUnsupported @@ -63,8 +63,8 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h return generator.GenerateThumbnail(buffered.GetRewoundReader(), contentType, width, height, method, animated, ctx) } -func GetGenerator(imgStream io.Reader, contentType string, animated bool) (i.Generator, io.Reader, error) { - generator, reconstructed := i.GetGenerator(imgStream, contentType, animated) +func GetGenerator(imgStream io.Reader, contentType string, animated bool) (preview.Generator, io.Reader, error) { + generator, reconstructed := preview.GetGenerator(imgStream, contentType, animated) if generator == nil { return nil, reconstructed, ErrUnsupported } From c518070976974bda6ddccbaa06ab9586927bacfe Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sat, 2 Mar 2024 22:55:51 +0100 Subject: [PATCH 09/22] drop model package --- pipelines/_steps/thumbnails/generate.go | 4 ++-- thumbnailing/m/thumbnail.go | 11 ----------- thumbnailing/preview/01-factories.go | 5 ++--- thumbnailing/preview/apng.go | 5 ++--- thumbnailing/preview/bmp.go | 3 +-- thumbnailing/preview/flac.go | 5 ++--- thumbnailing/preview/gif.go | 7 +++---- thumbnailing/preview/heif.go | 3 +-- thumbnailing/preview/jpegxl.go | 3 +-- thumbnailing/preview/jpg.go | 5 ++--- thumbnailing/preview/mp3.go | 13 ++++++------- thumbnailing/preview/mp4.go | 3 +-- thumbnailing/preview/ogg.go | 5 ++--- thumbnailing/preview/png.go | 7 +++---- thumbnailing/preview/svg.go | 3 +-- thumbnailing/preview/tiff.go | 3 +-- thumbnailing/{m/audio_info.go => preview/types.go} | 9 ++++++++- thumbnailing/preview/wav.go | 5 ++--- thumbnailing/preview/webp.go | 3 +-- thumbnailing/thumbnail.go | 3 +-- 20 files changed, 42 insertions(+), 63 deletions(-) delete mode 100644 thumbnailing/m/thumbnail.go rename thumbnailing/{m/audio_info.go => preview/types.go} (54%) diff --git a/pipelines/_steps/thumbnails/generate.go b/pipelines/_steps/thumbnails/generate.go index a25f9d93..27266c91 100644 --- a/pipelines/_steps/thumbnails/generate.go +++ b/pipelines/_steps/thumbnails/generate.go @@ -16,12 +16,12 @@ import ( "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/thumbnailing" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/util" ) type generateResult struct { - i *m.Thumbnail + i *preview.Thumbnail err error } diff --git a/thumbnailing/m/thumbnail.go b/thumbnailing/m/thumbnail.go deleted file mode 100644 index 04017f63..00000000 --- a/thumbnailing/m/thumbnail.go +++ /dev/null @@ -1,11 +0,0 @@ -package m - -import ( - "io" -) - -type Thumbnail struct { - Animated bool - ContentType string - Reader io.ReadCloser -} diff --git a/thumbnailing/preview/01-factories.go b/thumbnailing/preview/01-factories.go index db08b6ee..fbc33278 100644 --- a/thumbnailing/preview/01-factories.go +++ b/thumbnailing/preview/01-factories.go @@ -4,7 +4,6 @@ import ( "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -12,13 +11,13 @@ type Generator interface { supportedContentTypes() []string supportsAnimation() bool matches(img io.Reader, contentType string) bool - GenerateThumbnail(img io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) + GenerateThumbnail(img io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) } type AudioGenerator interface { Generator - GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) + GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) } var generators = make([]Generator, 0) diff --git a/thumbnailing/preview/apng.go b/thumbnailing/preview/apng.go index fc807d36..be62ce3f 100644 --- a/thumbnailing/preview/apng.go +++ b/thumbnailing/preview/apng.go @@ -9,7 +9,6 @@ import ( "github.com/getsentry/sentry-go" "github.com/kettek/apng" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) @@ -35,7 +34,7 @@ func (d apngGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { if !animated { return pngGenerator{}.GenerateThumbnail(b, "image/png", width, height, method, false, ctx) } @@ -99,7 +98,7 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width } }(pw, p) - return &m.Thumbnail{ + return &Thumbnail{ ContentType: "image/png", Animated: true, Reader: pr, diff --git a/thumbnailing/preview/bmp.go b/thumbnailing/preview/bmp.go index 7195e13f..67c679b9 100644 --- a/thumbnailing/preview/bmp.go +++ b/thumbnailing/preview/bmp.go @@ -6,7 +6,6 @@ import ( "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "golang.org/x/image/bmp" ) @@ -32,7 +31,7 @@ func (d bmpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return true, i.Width, i.Height, nil } -func (d bmpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d bmpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := bmp.Decode(b) if err != nil { return nil, fmt.Errorf("bmp: error decoding thumbnail: %w", err) diff --git a/thumbnailing/preview/flac.go b/thumbnailing/preview/flac.go index e7186727..edd6cf08 100644 --- a/thumbnailing/preview/flac.go +++ b/thumbnailing/preview/flac.go @@ -7,7 +7,6 @@ import ( "github.com/faiface/beep" "github.com/faiface/beep/flac" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) @@ -37,7 +36,7 @@ func (d flacGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return false, 0, 0, nil } -func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { tags, rc, err := u.GetID3Tags(r) if err != nil { return nil, fmt.Errorf("flac: error getting tags: %w", err) @@ -55,7 +54,7 @@ func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d flacGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d flacGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/preview/gif.go b/thumbnailing/preview/gif.go index 32c80a1d..97e1ee93 100644 --- a/thumbnailing/preview/gif.go +++ b/thumbnailing/preview/gif.go @@ -9,7 +9,6 @@ import ( "math" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) @@ -31,7 +30,7 @@ func (d gifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return pngGenerator{}.GetOriginDimensions(b, contentType, ctx) } -func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { g, err := gif.DecodeAll(b) if err != nil { return nil, fmt.Errorf("gif: error decoding image: %w", err) @@ -84,7 +83,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i _ = pw.Close() } }(pw, targetImg) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, @@ -119,7 +118,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i } }(pw, g) - return &m.Thumbnail{ + return &Thumbnail{ ContentType: "image/gif", Animated: true, Reader: pr, diff --git a/thumbnailing/preview/heif.go b/thumbnailing/preview/heif.go index b5933f31..7a6f3112 100644 --- a/thumbnailing/preview/heif.go +++ b/thumbnailing/preview/heif.go @@ -8,7 +8,6 @@ import ( _ "github.com/strukturag/libheif/go/heif" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) type heifGenerator struct{} @@ -33,7 +32,7 @@ func (d heifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, cfg.Width, cfg.Height, nil } -func (d heifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d heifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, _, err := image.Decode(b) if err != nil { return nil, fmt.Errorf("heif: error decoding thumbnail: %w", err) diff --git a/thumbnailing/preview/jpegxl.go b/thumbnailing/preview/jpegxl.go index 0bfe8e58..928b8b77 100644 --- a/thumbnailing/preview/jpegxl.go +++ b/thumbnailing/preview/jpegxl.go @@ -8,7 +8,6 @@ import ( "path" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) type jpegxlGenerator struct{} @@ -29,7 +28,7 @@ func (d jpegxlGenerator) GetOriginDimensions(b io.Reader, contentType string, ct return false, 0, 0, nil } -func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-jpegxl") if err != nil { return nil, fmt.Errorf("jpegxl: error creating temporary directory: %w", err) diff --git a/thumbnailing/preview/jpg.go b/thumbnailing/preview/jpg.go index 762c74f7..5baa38af 100644 --- a/thumbnailing/preview/jpg.go +++ b/thumbnailing/preview/jpg.go @@ -9,7 +9,6 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -32,7 +31,7 @@ func (d jpgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return pngGenerator{}.GetOriginDimensions(b, contentType, ctx) } -func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { br := readers.NewBufferReadsReader(b) orientation := u.ExtractExifOrientation(br) b = br.GetRewoundReader() @@ -59,7 +58,7 @@ func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i } }(pw, thumb) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/jpeg", Reader: pr, diff --git a/thumbnailing/preview/mp3.go b/thumbnailing/preview/mp3.go index 95c64b67..3798600e 100644 --- a/thumbnailing/preview/mp3.go +++ b/thumbnailing/preview/mp3.go @@ -17,7 +17,6 @@ import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -48,7 +47,7 @@ func (d mp3Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { return nil, fmt.Errorf("mp3: error getting tags: %w", err) @@ -66,7 +65,7 @@ func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width i return d.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err @@ -77,14 +76,14 @@ func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestC return d.GetDataFromStream(audio, format, nKeys) } -func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*m.AudioInfo, error) { +func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*AudioInfo, error) { totalSamples := audio.Len() downsampled, err := u.FastSampleAudio(audio, nKeys) if err != nil { return nil, err } - return &m.AudioInfo{ + return &AudioInfo{ Duration: format.SampleRate.D(totalSamples), Channels: format.NumChannels, TotalSamples: totalSamples, @@ -92,7 +91,7 @@ func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep }, nil } -func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format beep.Format, meta tag.Metadata, width int, height int, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format beep.Format, meta tag.Metadata, width int, height int, ctx rcontext.RequestContext) (*Thumbnail, error) { bgColor := color.RGBA{A: 255, R: 41, G: 57, B: 92} fgColor := color.RGBA{A: 255, R: 240, G: 240, B: 240} @@ -205,7 +204,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee } }(pw, img) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, diff --git a/thumbnailing/preview/mp4.go b/thumbnailing/preview/mp4.go index 3796b7cf..6d13a32f 100644 --- a/thumbnailing/preview/mp4.go +++ b/thumbnailing/preview/mp4.go @@ -9,7 +9,6 @@ import ( "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) type mp4Generator struct{} @@ -30,7 +29,7 @@ func (d mp4Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-mp4") if err != nil { return nil, fmt.Errorf("mp4: error creating temporary directory: %w", err) diff --git a/thumbnailing/preview/ogg.go b/thumbnailing/preview/ogg.go index e8005fd3..9d144046 100644 --- a/thumbnailing/preview/ogg.go +++ b/thumbnailing/preview/ogg.go @@ -7,7 +7,6 @@ import ( "github.com/faiface/beep" "github.com/faiface/beep/vorbis" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -38,7 +37,7 @@ func (d oggGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { return nil, fmt.Errorf("ogg: error getting tags: %v", err) @@ -56,7 +55,7 @@ func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d oggGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d oggGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/preview/png.go b/thumbnailing/preview/png.go index 8ce028a8..b9ba0a93 100644 --- a/thumbnailing/preview/png.go +++ b/thumbnailing/preview/png.go @@ -8,7 +8,6 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) @@ -34,7 +33,7 @@ func (d pngGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return true, i.Width, i.Height, nil } -func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := imaging.Decode(b) if err != nil { return nil, fmt.Errorf("png: error decoding thumbnail: %w", err) @@ -43,7 +42,7 @@ func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return d.GenerateThumbnailOf(src, width, height, method, ctx) } -func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int, method string, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int, method string, ctx rcontext.RequestContext) (*Thumbnail, error) { thumb, err := u.MakeThumbnail(src, method, width, height) if err != nil || thumb == nil { return nil, err @@ -59,7 +58,7 @@ func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int } }(pw, thumb) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, diff --git a/thumbnailing/preview/svg.go b/thumbnailing/preview/svg.go index 56bf25c2..1d3aff16 100644 --- a/thumbnailing/preview/svg.go +++ b/thumbnailing/preview/svg.go @@ -8,7 +8,6 @@ import ( "path" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) type svgGenerator struct{} @@ -29,7 +28,7 @@ func (d svgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-svg") if err != nil { return nil, fmt.Errorf("svg: error creating temporary directory: %w", err) diff --git a/thumbnailing/preview/tiff.go b/thumbnailing/preview/tiff.go index 1e6d3301..d77e02fa 100644 --- a/thumbnailing/preview/tiff.go +++ b/thumbnailing/preview/tiff.go @@ -5,7 +5,6 @@ import ( "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "golang.org/x/image/tiff" ) @@ -31,7 +30,7 @@ func (d tiffGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d tiffGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d tiffGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := tiff.Decode(b) if err != nil { return nil, fmt.Errorf("tiff: error decoding thumbnail: %w", err) diff --git a/thumbnailing/m/audio_info.go b/thumbnailing/preview/types.go similarity index 54% rename from thumbnailing/m/audio_info.go rename to thumbnailing/preview/types.go index a194cd4d..8f11bda8 100644 --- a/thumbnailing/m/audio_info.go +++ b/thumbnailing/preview/types.go @@ -1,6 +1,7 @@ -package m +package preview import ( + "io" "time" ) @@ -10,3 +11,9 @@ type AudioInfo struct { TotalSamples int Channels int } + +type Thumbnail struct { + Animated bool + ContentType string + Reader io.ReadCloser +} diff --git a/thumbnailing/preview/wav.go b/thumbnailing/preview/wav.go index 91fddd35..15c34739 100644 --- a/thumbnailing/preview/wav.go +++ b/thumbnailing/preview/wav.go @@ -7,7 +7,6 @@ import ( "github.com/faiface/beep" "github.com/faiface/beep/wav" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) @@ -37,7 +36,7 @@ func (d wavGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { tags, rc, err := u.GetID3Tags(b) if err != nil { return nil, fmt.Errorf("wav: error getting tags: %w", err) @@ -55,7 +54,7 @@ func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d wavGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d wavGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/preview/webp.go b/thumbnailing/preview/webp.go index 149d8632..14f9d22d 100644 --- a/thumbnailing/preview/webp.go +++ b/thumbnailing/preview/webp.go @@ -5,7 +5,6 @@ import ( "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "golang.org/x/image/webp" ) @@ -31,7 +30,7 @@ func (d webpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d webpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d webpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := webp.Decode(b) if err != nil { return nil, fmt.Errorf("webp: error decoding thumbnail: %w", err) diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index eefa5b51..82118e6e 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -9,7 +9,6 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" @@ -21,7 +20,7 @@ func IsSupported(contentType string) bool { return slices.Contains(preview.GetSupportedContentTypes(), contentType) } -func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*preview.Thumbnail, error) { defer imgStream.Close() if !IsSupported(contentType) { ctx.Log.Debugf("Unsupported content type '%s'", contentType) From cc20a34d7932873b5f76a51572c30cb99378c9d6 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sat, 2 Mar 2024 23:17:01 +0100 Subject: [PATCH 10/22] minor preview metadata backend refactoring --- thumbnailing/u/dimensions.go | 6 +----- thumbnailing/u/framing.go | 13 ++++++------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/thumbnailing/u/dimensions.go b/thumbnailing/u/dimensions.go index 3b83c4e2..2b19a40a 100644 --- a/thumbnailing/u/dimensions.go +++ b/thumbnailing/u/dimensions.go @@ -9,11 +9,7 @@ func AdjustProperties(srcWidth int, srcHeight int, desiredWidth int, desiredHeig } if srcWidth <= desiredWidth && srcHeight <= desiredHeight { - if wantAnimated { - return true, srcWidth, srcHeight, method - } else { - return false, desiredWidth, desiredHeight, method - } + return wantAnimated, srcWidth, srcHeight, method } return true, desiredWidth, desiredHeight, method } diff --git a/thumbnailing/u/framing.go b/thumbnailing/u/framing.go index 4ffcb7d8..18b11127 100644 --- a/thumbnailing/u/framing.go +++ b/thumbnailing/u/framing.go @@ -11,15 +11,14 @@ import ( ) func MakeThumbnail(src image.Image, method string, width int, height int) (image.Image, error) { - var result image.Image - if method == "scale" { - result = imaging.Fit(src, width, height, imaging.Linear) - } else if method == "crop" { - result = imaging.Fill(src, width, height, imaging.Center, imaging.Linear) - } else { + switch method { + case "scale": + return imaging.Fit(src, width, height, imaging.Linear), nil + case "crop": + return imaging.Fill(src, width, height, imaging.Center, imaging.Linear), nil + default: return nil, errors.New("unrecognized method: " + method) } - return result, nil } func ExtractExifOrientation(r io.Reader) *ExifOrientation { From f81662d490df6e3c648d86ee91812a755b4ab655 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 00:03:04 +0100 Subject: [PATCH 11/22] inline GetID3Tags --- thumbnailing/preview/mp3.go | 11 ++++--- thumbnailing/preview/readSeeker.go | 42 ++++++++++++++++++++++++ thumbnailing/u/metadata.go | 52 ------------------------------ 3 files changed, 49 insertions(+), 56 deletions(-) create mode 100644 thumbnailing/preview/readSeeker.go delete mode 100644 thumbnailing/u/metadata.go diff --git a/thumbnailing/preview/mp3.go b/thumbnailing/preview/mp3.go index 3798600e..fc9e0cdd 100644 --- a/thumbnailing/preview/mp3.go +++ b/thumbnailing/preview/mp3.go @@ -48,14 +48,17 @@ func (d mp3Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r } func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) + rd, err := newReadSeekerWrapper(b) + tags, err := u.GetID3Tags(rd) + if err != nil { + return nil, fmt.Errorf("error wrapping reader: %w", err) + } + tags, err = tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { return nil, fmt.Errorf("mp3: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } diff --git a/thumbnailing/preview/readSeeker.go b/thumbnailing/preview/readSeeker.go new file mode 100644 index 00000000..d3244170 --- /dev/null +++ b/thumbnailing/preview/readSeeker.go @@ -0,0 +1,42 @@ +package preview + +import "io" + +type readSeekerWrapper struct { + data []byte + offset int64 +} + +func (r *readSeekerWrapper) Read(p []byte) (n int, err error) { + if r.offset >= int64(len(r.data)) { + return 0, io.EOF + } + n = copy(p, r.data[r.offset:]) + r.offset += int64(n) + return +} + +func (r *readSeekerWrapper) Seek(offset int64, whence int) (int64, error) { + var absOffset int64 + switch whence { + case io.SeekStart: + absOffset = offset + case io.SeekCurrent: + absOffset = r.offset + offset + case io.SeekEnd: + absOffset = int64(len(r.data)) + offset + } + if absOffset < 0 || absOffset > int64(len(r.data)) { + return 0, io.EOF + } + r.offset = absOffset + return absOffset, nil +} + +func newReadSeekerWrapper(reader io.Reader) (*readSeekerWrapper, error) { + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return &readSeekerWrapper{data: data}, nil +} diff --git a/thumbnailing/u/metadata.go b/thumbnailing/u/metadata.go deleted file mode 100644 index 6a2eedb9..00000000 --- a/thumbnailing/u/metadata.go +++ /dev/null @@ -1,52 +0,0 @@ -package u - -import ( - "fmt" - "io" - "os" - - "github.com/dhowden/tag" - "github.com/getsentry/sentry-go" - "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/util/readers" -) - -func GetID3Tags(b io.Reader) (tag.Metadata, io.ReadSeekCloser, error) { - var f *os.File - var err error - - tryCleanup := func() { - if f != nil { - if err = os.Remove(f.Name()); err != nil && !os.IsNotExist(err) { - logrus.Warnf("Error deleting temp file '%s': %v", f.Name(), err) - sentry.CaptureException(fmt.Errorf("id3: error deleting temp file: %v", err)) - } - } - } - - f, err = os.CreateTemp(os.TempDir(), "mmr-id3") - if err != nil { - tryCleanup() - return nil, nil, err - } - if _, err = io.Copy(f, b); err != nil { - tryCleanup() - return nil, nil, err - } - if err = f.Close(); err != nil { - tryCleanup() - return nil, nil, err - } - if f, err = os.OpenFile(f.Name(), os.O_WRONLY, 0644); err != nil { - tryCleanup() - return nil, nil, err - } - - meta, _ := tag.ReadFrom(f) // we don't care about errors in this process - if _, err = f.Seek(0, io.SeekStart); err != nil { - tryCleanup() - return nil, nil, err - } - - return meta, readers.NewTempFileCloser("", f.Name(), f), nil -} From 682b41f0aed37074cfff914faa1531e6153b2628 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 00:13:42 +0100 Subject: [PATCH 12/22] move thumbnailing/u to thumbnailing/preview/metadata --- thumbnailing/preview/apng.go | 4 ++-- thumbnailing/preview/flac.go | 12 +++++++----- thumbnailing/preview/gif.go | 6 +++--- thumbnailing/preview/jpg.go | 10 +++++----- .../{u => preview/metadata}/dimensions.go | 2 +- thumbnailing/{u => preview/metadata}/encode.go | 2 +- thumbnailing/{u => preview/metadata}/exif.go | 2 +- thumbnailing/{u => preview/metadata}/framing.go | 2 +- thumbnailing/{u => preview/metadata}/sample.go | 2 +- thumbnailing/preview/mp3.go | 16 ++++++---------- thumbnailing/preview/ogg.go | 9 ++++----- thumbnailing/preview/png.go | 6 +++--- thumbnailing/preview/wav.go | 9 ++++----- thumbnailing/thumbnail.go | 4 ++-- 14 files changed, 41 insertions(+), 45 deletions(-) rename thumbnailing/{u => preview/metadata}/dimensions.go (97%) rename thumbnailing/{u => preview/metadata}/encode.go (97%) rename thumbnailing/{u => preview/metadata}/exif.go (99%) rename thumbnailing/{u => preview/metadata}/framing.go (98%) rename thumbnailing/{u => preview/metadata}/sample.go (98%) diff --git a/thumbnailing/preview/apng.go b/thumbnailing/preview/apng.go index be62ce3f..ea27095c 100644 --- a/thumbnailing/preview/apng.go +++ b/thumbnailing/preview/apng.go @@ -9,7 +9,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/kettek/apng" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) type apngGenerator struct{} @@ -68,7 +68,7 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width draw.Draw(frameImg, image.Rect(frame.XOffset, frame.YOffset, frameImg.Rect.Max.X, frameImg.Rect.Max.Y), img, image.Point{X: 0, Y: 0}, draw.Src) // Do the thumbnailing on the copied frame - frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) + frameThumb, err := metadata.MakeThumbnail(frameImg, method, width, height) if err != nil { return nil, fmt.Errorf("apng: error generating thumbnail frame: %w", err) } diff --git a/thumbnailing/preview/flac.go b/thumbnailing/preview/flac.go index edd6cf08..a62254c0 100644 --- a/thumbnailing/preview/flac.go +++ b/thumbnailing/preview/flac.go @@ -4,10 +4,10 @@ import ( "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/flac" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) type flacGenerator struct{} @@ -37,14 +37,16 @@ func (d flacGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx } func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { - tags, rc, err := u.GetID3Tags(r) + rd, err := newReadSeekerWrapper(r) + if err != nil { + return nil, fmt.Errorf("error wrapping reader: %w", err) + } + tags, err := tag.ReadFrom(rd) if err != nil { return nil, fmt.Errorf("flac: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } diff --git a/thumbnailing/preview/gif.go b/thumbnailing/preview/gif.go index 97e1ee93..b9c92fba 100644 --- a/thumbnailing/preview/gif.go +++ b/thumbnailing/preview/gif.go @@ -9,7 +9,7 @@ import ( "math" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) type gifGenerator struct{} @@ -54,7 +54,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i draw.Draw(frameImg, frameImg.Bounds(), img, image.Point{X: 0, Y: 0}, draw.Over) // Do the thumbnailing on the copied frame - frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) + frameThumb, err := metadata.MakeThumbnail(frameImg, method, width, height) if err != nil { return nil, fmt.Errorf("gif: error generating thumbnail frame: %w", err) } @@ -76,7 +76,7 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i // The thumbnailer decided that it shouldn't thumbnail, so encode it ourselves pr, pw := io.Pipe() go func(pw *io.PipeWriter, p *image.Paletted) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { _ = pw.CloseWithError(fmt.Errorf("gif: error encoding still frame thumbnail: %w", err)) } else { diff --git a/thumbnailing/preview/jpg.go b/thumbnailing/preview/jpg.go index 5baa38af..5f4bc113 100644 --- a/thumbnailing/preview/jpg.go +++ b/thumbnailing/preview/jpg.go @@ -9,7 +9,7 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -33,7 +33,7 @@ func (d jpgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { br := readers.NewBufferReadsReader(b) - orientation := u.ExtractExifOrientation(br) + orientation := metadata.ExtractExifOrientation(br) b = br.GetRewoundReader() src, err := imaging.Decode(b) @@ -41,16 +41,16 @@ func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return nil, fmt.Errorf("jpg: error decoding thumbnail: %w", err) } - thumb, err := u.MakeThumbnail(src, method, width, height) + thumb, err := metadata.MakeThumbnail(src, method, width, height) if err != nil { return nil, fmt.Errorf("jpg: error making thumbnail: %w", err) } - thumb = u.ApplyOrientation(thumb, orientation) + thumb = metadata.ApplyOrientation(thumb, orientation) pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p, u.JpegSource) + err = metadata.Encode(ctx, pw, p, metadata.JpegSource) if err != nil { _ = pw.CloseWithError(fmt.Errorf("jpg: error encoding thumbnail: %w", err)) } else { diff --git a/thumbnailing/u/dimensions.go b/thumbnailing/preview/metadata/dimensions.go similarity index 97% rename from thumbnailing/u/dimensions.go rename to thumbnailing/preview/metadata/dimensions.go index 2b19a40a..dd8a06a3 100644 --- a/thumbnailing/u/dimensions.go +++ b/thumbnailing/preview/metadata/dimensions.go @@ -1,4 +1,4 @@ -package u +package metadata func AdjustProperties(srcWidth int, srcHeight int, desiredWidth int, desiredHeight int, wantAnimated bool, method string) (bool, int, int, string) { aspectRatio := float32(srcHeight) / float32(srcWidth) diff --git a/thumbnailing/u/encode.go b/thumbnailing/preview/metadata/encode.go similarity index 97% rename from thumbnailing/u/encode.go rename to thumbnailing/preview/metadata/encode.go index 9effd8e3..bb339226 100644 --- a/thumbnailing/u/encode.go +++ b/thumbnailing/preview/metadata/encode.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "image" diff --git a/thumbnailing/u/exif.go b/thumbnailing/preview/metadata/exif.go similarity index 99% rename from thumbnailing/u/exif.go rename to thumbnailing/preview/metadata/exif.go index 5123911c..65756264 100644 --- a/thumbnailing/u/exif.go +++ b/thumbnailing/preview/metadata/exif.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "errors" diff --git a/thumbnailing/u/framing.go b/thumbnailing/preview/metadata/framing.go similarity index 98% rename from thumbnailing/u/framing.go rename to thumbnailing/preview/metadata/framing.go index 18b11127..43ba044c 100644 --- a/thumbnailing/u/framing.go +++ b/thumbnailing/preview/metadata/framing.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "errors" diff --git a/thumbnailing/u/sample.go b/thumbnailing/preview/metadata/sample.go similarity index 98% rename from thumbnailing/u/sample.go rename to thumbnailing/preview/metadata/sample.go index c6854575..75fcc449 100644 --- a/thumbnailing/u/sample.go +++ b/thumbnailing/preview/metadata/sample.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "errors" diff --git a/thumbnailing/preview/mp3.go b/thumbnailing/preview/mp3.go index fc9e0cdd..b1ed1dd6 100644 --- a/thumbnailing/preview/mp3.go +++ b/thumbnailing/preview/mp3.go @@ -17,7 +17,7 @@ import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -49,11 +49,7 @@ func (d mp3Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { rd, err := newReadSeekerWrapper(b) - tags, err := u.GetID3Tags(rd) - if err != nil { - return nil, fmt.Errorf("error wrapping reader: %w", err) - } - tags, err = tag.ReadFrom(rd) // we don't care about errors in this process + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { return nil, fmt.Errorf("mp3: error getting tags: %w", err) } @@ -81,7 +77,7 @@ func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestC func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*AudioInfo, error) { totalSamples := audio.Len() - downsampled, err := u.FastSampleAudio(audio, nKeys) + downsampled, err := metadata.FastSampleAudio(audio, nKeys) if err != nil { return nil, err } @@ -106,7 +102,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee if meta != nil && meta.Picture() != nil { artwork, _, _ := image.Decode(bytes.NewBuffer(meta.Picture().Data)) if artwork != nil { - artworkImg, _ = u.MakeThumbnail(artwork, "crop", sq, sq) + artworkImg, _ = metadata.MakeThumbnail(artwork, "crop", sq, sq) } } @@ -130,7 +126,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee defer f.Close() tmp, _, _ := image.Decode(f) if tmp != nil { - artworkImg, _ = u.MakeThumbnail(tmp, "crop", ax, ay) + artworkImg, _ = metadata.MakeThumbnail(tmp, "crop", ax, ay) } } if artworkImg == nil { @@ -199,7 +195,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee // Encode to a png pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { _ = pw.CloseWithError(fmt.Errorf("beep-visual: error encoding thumbnail: %w", err)) } else { diff --git a/thumbnailing/preview/ogg.go b/thumbnailing/preview/ogg.go index 9d144046..b8b22b73 100644 --- a/thumbnailing/preview/ogg.go +++ b/thumbnailing/preview/ogg.go @@ -4,10 +4,10 @@ import ( "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/vorbis" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -38,14 +38,13 @@ func (d oggGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r } func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) + rd, err := newReadSeekerWrapper(b) + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { return nil, fmt.Errorf("ogg: error getting tags: %v", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } diff --git a/thumbnailing/preview/png.go b/thumbnailing/preview/png.go index b9ba0a93..9835afc7 100644 --- a/thumbnailing/preview/png.go +++ b/thumbnailing/preview/png.go @@ -8,7 +8,7 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) type pngGenerator struct{} @@ -43,14 +43,14 @@ func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width i } func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int, method string, ctx rcontext.RequestContext) (*Thumbnail, error) { - thumb, err := u.MakeThumbnail(src, method, width, height) + thumb, err := metadata.MakeThumbnail(src, method, width, height) if err != nil || thumb == nil { return nil, err } pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { _ = pw.CloseWithError(fmt.Errorf("png: error encoding thumbnail: %w", err)) } else { diff --git a/thumbnailing/preview/wav.go b/thumbnailing/preview/wav.go index 15c34739..5f1cdff8 100644 --- a/thumbnailing/preview/wav.go +++ b/thumbnailing/preview/wav.go @@ -4,10 +4,10 @@ import ( "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/wav" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) type wavGenerator struct{} @@ -37,14 +37,13 @@ func (d wavGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r } func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) + rd, err := newReadSeekerWrapper(b) + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { return nil, fmt.Errorf("wav: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index 82118e6e..15639236 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -10,7 +10,7 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/thumbnailing/preview" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -53,7 +53,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h // While we're here, check to ensure we're not about to produce a thumbnail which is larger than the source material shouldThumbnail := true - shouldThumbnail, width, height, method = u.AdjustProperties(w, h, width, height, animated, method) + shouldThumbnail, width, height, method = metadata.AdjustProperties(w, h, width, height, animated, method) if !shouldThumbnail { return nil, common.ErrMediaDimensionsTooSmall } From edd19b2d6b27bad1f06e22db32f5ed091762eef6 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 14:48:57 +0100 Subject: [PATCH 13/22] move pipelines/_steps to pipelines/steps --- api/r0/public_config.go | 2 +- api/unstable/public_usage.go | 2 +- pipelines/pipeline_create/pipeline.go | 4 ++-- pipelines/pipeline_download/pipeline.go | 8 ++++---- pipelines/pipeline_preview/pipeline.go | 2 +- pipelines/pipeline_thumbnail/pipeline.go | 6 +++--- pipelines/pipeline_upload/pipeline.go | 6 +++--- .../datastore_op/put_and_return_stream.go | 0 pipelines/{_steps => steps}/download/open_stream.go | 0 pipelines/{_steps => steps}/download/try_download.go | 2 +- pipelines/{_steps => steps}/download/wait.go | 0 pipelines/{_steps => steps}/meta/flag_access.go | 0 pipelines/{_steps => steps}/quarantine/logic.go | 0 pipelines/{_steps => steps}/quarantine/thumbnail.go | 0 pipelines/{_steps => steps}/quota/check.go | 0 pipelines/{_steps => steps}/thumbnails/generate.go | 4 ++-- pipelines/{_steps => steps}/thumbnails/pick_dimensions.go | 0 pipelines/{_steps => steps}/upload/deduplicate.go | 0 pipelines/{_steps => steps}/upload/generate_media_id.go | 0 pipelines/{_steps => steps}/upload/limit.go | 0 pipelines/{_steps => steps}/upload/lock.go | 0 pipelines/{_steps => steps}/upload/quarantine.go | 0 pipelines/{_steps => steps}/upload/redis_async.go | 0 pipelines/{_steps => steps}/upload/spam.go | 0 pipelines/{_steps => steps}/url_preview/preview.go | 0 pipelines/{_steps => steps}/url_preview/process.go | 0 pipelines/{_steps => steps}/url_preview/upload_image.go | 0 27 files changed, 18 insertions(+), 18 deletions(-) rename pipelines/{_steps => steps}/datastore_op/put_and_return_stream.go (100%) rename pipelines/{_steps => steps}/download/open_stream.go (100%) rename pipelines/{_steps => steps}/download/try_download.go (97%) rename pipelines/{_steps => steps}/download/wait.go (100%) rename pipelines/{_steps => steps}/meta/flag_access.go (100%) rename pipelines/{_steps => steps}/quarantine/logic.go (100%) rename pipelines/{_steps => steps}/quarantine/thumbnail.go (100%) rename pipelines/{_steps => steps}/quota/check.go (100%) rename pipelines/{_steps => steps}/thumbnails/generate.go (97%) rename pipelines/{_steps => steps}/thumbnails/pick_dimensions.go (100%) rename pipelines/{_steps => steps}/upload/deduplicate.go (100%) rename pipelines/{_steps => steps}/upload/generate_media_id.go (100%) rename pipelines/{_steps => steps}/upload/limit.go (100%) rename pipelines/{_steps => steps}/upload/lock.go (100%) rename pipelines/{_steps => steps}/upload/quarantine.go (100%) rename pipelines/{_steps => steps}/upload/redis_async.go (100%) rename pipelines/{_steps => steps}/upload/spam.go (100%) rename pipelines/{_steps => steps}/url_preview/preview.go (100%) rename pipelines/{_steps => steps}/url_preview/process.go (100%) rename pipelines/{_steps => steps}/url_preview/upload_image.go (100%) diff --git a/api/r0/public_config.go b/api/r0/public_config.go index 55715565..695c17db 100644 --- a/api/r0/public_config.go +++ b/api/r0/public_config.go @@ -6,7 +6,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) type PublicConfigResponse struct { diff --git a/api/unstable/public_usage.go b/api/unstable/public_usage.go index 7a6e2a96..a5572ad6 100644 --- a/api/unstable/public_usage.go +++ b/api/unstable/public_usage.go @@ -6,7 +6,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) type PublicUsageResponse struct { diff --git a/pipelines/pipeline_create/pipeline.go b/pipelines/pipeline_create/pipeline.go index 4946766b..daf12ae2 100644 --- a/pipelines/pipeline_create/pipeline.go +++ b/pipelines/pipeline_create/pipeline.go @@ -5,8 +5,8 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/upload" ) const DefaultExpirationTime = 0 diff --git a/pipelines/pipeline_download/pipeline.go b/pipelines/pipeline_download/pipeline.go index a20eaf23..b8f5c396 100644 --- a/pipelines/pipeline_download/pipeline.go +++ b/pipelines/pipeline_download/pipeline.go @@ -8,13 +8,13 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/go-singleflight-streams" + sfstreams "github.com/t2bot/go-singleflight-streams" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/meta" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quarantine" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/meta" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quarantine" "github.com/t2bot/matrix-media-repo/util/readers" "github.com/t2bot/matrix-media-repo/util/sfcache" ) diff --git a/pipelines/pipeline_preview/pipeline.go b/pipelines/pipeline_preview/pipeline.go index 2de237f7..027494de 100644 --- a/pipelines/pipeline_preview/pipeline.go +++ b/pipelines/pipeline_preview/pipeline.go @@ -9,7 +9,7 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/url_preview" + "github.com/t2bot/matrix-media-repo/pipelines/steps/url_preview" "github.com/t2bot/matrix-media-repo/url_previewing/m" "golang.org/x/sync/singleflight" ) diff --git a/pipelines/pipeline_thumbnail/pipeline.go b/pipelines/pipeline_thumbnail/pipeline.go index e3b19204..7ddc8110 100644 --- a/pipelines/pipeline_thumbnail/pipeline.go +++ b/pipelines/pipeline_thumbnail/pipeline.go @@ -11,10 +11,10 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quarantine" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/thumbnails" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quarantine" + "github.com/t2bot/matrix-media-repo/pipelines/steps/thumbnails" "github.com/t2bot/matrix-media-repo/util/readers" "github.com/t2bot/matrix-media-repo/util/sfcache" ) diff --git a/pipelines/pipeline_upload/pipeline.go b/pipelines/pipeline_upload/pipeline.go index 82e7620e..cbf9e4c5 100644 --- a/pipelines/pipeline_upload/pipeline.go +++ b/pipelines/pipeline_upload/pipeline.go @@ -12,9 +12,9 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/notifier" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/meta" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" + "github.com/t2bot/matrix-media-repo/pipelines/steps/meta" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/upload" "github.com/t2bot/matrix-media-repo/util/readers" ) diff --git a/pipelines/_steps/datastore_op/put_and_return_stream.go b/pipelines/steps/datastore_op/put_and_return_stream.go similarity index 100% rename from pipelines/_steps/datastore_op/put_and_return_stream.go rename to pipelines/steps/datastore_op/put_and_return_stream.go diff --git a/pipelines/_steps/download/open_stream.go b/pipelines/steps/download/open_stream.go similarity index 100% rename from pipelines/_steps/download/open_stream.go rename to pipelines/steps/download/open_stream.go diff --git a/pipelines/_steps/download/try_download.go b/pipelines/steps/download/try_download.go similarity index 97% rename from pipelines/_steps/download/try_download.go rename to pipelines/steps/download/try_download.go index bda7aa62..ddef63f2 100644 --- a/pipelines/_steps/download/try_download.go +++ b/pipelines/steps/download/try_download.go @@ -17,7 +17,7 @@ import ( "github.com/t2bot/matrix-media-repo/errcache" "github.com/t2bot/matrix-media-repo/matrix" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/datastore_op" + "github.com/t2bot/matrix-media-repo/pipelines/steps/datastore_op" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/readers" diff --git a/pipelines/_steps/download/wait.go b/pipelines/steps/download/wait.go similarity index 100% rename from pipelines/_steps/download/wait.go rename to pipelines/steps/download/wait.go diff --git a/pipelines/_steps/meta/flag_access.go b/pipelines/steps/meta/flag_access.go similarity index 100% rename from pipelines/_steps/meta/flag_access.go rename to pipelines/steps/meta/flag_access.go diff --git a/pipelines/_steps/quarantine/logic.go b/pipelines/steps/quarantine/logic.go similarity index 100% rename from pipelines/_steps/quarantine/logic.go rename to pipelines/steps/quarantine/logic.go diff --git a/pipelines/_steps/quarantine/thumbnail.go b/pipelines/steps/quarantine/thumbnail.go similarity index 100% rename from pipelines/_steps/quarantine/thumbnail.go rename to pipelines/steps/quarantine/thumbnail.go diff --git a/pipelines/_steps/quota/check.go b/pipelines/steps/quota/check.go similarity index 100% rename from pipelines/_steps/quota/check.go rename to pipelines/steps/quota/check.go diff --git a/pipelines/_steps/thumbnails/generate.go b/pipelines/steps/thumbnails/generate.go similarity index 97% rename from pipelines/_steps/thumbnails/generate.go rename to pipelines/steps/thumbnails/generate.go index 27266c91..da36977a 100644 --- a/pipelines/_steps/thumbnails/generate.go +++ b/pipelines/steps/thumbnails/generate.go @@ -12,8 +12,8 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/datastore_op" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/datastore_op" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/thumbnailing" "github.com/t2bot/matrix-media-repo/thumbnailing/preview" diff --git a/pipelines/_steps/thumbnails/pick_dimensions.go b/pipelines/steps/thumbnails/pick_dimensions.go similarity index 100% rename from pipelines/_steps/thumbnails/pick_dimensions.go rename to pipelines/steps/thumbnails/pick_dimensions.go diff --git a/pipelines/_steps/upload/deduplicate.go b/pipelines/steps/upload/deduplicate.go similarity index 100% rename from pipelines/_steps/upload/deduplicate.go rename to pipelines/steps/upload/deduplicate.go diff --git a/pipelines/_steps/upload/generate_media_id.go b/pipelines/steps/upload/generate_media_id.go similarity index 100% rename from pipelines/_steps/upload/generate_media_id.go rename to pipelines/steps/upload/generate_media_id.go diff --git a/pipelines/_steps/upload/limit.go b/pipelines/steps/upload/limit.go similarity index 100% rename from pipelines/_steps/upload/limit.go rename to pipelines/steps/upload/limit.go diff --git a/pipelines/_steps/upload/lock.go b/pipelines/steps/upload/lock.go similarity index 100% rename from pipelines/_steps/upload/lock.go rename to pipelines/steps/upload/lock.go diff --git a/pipelines/_steps/upload/quarantine.go b/pipelines/steps/upload/quarantine.go similarity index 100% rename from pipelines/_steps/upload/quarantine.go rename to pipelines/steps/upload/quarantine.go diff --git a/pipelines/_steps/upload/redis_async.go b/pipelines/steps/upload/redis_async.go similarity index 100% rename from pipelines/_steps/upload/redis_async.go rename to pipelines/steps/upload/redis_async.go diff --git a/pipelines/_steps/upload/spam.go b/pipelines/steps/upload/spam.go similarity index 100% rename from pipelines/_steps/upload/spam.go rename to pipelines/steps/upload/spam.go diff --git a/pipelines/_steps/url_preview/preview.go b/pipelines/steps/url_preview/preview.go similarity index 100% rename from pipelines/_steps/url_preview/preview.go rename to pipelines/steps/url_preview/preview.go diff --git a/pipelines/_steps/url_preview/process.go b/pipelines/steps/url_preview/process.go similarity index 100% rename from pipelines/_steps/url_preview/process.go rename to pipelines/steps/url_preview/process.go diff --git a/pipelines/_steps/url_preview/upload_image.go b/pipelines/steps/url_preview/upload_image.go similarity index 100% rename from pipelines/_steps/url_preview/upload_image.go rename to pipelines/steps/url_preview/upload_image.go From ca68585eb4ddd876c606fadbd9885221d4bace49 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 14:55:37 +0100 Subject: [PATCH 14/22] move cmd/utilities/_common to cmd/utilities/common --- cmd/utilities/combine_signing_keys/main.go | 4 ++-- cmd/utilities/{_common => common}/signing_key_export.go | 2 +- cmd/utilities/generate_signing_key/main.go | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) rename cmd/utilities/{_common => common}/signing_key_export.go (98%) diff --git a/cmd/utilities/combine_signing_keys/main.go b/cmd/utilities/combine_signing_keys/main.go index b6026cf4..6ed97a5e 100644 --- a/cmd/utilities/combine_signing_keys/main.go +++ b/cmd/utilities/combine_signing_keys/main.go @@ -5,7 +5,7 @@ import ( "os" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/utilities/_common" + "github.com/t2bot/matrix-media-repo/cmd/utilities/common" "github.com/t2bot/matrix-media-repo/homeserver_interop" "github.com/t2bot/matrix-media-repo/homeserver_interop/any_server" "github.com/t2bot/matrix-media-repo/util" @@ -36,7 +36,7 @@ func main() { } } - _common.EncodeSigningKeys(keysArray, *outputFormat, *outputFile) + common.EncodeSigningKeys(keysArray, *outputFormat, *outputFile) } func decodeKeys(fileName string) ([]*homeserver_interop.SigningKey, error) { diff --git a/cmd/utilities/_common/signing_key_export.go b/cmd/utilities/common/signing_key_export.go similarity index 98% rename from cmd/utilities/_common/signing_key_export.go rename to cmd/utilities/common/signing_key_export.go index e8a5d5a1..092a861e 100644 --- a/cmd/utilities/_common/signing_key_export.go +++ b/cmd/utilities/common/signing_key_export.go @@ -1,4 +1,4 @@ -package _common +package common import ( "os" diff --git a/cmd/utilities/generate_signing_key/main.go b/cmd/utilities/generate_signing_key/main.go index 3367164d..1853452e 100644 --- a/cmd/utilities/generate_signing_key/main.go +++ b/cmd/utilities/generate_signing_key/main.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/utilities/_common" + "github.com/t2bot/matrix-media-repo/cmd/utilities/common" "github.com/t2bot/matrix-media-repo/homeserver_interop" "github.com/t2bot/matrix-media-repo/homeserver_interop/any_server" ) @@ -44,7 +44,7 @@ func main() { logrus.Infof("Key ID will be 'ed25519:%s'", key.KeyVersion) - _common.EncodeSigningKeys([]*homeserver_interop.SigningKey{key}, *outputFormat, *outputFile) + common.EncodeSigningKeys([]*homeserver_interop.SigningKey{key}, *outputFormat, *outputFile) } func makeKeyVersion() string { @@ -53,7 +53,6 @@ func makeKeyVersion() string { for i := 0; i < len(chars); i++ { sort.Slice(chars, func(i int, j int) bool { c, err := rand.Read(buf) - // "should never happen" clauses if err != nil { panic(err) From fadeb9af3bd88668f35db390106788f19890d2a5 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:06:16 +0100 Subject: [PATCH 15/22] move api/_apimeta to api/apimeta --- api/_routers/97-optional-access-token.go | 8 ++++---- api/_routers/97-require-access-token.go | 8 ++++---- api/_routers/97-require-repo-admin.go | 4 ++-- api/{_apimeta => apimeta}/auth.go | 2 +- api/custom/datastores.go | 8 ++++---- api/custom/exports.go | 14 +++++++------- api/custom/federation.go | 4 ++-- api/custom/health.go | 4 ++-- api/custom/imports.go | 8 ++++---- api/custom/media_attributes.go | 8 ++++---- api/custom/purge.go | 20 ++++++++++---------- api/custom/quarantine.go | 12 ++++++------ api/custom/tasks.go | 8 ++++---- api/custom/usage.go | 12 ++++++------ api/custom/version.go | 4 ++-- api/r0/download.go | 4 ++-- api/r0/identicon.go | 4 ++-- api/r0/logout.go | 6 +++--- api/r0/preview_url.go | 4 ++-- api/r0/public_config.go | 4 ++-- api/r0/thumbnail.go | 4 ++-- api/r0/upload_async.go | 4 ++-- api/r0/upload_sync.go | 4 ++-- api/unstable/info.go | 4 ++-- api/unstable/local_copy.go | 4 ++-- api/unstable/public_usage.go | 4 ++-- api/v1/create.go | 4 ++-- 27 files changed, 87 insertions(+), 87 deletions(-) rename api/{_apimeta => apimeta}/auth.go (97%) diff --git a/api/_routers/97-optional-access-token.go b/api/_routers/97-optional-access-token.go index 5735d6eb..8830ac50 100644 --- a/api/_routers/97-optional-access-token.go +++ b/api/_routers/97-optional-access-token.go @@ -6,9 +6,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/matrix" @@ -19,7 +19,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { accessToken := util.GetAccessTokenFromRequest(r) if accessToken == "" { - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "", AccessToken: "", IsShared: false, @@ -27,7 +27,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { } if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token { ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true, @@ -47,7 +47,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { } ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: userId, AccessToken: accessToken, IsShared: false, diff --git a/api/_routers/97-require-access-token.go b/api/_routers/97-require-access-token.go index 995071a7..081f5d7e 100644 --- a/api/_routers/97-require-access-token.go +++ b/api/_routers/97-require-access-token.go @@ -6,9 +6,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -16,7 +16,7 @@ import ( "github.com/t2bot/matrix-media-repo/util" ) -type GeneratorWithUserFn = func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} +type GeneratorWithUserFn = func(r *http.Request, ctx rcontext.RequestContext, user apimeta.UserInfo) interface{} func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { @@ -30,7 +30,7 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { } if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token { ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true, @@ -51,7 +51,7 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { } ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: userId, AccessToken: accessToken, IsShared: false, diff --git a/api/_routers/97-require-repo-admin.go b/api/_routers/97-require-repo-admin.go index 25b93baa..d5044255 100644 --- a/api/_routers/97-require-repo-admin.go +++ b/api/_routers/97-require-repo-admin.go @@ -4,15 +4,15 @@ import ( "net/http" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" ) func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { - return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { + return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if user.UserId == "" { logrus.Error("safety check failed: Repo admin access check received empty user ID") return _responses.AuthFailed() diff --git a/api/_apimeta/auth.go b/api/apimeta/auth.go similarity index 97% rename from api/_apimeta/auth.go rename to api/apimeta/auth.go index 5bd5dfd6..7690ec62 100644 --- a/api/_apimeta/auth.go +++ b/api/apimeta/auth.go @@ -1,4 +1,4 @@ -package _apimeta +package apimeta import ( "net/http" diff --git a/api/custom/datastores.go b/api/custom/datastores.go index e96b1b1c..ed9ef394 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -24,7 +24,7 @@ type DatastoreMigration struct { TaskID int `json:"task_id"` } -func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { response := make(map[string]interface{}) for _, store := range config.UniqueDatastores() { uri, err := datastores.GetUri(store) @@ -42,7 +42,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta. return &_responses.DoNotCacheResponse{Payload: response} } -func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") beforeTs := time.Now().UnixMilli() var err error @@ -95,7 +95,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use return &_responses.DoNotCacheResponse{Payload: migration} } -func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") beforeTs := time.Now().UnixMilli() var err error diff --git a/api/custom/exports.go b/api/custom/exports.go index 6fd71c8d..b8c4a1d4 100644 --- a/api/custom/exports.go +++ b/api/custom/exports.go @@ -7,9 +7,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -38,7 +38,7 @@ type ExportMetadata struct { Parts []*ExportPartMetadata `json:"parts"` } -func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -73,7 +73,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta }} } -func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -122,7 +122,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime }} } -func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -190,7 +190,7 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use return &_responses.HtmlResponse{HTML: html.String()} } -func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -237,7 +237,7 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apim return &_responses.DoNotCacheResponse{Payload: metadata} } -func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -293,7 +293,7 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api } } -func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } diff --git a/api/custom/federation.go b/api/custom/federation.go index f0b51eae..3ff69160 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -6,16 +6,16 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/matrix" ) -func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { diff --git a/api/custom/health.go b/api/custom/health.go index ac4780d8..fe35d86b 100644 --- a/api/custom/health.go +++ b/api/custom/health.go @@ -3,8 +3,8 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -13,7 +13,7 @@ type HealthzResponse struct { Status string `json:"status"` } -func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { return &_responses.DoNotCacheResponse{ Payload: &HealthzResponse{ OK: true, diff --git a/api/custom/imports.go b/api/custom/imports.go index 50edfe0e..5aa52f4f 100644 --- a/api/custom/imports.go +++ b/api/custom/imports.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/tasks" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -21,7 +21,7 @@ type ImportStarted struct { TaskID int `json:"task_id"` } -func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func StartImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -49,7 +49,7 @@ func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Us }} } -func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } @@ -76,7 +76,7 @@ func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} } -func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func StopImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { return _responses.BadRequest(errors.New("archiving is not enabled")) } diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go index a31a884d..27a40601 100644 --- a/api/custom/media_attributes.go +++ b/api/custom/media_attributes.go @@ -7,9 +7,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/matrix" @@ -20,7 +20,7 @@ type Attributes struct { Purpose database.Purpose `json:"purpose"` } -func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user _apimeta.UserInfo) bool { +func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user apimeta.UserInfo) bool { isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if isGlobalAdmin { return true @@ -33,7 +33,7 @@ func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin s return isLocalAdmin } -func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { origin := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) @@ -79,7 +79,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. return &_responses.DoNotCacheResponse{Payload: retAttrs} } -func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { origin := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) diff --git a/api/custom/purge.go b/api/custom/purge.go index ee8c273d..667c010d 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -25,7 +25,7 @@ type MediaPurgedResponse struct { NumRemoved int `json:"total_removed"` } -func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr == "" { return _responses.BadRequest(errors.New("Missing before_ts argument")) @@ -51,7 +51,7 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apime return &_responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} } -func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, _, _ := getPurgeAuthContext(rctx, r, user) server := _routers.GetParam("server", r) @@ -84,7 +84,7 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _ return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} } -func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) var affected []*database.DbMedia @@ -119,7 +119,7 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { var err error before := time.Now() beforeTsStr := r.URL.Query().Get("before_ts") @@ -173,7 +173,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { return _responses.AuthFailed() @@ -230,7 +230,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { return _responses.AuthFailed() @@ -303,7 +303,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} } -func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { return _responses.AuthFailed() @@ -357,8 +357,8 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func getPurgeAuthContext(ctx rcontext.RequestContext, r *http.Request, user _apimeta.UserInfo) (*task_runner.PurgeAuthContext, bool, bool) { - globalAdmin, localAdmin := _apimeta.GetRequestUserAdminStatus(r, ctx, user) +func getPurgeAuthContext(ctx rcontext.RequestContext, r *http.Request, user apimeta.UserInfo) (*task_runner.PurgeAuthContext, bool, bool) { + globalAdmin, localAdmin := apimeta.GetRequestUserAdminStatus(r, ctx, user) if globalAdmin { return &task_runner.PurgeAuthContext{}, true, localAdmin } diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go index 4c0d7bd9..87e09c6e 100644 --- a/api/custom/quarantine.go +++ b/api/custom/quarantine.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -21,7 +21,7 @@ type MediaQuarantinedResponse struct { NumQuarantined int64 `json:"num_quarantined"` } -func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { return _responses.AuthFailed() @@ -52,7 +52,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _ap }) } -func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { return _responses.AuthFailed() @@ -89,7 +89,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap }) } -func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { return _responses.AuthFailed() @@ -123,7 +123,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ }) } -func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { return _responses.AuthFailed() @@ -170,7 +170,7 @@ func performQuarantineRequest(ctx rcontext.RequestContext, host string, allowOth return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} } -func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) (bool, bool, bool) { +func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) (bool, bool, bool) { isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared canQuarantine := isGlobalAdmin allowOtherHosts := isGlobalAdmin diff --git a/api/custom/tasks.go b/api/custom/tasks.go index b3814f25..35a265ee 100644 --- a/api/custom/tasks.go +++ b/api/custom/tasks.go @@ -6,9 +6,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/sirupsen/logrus" @@ -25,7 +25,7 @@ type TaskStatus struct { Error string `json:"error_message"` } -func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetTask(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { taskIdStr := _routers.GetParam("taskId", r) taskId, err := strconv.Atoi(taskIdStr) if err != nil { @@ -60,7 +60,7 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserIn }} } -func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { db := database.GetInstance().Tasks.Prepare(rctx) tasks, err := db.GetAll(true) @@ -86,7 +86,7 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U return &_responses.DoNotCacheResponse{Payload: statusObjs} } -func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { db := database.GetInstance().Tasks.Prepare(rctx) tasks, err := db.GetAll(false) diff --git a/api/custom/usage.go b/api/custom/usage.go index 48cff4ea..fade3dbe 100644 --- a/api/custom/usage.go +++ b/api/custom/usage.go @@ -8,9 +8,9 @@ import ( "strings" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -54,7 +54,7 @@ type MediaUsageEntry struct { CreatedTs int64 `json:"created_ts"` } -func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { @@ -101,7 +101,7 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta } } -func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { serverName := _routers.GetParam("serverName", r) userIds := r.URL.Query()["user_id"] @@ -160,7 +160,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U return &_responses.DoNotCacheResponse{Payload: parsed} } -func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { serverName := _routers.GetParam("serverName", r) mxcs := r.URL.Query()["mxc"] @@ -224,7 +224,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet // SynGetUsersMediaStats attempts to provide a loose equivalent to this Synapse admin endpoint: // https://matrix-org.github.io/synapse/v1.88/admin_api/statistics.html#users-media-usage-statistics -func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { qs := r.URL.Query() var err error @@ -237,7 +237,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ return _responses.BadRequest(errors.New("invalid server name")) } - isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user) + isGlobalAdmin, isLocalAdmin := apimeta.GetRequestUserAdminStatus(r, rctx, user) if !isGlobalAdmin && (serverName != r.Host || !isLocalAdmin) { return _responses.AuthFailed() } diff --git a/api/custom/version.go b/api/custom/version.go index 11ff3b51..e98243de 100644 --- a/api/custom/version.go +++ b/api/custom/version.go @@ -3,13 +3,13 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/common/version" ) -func GetVersion(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetVersion(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { unstableFeatures := make(map[string]bool) return &_responses.DoNotCacheResponse{ diff --git a/api/r0/download.go b/api/r0/download.go index 9cc59709..1cfec4d9 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/util" @@ -19,7 +19,7 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) filename := _routers.GetParam("filename", r) diff --git a/api/r0/identicon.go b/api/r0/identicon.go index 64c6e527..8244eb2e 100644 --- a/api/r0/identicon.go +++ b/api/r0/identicon.go @@ -11,13 +11,13 @@ import ( "github.com/cupcake/sigil/gen" "github.com/disintegration/imaging" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Identicons.Enabled { return _responses.NotFoundError() } diff --git a/api/r0/logout.go b/api/r0/logout.go index 18a5e6b8..dcdcb0fa 100644 --- a/api/r0/logout.go +++ b/api/r0/logout.go @@ -5,14 +5,14 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func Logout(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func Logout(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { err := _auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) @@ -22,7 +22,7 @@ func Logout(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInf return _responses.EmptyResponse{} } -func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { err := _auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go index 5476b4c2..fbda1fe2 100644 --- a/api/r0/preview_url.go +++ b/api/r0/preview_url.go @@ -8,8 +8,8 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_preview" "github.com/t2bot/matrix-media-repo/common" @@ -29,7 +29,7 @@ type MatrixOpenGraph struct { ImageHeight int `json:"og:image:height,omitempty"` } -func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.UrlPreviews.Enabled { return _responses.NotFoundError() } diff --git a/api/r0/public_config.go b/api/r0/public_config.go index 695c17db..9a34fe66 100644 --- a/api/r0/public_config.go +++ b/api/r0/public_config.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) @@ -15,7 +15,7 @@ type PublicConfigResponse struct { StorageMaxFiles int64 `json:"org.matrix.msc4034.storage.max_files,omitempty"` } -func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { uploadSize := rctx.Config.Uploads.ReportedMaxSizeBytes if uploadSize == 0 { uploadSize = rctx.Config.Uploads.MaxSizeBytes diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 31126f5b..05cda1f2 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" @@ -21,7 +21,7 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go index 24480609..2b9685db 100644 --- a/api/r0/upload_async.go +++ b/api/r0/upload_async.go @@ -7,15 +7,15 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_upload" ) -func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) filename := filepath.Base(r.URL.Query().Get("filename")) diff --git a/api/r0/upload_sync.go b/api/r0/upload_sync.go index 2af0732e..875a6111 100644 --- a/api/r0/upload_sync.go +++ b/api/r0/upload_sync.go @@ -8,8 +8,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -21,7 +21,7 @@ type MediaUploadedResponse struct { ContentUri string `json:"content_uri,omitempty"` } -func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { filename := filepath.Base(r.URL.Query().Get("filename")) rctx = rctx.LogWithFields(logrus.Fields{ diff --git a/api/unstable/info.go b/api/unstable/info.go index 78f92307..b037efdb 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -10,9 +10,9 @@ import ( "github.com/disintegration/imaging" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" @@ -48,7 +48,7 @@ type MediaInfoResponse struct { NumChannels int `json:"num_channels,omitempty"` } -func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go index 63b6cfda..715d6cf6 100644 --- a/api/unstable/local_copy.go +++ b/api/unstable/local_copy.go @@ -8,9 +8,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/r0" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -20,7 +20,7 @@ import ( "github.com/t2bot/matrix-media-repo/util" ) -func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") diff --git a/api/unstable/public_usage.go b/api/unstable/public_usage.go index a5572ad6..d6380e2f 100644 --- a/api/unstable/public_usage.go +++ b/api/unstable/public_usage.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) @@ -14,7 +14,7 @@ type PublicUsageResponse struct { StorageFiles int64 `json:"org.matrix.msc4034.storage.files,omitempty"` } -func PublicUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PublicUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { storageUsed := int64(0) current, err := quota.Current(rctx, user.UserId, quota.MaxBytes) if err != nil { diff --git a/api/v1/create.go b/api/v1/create.go index aecf7448..1aebac0d 100644 --- a/api/v1/create.go +++ b/api/v1/create.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_create" "github.com/t2bot/matrix-media-repo/util" @@ -17,7 +17,7 @@ type MediaCreatedResponse struct { ExpiresTs int64 `json:"unused_expires_at"` } -func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { id, err := pipeline_create.Execute(rctx, r.Host, user.UserId, pipeline_create.DefaultExpirationTime) if err != nil { rctx.Log.Error("Unexpected error creating media ID:", err) From 8515e0be94de466190c76c59fb0a735b0e9df36e Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:07:56 +0100 Subject: [PATCH 16/22] move api/_responses to api/responses --- api/_routers/03-host_detection.go | 4 +- api/_routers/97-optional-access-token.go | 4 +- api/_routers/97-require-access-token.go | 10 +-- api/_routers/97-require-repo-admin.go | 6 +- api/_routers/98-use-rcontext.go | 22 +++--- api/custom/datastores.go | 26 +++---- api/custom/exports.go | 80 ++++++++++----------- api/custom/federation.go | 12 ++-- api/custom/health.go | 4 +- api/custom/imports.go | 36 +++++----- api/custom/media_attributes.go | 28 ++++---- api/custom/purge.go | 84 +++++++++++------------ api/custom/quarantine.go | 32 ++++----- api/custom/tasks.go | 18 ++--- api/custom/usage.go | 50 +++++++------- api/custom/version.go | 4 +- api/r0/download.go | 28 ++++---- api/r0/identicon.go | 10 +-- api/r0/logout.go | 10 +-- api/r0/preview_url.go | 16 ++--- api/r0/thumbnail.go | 44 ++++++------ api/r0/upload_async.go | 14 ++-- api/r0/upload_sync.go | 16 ++--- api/{_responses => responses}/content.go | 2 +- api/{_responses => responses}/errors.go | 2 +- api/{_responses => responses}/meta.go | 2 +- api/{_responses => responses}/redirect.go | 2 +- api/router.go | 8 +-- api/unstable/info.go | 22 +++--- api/unstable/local_copy.go | 26 +++---- api/v1/create.go | 4 +- api/webserver.go | 4 +- metrics/metrics.go | 2 +- 33 files changed, 316 insertions(+), 316 deletions(-) rename api/{_responses => responses}/content.go (96%) rename api/{_responses => responses}/errors.go (99%) rename api/{_responses => responses}/meta.go (75%) rename api/{_responses => responses}/redirect.go (87%) diff --git a/api/_routers/03-host_detection.go b/api/_routers/03-host_detection.go index c4c1040c..fdf8519d 100644 --- a/api/_routers/03-host_detection.go +++ b/api/_routers/03-host_detection.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sebest/xff" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/metrics" @@ -62,7 +62,7 @@ func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadGateway) var b []byte - if b, err = json.Marshal(_responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { + if b, err = json.Marshal(responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { logger.Errorf("Error preparing BadGateway: %v", err) sentry.CaptureException(err) return diff --git a/api/_routers/97-optional-access-token.go b/api/_routers/97-optional-access-token.go index 8830ac50..a2376cea 100644 --- a/api/_routers/97-optional-access-token.go +++ b/api/_routers/97-optional-access-token.go @@ -7,8 +7,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_auth_cache" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/matrix" @@ -39,7 +39,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { if !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError(errors.New("unexpected error validating access token")) + return responses.InternalServerError(errors.New("unexpected error validating access token")) } ctx.Log.Warn("Failed to verify token (non-fatal): ", err) diff --git a/api/_routers/97-require-access-token.go b/api/_routers/97-require-access-token.go index 081f5d7e..ea1aeb6c 100644 --- a/api/_routers/97-require-access-token.go +++ b/api/_routers/97-require-access-token.go @@ -7,8 +7,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_auth_cache" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -22,7 +22,7 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { accessToken := util.GetAccessTokenFromRequest(r) if accessToken == "" { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeMissingToken, Message: "no token provided (required)", InternalCode: common.ErrCodeMissingToken, @@ -40,14 +40,14 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId) if err != nil || userId == "" { if errors.Is(err, matrix.ErrGuestToken) { - return _responses.GuestAuthFailed() + return responses.GuestAuthFailed() } if err != nil && !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError(errors.New("unexpected error validating access token")) + return responses.InternalServerError(errors.New("unexpected error validating access token")) } - return _responses.AuthFailed() + return responses.AuthFailed() } ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId}) diff --git a/api/_routers/97-require-repo-admin.go b/api/_routers/97-require-repo-admin.go index d5044255..95fca9ef 100644 --- a/api/_routers/97-require-repo-admin.go +++ b/api/_routers/97-require-repo-admin.go @@ -4,8 +4,8 @@ import ( "net/http" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" ) @@ -15,11 +15,11 @@ func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn { return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if user.UserId == "" { logrus.Error("safety check failed: Repo admin access check received empty user ID") - return _responses.AuthFailed() + return responses.AuthFailed() } if !user.IsShared && !util.IsGlobalAdmin(user.UserId) { - return _responses.AuthFailed() + return responses.AuthFailed() } return generator(r, ctx, user) diff --git a/api/_routers/98-use-rcontext.go b/api/_routers/98-use-rcontext.go index 7a3045fe..25497112 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/_routers/98-use-rcontext.go @@ -16,7 +16,7 @@ import ( "github.com/alioygur/is" "github.com/getsentry/sentry-go" "github.com/t2bot/gotd-contrib/http_range" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -52,11 +52,11 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { var res interface{} res = c.generatorFn(r, rctx) if res == nil { - res = &_responses.EmptyResponse{} + res = &responses.EmptyResponse{} } shouldCache := true - wrappedRes, isNoCache := res.(*_responses.DoNotCacheResponse) + wrappedRes, isNoCache := res.(*responses.DoNotCacheResponse) if isNoCache { shouldCache = false res = wrappedRes.Payload @@ -65,7 +65,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { headers := w.Header() // Check for redirection early - if redirect, isRedirect := res.(*_responses.RedirectResponse); isRedirect { + if redirect, isRedirect := res.(*responses.RedirectResponse); isRedirect { log.Infof("Replying with result: %T <%s>", res, redirect.ToUrl) headers.Set("Location", redirect.ToUrl) r = writeStatusCode(w, r, http.StatusTemporaryRedirect) @@ -73,7 +73,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Check for HTML response and reply accordingly - if htmlRes, isHtml := res.(*_responses.HtmlResponse); isHtml { + if htmlRes, isHtml := res.(*responses.HtmlResponse); isHtml { log.Infof("Replying with result: %T <%d chars of html>", res, len(htmlRes.HTML)) // Write out HTML here, now that we know it's happening @@ -100,20 +100,20 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { var contentType string beforeParseDownload: log.Infof("Replying with result: %T %+v", res, res) - if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload { + if downloadRes, isDownload := res.(*responses.DownloadResponse); isDownload { ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes) if errors.Is(err, http_range.ErrInvalid) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest(errors.New("invalid range header")) + res = responses.BadRequest(errors.New("invalid range header")) goto beforeParseDownload // reprocess `res` } else if errors.Is(err, http_range.ErrNoOverlap) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest(errors.New("out of range")) + res = responses.BadRequest(errors.New("out of range")) goto beforeParseDownload // reprocess `res` } if len(ranges) > 1 { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest(errors.New("only 1 range is supported")) + res = responses.BadRequest(errors.New("only 1 range is supported")) goto beforeParseDownload // reprocess `res` } @@ -182,10 +182,10 @@ beforeParseDownload: } // Try to find a suitable error code, if one is needed - if errRes, isError := res.(_responses.ErrorResponse); isError { + if errRes, isError := res.(responses.ErrorResponse); isError { res = &errRes // just fix it } - if errRes, isError := res.(*_responses.ErrorResponse); isError && proposedStatusCode == http.StatusOK { + if errRes, isError := res.(*responses.ErrorResponse); isError && proposedStatusCode == http.StatusOK { switch errRes.InternalCode { case common.ErrCodeMissingToken: proposedStatusCode = http.StatusUnauthorized diff --git a/api/custom/datastores.go b/api/custom/datastores.go index ed9ef394..f15455c2 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -31,7 +31,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { sentry.CaptureException(err) rctx.Log.Error("Error getting datastore URI: ", err) - return _responses.InternalServerError(errors.New("unexpected error getting datastore information")) + return responses.InternalServerError(errors.New("unexpected error getting datastore information")) } dataStoreMap := make(map[string]interface{}) dataStoreMap["type"] = store.Type @@ -39,7 +39,7 @@ func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.U response[store.Id] = dataStoreMap } - return &_responses.DoNotCacheResponse{Payload: response} + return &responses.DoNotCacheResponse{Payload: response} } func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -49,7 +49,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -63,20 +63,20 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use }) if sourceDsId == targetDsId { - return _responses.BadRequest(errors.New("Source and target datastore cannot be the same")) + return responses.BadRequest(errors.New("Source and target datastore cannot be the same")) } if _, ok := datastores.Get(rctx, sourceDsId); !ok { - return _responses.BadRequest(errors.New("Source datastore does not appear to exist")) + return responses.BadRequest(errors.New("Source datastore does not appear to exist")) } if _, ok := datastores.Get(rctx, targetDsId); !ok { - return _responses.BadRequest(errors.New("Target datastore does not appear to exist")) + return responses.BadRequest(errors.New("Target datastore does not appear to exist")) } estimate, err := datastores.SizeOfDsIdWithAge(rctx, sourceDsId, beforeTs) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) + return responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } rctx.Log.Infof("User %s has started a datastore media transfer", user.UserId) @@ -84,7 +84,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected error starting migration")) + return responses.InternalServerError(errors.New("Unexpected error starting migration")) } migration := &DatastoreMigration{ @@ -92,7 +92,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use TaskID: task.TaskId, } - return &_responses.DoNotCacheResponse{Payload: migration} + return &responses.DoNotCacheResponse{Payload: migration} } func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -102,7 +102,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -117,7 +117,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) + return responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } - return &_responses.DoNotCacheResponse{Payload: result} + return &responses.DoNotCacheResponse{Payload: result} } diff --git a/api/custom/exports.go b/api/custom/exports.go index b8c4a1d4..a65b796b 100644 --- a/api/custom/exports.go +++ b/api/custom/exports.go @@ -7,9 +7,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -40,12 +40,12 @@ type ExportMetadata struct { func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if !rctx.Config.Archiving.SelfService && !isAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } s3urls := r.URL.Query().Get("s3_urls") != "false" @@ -53,7 +53,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta. userId := _routers.GetParam("userId", r) if !isAdmin && user.UserId != userId { - return _responses.BadRequest(errors.New("cannot export data for another user")) + return responses.BadRequest(errors.New("cannot export data for another user")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -64,10 +64,10 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("fatal error starting export")) + return responses.InternalServerError(errors.New("fatal error starting export")) } - return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ + return &responses.DoNotCacheResponse{Payload: &ExportStarted{ TaskID: task.TaskId, ExportID: exportId, }} @@ -75,12 +75,12 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta. func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if !rctx.Config.Archiving.SelfService && !isAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } s3urls := r.URL.Query().Get("s3_urls") != "false" @@ -92,7 +92,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimet // We won't be able to check unless we know about the homeserver though if !util.IsServerOurs(serverName) { - return _responses.BadRequest(errors.New("cannot export data for another server")) + return responses.BadRequest(errors.New("cannot export data for another server")) } isLocalAdmin, err := matrix.IsUserAdmin(rctx, serverName, user.AccessToken, r.RemoteAddr) @@ -101,7 +101,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimet isLocalAdmin = false } if !isLocalAdmin { - return _responses.BadRequest(errors.New("cannot export data for another server")) + return responses.BadRequest(errors.New("cannot export data for another server")) } } @@ -113,10 +113,10 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("fatal error starting export")) + return responses.InternalServerError(errors.New("fatal error starting export")) } - return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ + return &responses.DoNotCacheResponse{Payload: &ExportStarted{ TaskID: task.TaskId, ExportID: exportId, }} @@ -124,13 +124,13 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimet func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest(errors.New("invalid export ID")) + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -144,24 +144,24 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.User if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get entity for export ID")) + return responses.InternalServerError(errors.New("failed to get entity for export ID")) } if entityId == "" { - return _responses.NotFoundError() + return responses.NotFoundError() } parts, err := partsDb.GetForExport(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get export parts")) + return responses.InternalServerError(errors.New("failed to get export parts")) } template, err := templating.GetTemplate("view_export") if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get template")) + return responses.InternalServerError(errors.New("failed to get template")) } model := &templating.ViewExportModel{ @@ -184,21 +184,21 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.User if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to render template")) + return responses.InternalServerError(errors.New("failed to render template")) } - return &_responses.HtmlResponse{HTML: html.String()} + return &responses.HtmlResponse{HTML: html.String()} } func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest(errors.New("invalid export ID")) + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -212,14 +212,14 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get entity for export ID")) + return responses.InternalServerError(errors.New("failed to get entity for export ID")) } parts, err := partsDb.GetForExport(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get export parts")) + return responses.InternalServerError(errors.New("failed to get export parts")) } metadata := &ExportMetadata{ @@ -234,25 +234,25 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apime }) } - return &_responses.DoNotCacheResponse{Payload: metadata} + return &responses.DoNotCacheResponse{Payload: metadata} } func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) pid := _routers.GetParam("partId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest(errors.New("invalid export ID")) + responses.BadRequest(errors.New("invalid export ID")) } partId, err := strconv.Atoi(pid) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest(errors.New("invalid part index")) + return responses.BadRequest(errors.New("invalid part index")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -265,26 +265,26 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get part")) + return responses.InternalServerError(errors.New("failed to get part")) } if part == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } dsConf, ok := datastores.Get(rctx, part.DatastoreId) if !ok { sentry.CaptureMessage("failed to locate datastore") - return _responses.InternalServerError(errors.New("failed to locate datastore")) + return responses.InternalServerError(errors.New("failed to locate datastore")) } s, err := datastores.Download(rctx, dsConf, part.Location) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to start download")) + return responses.InternalServerError(errors.New("failed to start download")) } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: "application/gzip", // TODO: We should be detecting type rather than assuming SizeBytes: part.SizeBytes, Data: s, @@ -295,13 +295,13 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apim func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } exportId := _routers.GetParam("exportId", r) if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest(errors.New("invalid export ID")) + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -316,7 +316,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get export parts")) + return responses.InternalServerError(errors.New("failed to get export parts")) } for _, part := range parts { @@ -325,7 +325,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to delete export part")) + return responses.InternalServerError(errors.New("failed to delete export part")) } } @@ -334,14 +334,14 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to delete export parts")) + return responses.InternalServerError(errors.New("failed to delete export parts")) } err = exportDb.Delete(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to delete export record")) + return responses.InternalServerError(errors.New("failed to delete export record")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } diff --git a/api/custom/federation.go b/api/custom/federation.go index 3ff69160..5ad0522f 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -6,9 +6,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -19,7 +19,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apime serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -30,7 +30,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err) + return responses.InternalServerError(err) } versionUrl := url + "/_matrix/federation/v1/version" @@ -38,7 +38,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err) + return responses.InternalServerError(err) } decoder := json.NewDecoder(versionResponse.Body) @@ -47,12 +47,12 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err) + return responses.InternalServerError(err) } resp := make(map[string]interface{}) resp["base_url"] = url resp["hostname"] = hostname resp["versions_response"] = out - return &_responses.DoNotCacheResponse{Payload: resp} + return &responses.DoNotCacheResponse{Payload: resp} } diff --git a/api/custom/health.go b/api/custom/health.go index fe35d86b..3ca4154c 100644 --- a/api/custom/health.go +++ b/api/custom/health.go @@ -3,8 +3,8 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -14,7 +14,7 @@ type HealthzResponse struct { } func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - return &_responses.DoNotCacheResponse{ + return &responses.DoNotCacheResponse{ Payload: &HealthzResponse{ OK: true, Status: "Probably not dead", diff --git a/api/custom/imports.go b/api/custom/imports.go index 5aa52f4f..1b2e2153 100644 --- a/api/custom/imports.go +++ b/api/custom/imports.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/tasks" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -23,27 +23,27 @@ type ImportStarted struct { func StartImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } task, importId, err := tasks.RunImport(rctx) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("fatal error starting import")) + return responses.InternalServerError(errors.New("fatal error starting import")) } err = task_runner.AppendImportFile(rctx, importId, r.Body) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error appending first file to import")) + return responses.InternalServerError(errors.New("error appending first file to import")) } - return &_responses.DoNotCacheResponse{Payload: &ImportStarted{ + return &responses.DoNotCacheResponse{Payload: &ImportStarted{ TaskID: task.TaskId, ImportID: importId, }} @@ -51,54 +51,54 @@ func StartImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.Use func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } importId := _routers.GetParam("importId", r) if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest(errors.New("invalid import ID")) + return responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.AppendImportFile(rctx, importId, r.Body) if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error appending to import")) + return responses.InternalServerError(errors.New("error appending to import")) } - return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} + return &responses.DoNotCacheResponse{Payload: &responses.EmptyResponse{}} } func StopImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest(errors.New("archiving is not enabled")) + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } importId := _routers.GetParam("importId", r) if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest(errors.New("invalid import ID")) + return responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.FinishImport(rctx, importId) if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error stopping import")) + return responses.InternalServerError(errors.New("error stopping import")) } - return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} + return &responses.DoNotCacheResponse{Payload: &responses.EmptyResponse{}} } diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go index 27a40601..5203db5e 100644 --- a/api/custom/media_attributes.go +++ b/api/custom/media_attributes.go @@ -7,9 +7,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/matrix" @@ -38,7 +38,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest(errors.New("invalid origin")) + return responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -47,7 +47,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U }) if !canChangeAttributes(rctx, r, origin, user) { - return _responses.AuthFailed() + return responses.AuthFailed() } // Check to see if the media exists @@ -56,10 +56,10 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get media record")) + return responses.InternalServerError(errors.New("failed to get media record")) } if media == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } attrDb := database.GetInstance().MediaAttributes.Prepare(rctx) @@ -67,7 +67,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get attributes record")) + return responses.InternalServerError(errors.New("failed to get attributes record")) } retAttrs := &Attributes{ Purpose: database.PurposeNone, @@ -76,7 +76,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U retAttrs.Purpose = attrs.Purpose } - return &_responses.DoNotCacheResponse{Payload: retAttrs} + return &responses.DoNotCacheResponse{Payload: retAttrs} } func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -84,7 +84,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest(errors.New("invalid origin")) + return responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -93,7 +93,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U }) if !canChangeAttributes(rctx, r, origin, user) { - return _responses.AuthFailed() + return responses.AuthFailed() } defer r.Body.Close() @@ -103,7 +103,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to read attributes")) + return responses.InternalServerError(errors.New("failed to read attributes")) } attrDb := database.GetInstance().MediaAttributes.Prepare(rctx) @@ -111,20 +111,20 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get attributes")) + return responses.InternalServerError(errors.New("failed to get attributes")) } if attrs == nil || attrs.Purpose != newAttrs.Purpose { if !database.IsPurpose(newAttrs.Purpose) { - return _responses.BadRequest(errors.New("unknown purpose")) + return responses.BadRequest(errors.New("unknown purpose")) } err = attrDb.UpsertPurpose(origin, mediaId, newAttrs.Purpose) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to update attributes: purpose")) + return responses.InternalServerError(errors.New("failed to update attributes: purpose")) } } - return &_responses.DoNotCacheResponse{Payload: newAttrs} + return &responses.DoNotCacheResponse{Payload: newAttrs} } diff --git a/api/custom/purge.go b/api/custom/purge.go index 667c010d..a983cb93 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -28,11 +28,11 @@ type MediaPurgedResponse struct { func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr == "" { - return _responses.BadRequest(errors.New("Missing before_ts argument")) + return responses.BadRequest(errors.New("Missing before_ts argument")) } beforeTs, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } before := time.UnixMilli(beforeTs) @@ -45,10 +45,10 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user apimet if err != nil { rctx.Log.Error("Error purging remote media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Error purging remote media")) + return responses.InternalServerError(errors.New("Error purging remote media")) } - return &_responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} + return &responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} } func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -58,7 +58,7 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user a mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -74,14 +74,14 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user a }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} } func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -96,12 +96,12 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user apimet } else if isLocalAdmin { affected, err = mediaDb.GetByOriginQuarantine(r.Host) } else { - return _responses.AuthFailed() + return responses.AuthFailed() } if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error fetching media records")) + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -109,14 +109,14 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user apimet }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -126,7 +126,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if beforeTsStr != "" { beforeTS, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } before = time.UnixMilli(beforeTS) } @@ -136,7 +136,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if includeLocalStr != "" { includeLocal, err = strconv.ParseBool(includeLocalStr) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing include_local: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing include_local: %w", err)) } } @@ -155,7 +155,7 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error fetching media records")) + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, &task_runner.PurgeAuthContext{}, &task_runner.QuarantineThis{ @@ -163,20 +163,20 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error @@ -185,7 +185,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -200,11 +200,11 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Errorf("Error parsing user ID (%s): %v", userId, err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error parsing user ID")) + return responses.InternalServerError(errors.New("error parsing user ID")) } if !isGlobalAdmin && userDomain != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } mediaDb := database.GetInstance().Media.Prepare(rctx) @@ -212,7 +212,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error fetching media records")) + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -220,20 +220,20 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error @@ -242,7 +242,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } @@ -257,7 +257,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error retrieving media in room")) + return responses.InternalServerError(errors.New("error retrieving media in room")) } mxcs := make([]string, 0) @@ -293,20 +293,20 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} } func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error @@ -315,14 +315,14 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimet if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest(fmt.Errorf("Error parsing before_ts: %f", err)) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %f", err)) } } serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -331,7 +331,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimet }) if !isGlobalAdmin && serverName != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } mediaDb := database.GetInstance().Media.Prepare(rctx) @@ -339,7 +339,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error fetching media records")) + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -347,14 +347,14 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimet }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } func getPurgeAuthContext(ctx rcontext.RequestContext, r *http.Request, user apimeta.UserInfo) (*task_runner.PurgeAuthContext, bool, bool) { diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go index 87e09c6e..d59d10a8 100644 --- a/api/custom/quarantine.go +++ b/api/custom/quarantine.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -24,7 +24,7 @@ type MediaQuarantinedResponse struct { func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } roomId := _routers.GetParam("roomId", r) @@ -38,7 +38,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error retrieving media in room")) + return responses.InternalServerError(errors.New("error retrieving media in room")) } var mxcs []string @@ -55,7 +55,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } userId := _routers.GetParam("userId", r) @@ -69,11 +69,11 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api if err != nil { rctx.Log.Error("Error parsing user ID ("+userId+"): ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error parsing user ID")) + return responses.InternalServerError(errors.New("error parsing user ID")) } if !allowOtherHosts && userDomain != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } db := database.GetInstance().Media.Prepare(rctx) @@ -81,7 +81,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api if err != nil { rctx.Log.Error("Error while listing media for the user: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error retrieving media for user")) + return responses.InternalServerError(errors.New("error retrieving media for user")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -92,13 +92,13 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -107,7 +107,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a }) if !allowOtherHosts && serverName != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } db := database.GetInstance().Media.Prepare(rctx) @@ -115,7 +115,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a if err != nil { rctx.Log.Error("Error while listing media for the server: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error retrieving media for server")) + return responses.InternalServerError(errors.New("error retrieving media for server")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -126,14 +126,14 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } server := _routers.GetParam("server", r) mediaId := _routers.GetParam("mediaId", r) if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -143,7 +143,7 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta }) if !allowOtherHosts && r.Host != server { - return _responses.BadRequest(errors.New("unable to quarantine media on other homeservers")) + return responses.BadRequest(errors.New("unable to quarantine media on other homeservers")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -164,10 +164,10 @@ func performQuarantineRequest(ctx rcontext.RequestContext, host string, allowOth if err != nil { ctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("error quarantining media")) + return responses.InternalServerError(errors.New("error quarantining media")) } - return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} + return &responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} } func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) (bool, bool, bool) { diff --git a/api/custom/tasks.go b/api/custom/tasks.go index 35a265ee..0b7f0ae5 100644 --- a/api/custom/tasks.go +++ b/api/custom/tasks.go @@ -6,9 +6,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/sirupsen/logrus" @@ -30,7 +30,7 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInf taskId, err := strconv.Atoi(taskIdStr) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest(errors.New("invalid task ID")) + return responses.BadRequest(errors.New("invalid task ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -43,13 +43,13 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInf if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("failed to get task information")) + return responses.InternalServerError(errors.New("failed to get task information")) } if task == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } - return &_responses.DoNotCacheResponse{Payload: &TaskStatus{ + return &responses.DoNotCacheResponse{Payload: &TaskStatus{ TaskID: task.TaskId, Name: task.Name, Params: task.Params, @@ -67,7 +67,7 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get background tasks")) + return responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) @@ -83,7 +83,7 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us }) } - return &_responses.DoNotCacheResponse{Payload: statusObjs} + return &responses.DoNotCacheResponse{Payload: statusObjs} } func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -93,7 +93,7 @@ func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user api if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get background tasks")) + return responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) @@ -109,5 +109,5 @@ func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user api }) } - return &_responses.DoNotCacheResponse{Payload: statusObjs} + return &responses.DoNotCacheResponse{Payload: statusObjs} } diff --git a/api/custom/usage.go b/api/custom/usage.go index fade3dbe..e4640d03 100644 --- a/api/custom/usage.go +++ b/api/custom/usage.go @@ -8,9 +8,9 @@ import ( "strings" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -58,7 +58,7 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta. serverName := _routers.GetParam("serverName", r) if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -71,17 +71,17 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get byte usage for server")) + return responses.InternalServerError(errors.New("Failed to get byte usage for server")) } mediaCount, thumbCount, err := db.CountUsageForServer(serverName) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get count usage for server")) + return responses.InternalServerError(errors.New("Failed to get count usage for server")) } - return &_responses.DoNotCacheResponse{ + return &responses.DoNotCacheResponse{ Payload: &CountsUsageResponse{ RawBytes: &UsageInfo{ MinimalUsageInfo: &MinimalUsageInfo{ @@ -106,7 +106,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us userIds := r.URL.Query()["user_id"] if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -126,7 +126,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get media records for users")) + return responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*UserUsageEntry) @@ -157,7 +157,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us entry.UploadedMxcs = append(entry.UploadedMxcs, util.MxcUri(media.Origin, media.MediaId)) } - return &_responses.DoNotCacheResponse{Payload: parsed} + return &responses.DoNotCacheResponse{Payload: parsed} } func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -165,7 +165,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta mxcs := r.URL.Query()["mxc"] if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -185,11 +185,11 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(fmt.Errorf("Error parsing MXC %s", mxc)) + return responses.InternalServerError(fmt.Errorf("Error parsing MXC %s", mxc)) } if o != serverName { - return _responses.BadRequest(errors.New("MXC URIs must match the requested server")) + return responses.BadRequest(errors.New("MXC URIs must match the requested server")) } split = append(split, i) @@ -200,7 +200,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get media records for users")) + return responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*MediaUsageEntry) @@ -219,7 +219,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta } } - return &_responses.DoNotCacheResponse{Payload: parsed} + return &responses.DoNotCacheResponse{Payload: parsed} } // SynGetUsersMediaStats attempts to provide a loose equivalent to this Synapse admin endpoint: @@ -234,12 +234,12 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a } if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest(errors.New("invalid server name")) + return responses.BadRequest(errors.New("invalid server name")) } isGlobalAdmin, isLocalAdmin := apimeta.GetRequestUserAdminStatus(r, rctx, user) if !isGlobalAdmin && (serverName != r.Host || !isLocalAdmin) { - return _responses.AuthFailed() + return responses.AuthFailed() } orderBy := database.SynStatUserOrderBy(qs.Get("order_by")) @@ -247,14 +247,14 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a orderBy = database.DefaultSynStatUserOrderBy } if !database.IsSynStatUserOrderBy(orderBy) { - return _responses.BadRequest(errors.New("Query parameter 'order_by' must be one of the accepted values")) + return responses.BadRequest(errors.New("Query parameter 'order_by' must be one of the accepted values")) } var start int64 = 0 if len(qs["from"]) > 0 { start, err = strconv.ParseInt(qs.Get("from"), 10, 64) if err != nil || start < 0 { - return _responses.BadRequest(errors.New("Query parameter 'from' must be a non-negative integer")) + return responses.BadRequest(errors.New("Query parameter 'from' must be a non-negative integer")) } } @@ -262,7 +262,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a if len(qs["limit"]) > 0 { limit, err = strconv.ParseInt(qs.Get("limit"), 10, 64) if err != nil || limit < 0 { - return _responses.BadRequest(errors.New("Query parameter 'limit' must be a non-negative integer")) + return responses.BadRequest(errors.New("Query parameter 'limit' must be a non-negative integer")) } } if limit > 50 { @@ -274,7 +274,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a if len(qs["from_ts"]) > 0 { fromTS, err = strconv.ParseInt(qs.Get("from_ts"), 10, 64) if err != nil || fromTS < 0 { - return _responses.BadRequest(errors.New("Query parameter 'from_ts' must be a non-negative integer")) + return responses.BadRequest(errors.New("Query parameter 'from_ts' must be a non-negative integer")) } } @@ -282,15 +282,15 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a if len(qs["until_ts"]) > 0 { untilTS, err = strconv.ParseInt(qs.Get("until_ts"), 10, 64) if err != nil || untilTS < 0 { - return _responses.BadRequest(errors.New("Query parameter 'until_ts' must be a non-negative integer")) + return responses.BadRequest(errors.New("Query parameter 'until_ts' must be a non-negative integer")) } else if untilTS <= fromTS { - return _responses.BadRequest(errors.New("Query parameter 'until_ts' must be greater than 'from_ts'")) + return responses.BadRequest(errors.New("Query parameter 'until_ts' must be greater than 'from_ts'")) } } searchTerm := qs.Get("search_term") if searchTerm == "" && len(qs["search_term"]) > 0 { - return _responses.BadRequest(errors.New("Query parameter 'search_term' cannot be an empty string")) + return responses.BadRequest(errors.New("Query parameter 'search_term' cannot be an empty string")) } isAscendingOrder := true @@ -298,7 +298,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a if direction == "b" && len(qs["dir"]) > 0 { isAscendingOrder = false } else { - return _responses.BadRequest(errors.New("Query parameter 'dir' must be one of ['f', 'b']")) + return responses.BadRequest(errors.New("Query parameter 'dir' must be one of ['f', 'b']")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -318,7 +318,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Failed to get users' usage stats on specified server")) + return responses.InternalServerError(errors.New("Failed to get users' usage stats on specified server")) } result := &synapse.SynUserStatsResponse{ @@ -339,5 +339,5 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a result.NextToken = start + int64(len(stats)) } - return &_responses.DoNotCacheResponse{Payload: result} + return &responses.DoNotCacheResponse{Payload: result} } diff --git a/api/custom/version.go b/api/custom/version.go index e98243de..1ebb5c19 100644 --- a/api/custom/version.go +++ b/api/custom/version.go @@ -3,8 +3,8 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/common/version" ) @@ -12,7 +12,7 @@ import ( func GetVersion(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { unstableFeatures := make(map[string]bool) - return &_responses.DoNotCacheResponse{ + return &responses.DoNotCacheResponse{ Payload: map[string]interface{}{ "Version": version.Version, "GitCommit": version.GitCommit, diff --git a/api/r0/download.go b/api/r0/download.go index 1cfec4d9..e8c3bb7f 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/util" @@ -28,14 +28,14 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U timeoutMs := r.URL.Query().Get("timeout_ms") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) + return responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -44,14 +44,14 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) + return responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { - return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) + return responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } timeout := time.Duration(timeoutMS) * time.Millisecond if timeout > time.Minute { @@ -68,7 +68,7 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } media, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{ @@ -79,31 +79,31 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.U if err != nil { var redirect datastores.RedirectError if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return responses.Redirect(redirect.RedirectUrl) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } if filename == "" { filename = media.UploadName } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: media.ContentType, Filename: filename, SizeBytes: media.SizeBytes, diff --git a/api/r0/identicon.go b/api/r0/identicon.go index 8244eb2e..cd76c9b8 100644 --- a/api/r0/identicon.go +++ b/api/r0/identicon.go @@ -11,15 +11,15 @@ import ( "github.com/cupcake/sigil/gen" "github.com/disintegration/imaging" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" ) func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Identicons.Enabled { - return _responses.NotFoundError() + return responses.NotFoundError() } seed := _routers.GetParam("seed", r) @@ -33,14 +33,14 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI if widthStr != "" { width, err = strconv.Atoi(widthStr) if err != nil { - return _responses.InternalServerError(fmt.Errorf("Error parsing width: %w", err)) + return responses.InternalServerError(fmt.Errorf("Error parsing width: %w", err)) } height = width } if heightStr != "" { height, err = strconv.Atoi(heightStr) if err != nil { - return _responses.InternalServerError(fmt.Errorf("Error parsing height: %w", err)) + return responses.InternalServerError(fmt.Errorf("Error parsing height: %w", err)) } } @@ -99,7 +99,7 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI } }() - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: "image/png", Filename: string(hashed) + ".png", SizeBytes: 0, diff --git a/api/r0/logout.go b/api/r0/logout.go index dcdcb0fa..fa284aed 100644 --- a/api/r0/logout.go +++ b/api/r0/logout.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -17,9 +17,9 @@ func Logout(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unable to logout")) + return responses.InternalServerError(errors.New("unable to logout")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { @@ -27,7 +27,7 @@ func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unable to logout")) + return responses.InternalServerError(errors.New("unable to logout")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go index fbda1fe2..ef98a8ef 100644 --- a/api/r0/preview_url.go +++ b/api/r0/preview_url.go @@ -8,8 +8,8 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_preview" "github.com/t2bot/matrix-media-repo/common" @@ -31,7 +31,7 @@ type MatrixOpenGraph struct { func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.UrlPreviews.Enabled { - return _responses.NotFoundError() + return responses.NotFoundError() } params := r.URL.Query() @@ -46,7 +46,7 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.User ts, err = strconv.ParseInt(tsStr, 10, 64) if err != nil { rctx.Log.Error("Error parsing ts: ", err) - return _responses.BadRequest(err) + return responses.BadRequest(err) } } if ts == 0 { @@ -55,11 +55,11 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.User // Validate the URL if urlStr == "" { - return _responses.BadRequest(errors.New("No url provided")) + return responses.BadRequest(errors.New("No url provided")) } //goland:noinspection HttpUrlsUsage if strings.Index(urlStr, "http://") != 0 && strings.Index(urlStr, "https://") != 0 { - return _responses.BadRequest(errors.New("Scheme not accepted")) + return responses.BadRequest(errors.New("Scheme not accepted")) } languageHeader := rctx.Config.UrlPreviews.DefaultLanguage @@ -82,13 +82,13 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.User } if err != nil { if errors.Is(err, common.ErrMediaNotFound) || errors.Is(err, common.ErrHostNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) { - return _responses.BadRequest(err) + return responses.BadRequest(err) } sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MatrixOpenGraph{ diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 05cda1f2..49565dc5 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" @@ -29,14 +29,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. timeoutMs := r.URL.Query().Get("timeout_ms") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) + return responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -45,14 +45,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) + return responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { - return _responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) + return responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) } timeout := time.Duration(timeoutMS) * time.Millisecond if timeout > time.Minute { @@ -68,7 +68,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } widthStr := r.URL.Query().Get("width") @@ -80,7 +80,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. } if widthStr == "" || heightStr == "" { - return _responses.BadRequest(errors.New("Width and height are required")) + return responses.BadRequest(errors.New("Width and height are required")) } width := 0 @@ -90,21 +90,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if widthStr != "" { parsedWidth, err := strconv.Atoi(widthStr) if err != nil { - return _responses.BadRequest(errors.New("Width does not appear to be an integer")) + return responses.BadRequest(errors.New("Width does not appear to be an integer")) } width = parsedWidth } if heightStr != "" { parsedHeight, err := strconv.Atoi(heightStr) if err != nil { - return _responses.BadRequest(errors.New("Height does not appear to be an integer")) + return responses.BadRequest(errors.New("Height does not appear to be an integer")) } height = parsedHeight } if animatedStr != "" { parsedFlag, err := strconv.ParseBool(animatedStr) if err != nil { - return _responses.BadRequest(errors.New("Animated flag does not appear to be a boolean")) + return responses.BadRequest(errors.New("Animated flag does not appear to be a boolean")) } animated = parsedFlag } @@ -120,7 +120,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. }) if width <= 0 || height <= 0 { - return _responses.BadRequest(errors.New("Width and height must be greater than zero")) + return responses.BadRequest(errors.New("Width and height must be greater than zero")) } thumbnail, stream, err := pipeline_thumbnail.Execute(rctx, server, mediaId, pipeline_thumbnail.ThumbnailOpts{ @@ -138,21 +138,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { var redirect datastores.RedirectError if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } else if errors.Is(err, common.ErrMediaDimensionsTooSmall) { if stream == nil { - return _responses.NotFoundError() // something went wrong so just 404 the thumbnail + return responses.NotFoundError() // something went wrong so just 404 the thumbnail } // We have a stream, and an error about image size, so we know there should be a media record @@ -161,9 +161,9 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. if err != nil { rctx.Log.Error("Unexpected error locating media record: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } else { - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: record.ContentType, Filename: record.UploadName, SizeBytes: record.SizeBytes, @@ -172,14 +172,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. } } } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return responses.Redirect(redirect.RedirectUrl) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: thumbnail.ContentType, Filename: "thumbnail" + util.ExtensionForContentType(thumbnail.ContentType), SizeBytes: thumbnail.SizeBytes, diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go index 2b9685db..ab60047f 100644 --- a/api/r0/upload_async.go +++ b/api/r0/upload_async.go @@ -7,9 +7,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_upload" @@ -27,7 +27,7 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimet }) if r.Host != server { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeNotFound, Message: "Upload request is for another domain.", InternalCode: common.ErrCodeForbidden, @@ -48,21 +48,21 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimet _, err := pipeline_upload.ExecutePut(rctx, server, mediaId, r.Body, contentType, filename, user.UserId) if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } else if errors.Is(err, common.ErrAlreadyUploaded) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeCannotOverwrite, Message: "This media has already been uploaded.", InternalCode: common.ErrCodeCannotOverwrite, } } else if errors.Is(err, common.ErrWrongUser) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeForbidden, Message: "You do not have permission to upload this media.", InternalCode: common.ErrCodeForbidden, } } else if errors.Is(err, common.ErrExpired) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeNotFound, Message: "Media expired or not found.", InternalCode: common.ErrCodeNotFound, @@ -70,7 +70,7 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimet } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ diff --git a/api/r0/upload_sync.go b/api/r0/upload_sync.go index 875a6111..daf8df5b 100644 --- a/api/r0/upload_sync.go +++ b/api/r0/upload_sync.go @@ -8,8 +8,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -42,11 +42,11 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user apimeta media, err := pipeline_upload.Execute(rctx, r.Host, "", r.Body, contentType, filename, user.UserId, datastores.LocalMediaKind) if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ @@ -54,26 +54,26 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user apimeta } } -func uploadRequestSizeCheck(rctx rcontext.RequestContext, r *http.Request) *_responses.ErrorResponse { +func uploadRequestSizeCheck(rctx rcontext.RequestContext, r *http.Request) *responses.ErrorResponse { maxSize := rctx.Config.Uploads.MaxSizeBytes minSize := rctx.Config.Uploads.MinSizeBytes if maxSize > 0 || minSize > 0 { if r.ContentLength > 0 { if maxSize > 0 && maxSize < r.ContentLength { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } if minSize > 0 && minSize > r.ContentLength { - return _responses.RequestTooSmall() + return responses.RequestTooSmall() } } else { header := r.Header.Get("Content-Length") if header != "" { parsed, _ := strconv.ParseInt(header, 10, 64) if maxSize > 0 && maxSize < parsed { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } if minSize > 0 && minSize > parsed { - return _responses.RequestTooSmall() + return responses.RequestTooSmall() } } } diff --git a/api/_responses/content.go b/api/responses/content.go similarity index 96% rename from api/_responses/content.go rename to api/responses/content.go index 735e4b1b..775ef330 100644 --- a/api/_responses/content.go +++ b/api/responses/content.go @@ -1,4 +1,4 @@ -package _responses +package responses import "io" diff --git a/api/_responses/errors.go b/api/responses/errors.go similarity index 99% rename from api/_responses/errors.go rename to api/responses/errors.go index 1471b207..38006e27 100644 --- a/api/_responses/errors.go +++ b/api/responses/errors.go @@ -1,4 +1,4 @@ -package _responses +package responses import ( "fmt" diff --git a/api/_responses/meta.go b/api/responses/meta.go similarity index 75% rename from api/_responses/meta.go rename to api/responses/meta.go index 0971bd72..cbb7febb 100644 --- a/api/_responses/meta.go +++ b/api/responses/meta.go @@ -1,4 +1,4 @@ -package _responses +package responses type DoNotCacheResponse struct { Payload interface{} diff --git a/api/_responses/redirect.go b/api/responses/redirect.go similarity index 87% rename from api/_responses/redirect.go rename to api/responses/redirect.go index 1e8c66c4..b94e738c 100644 --- a/api/_responses/redirect.go +++ b/api/responses/redirect.go @@ -1,4 +1,4 @@ -package _responses +package responses type RedirectResponse struct { ToUrl string diff --git a/api/router.go b/api/router.go index dbe3d5ee..5405ec1f 100644 --- a/api/router.go +++ b/api/router.go @@ -9,8 +9,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/util" ) @@ -29,7 +29,7 @@ func buildPrimaryRouter() *httprouter.Router { func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusMethodNotAllowed) - reponse, err := json.Marshal(_responses.MethodNotAllowed()) + reponse, err := json.Marshal(responses.MethodNotAllowed()) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %v", err)) logrus.Errorf("error preparing MethodNotAllowed: %v", err) @@ -41,7 +41,7 @@ func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { func notFoundFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) - reponse, err := json.Marshal(_responses.NotFoundError()) + reponse, err := json.Marshal(responses.NotFoundError()) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing NotFound: %v", err)) logrus.Errorf("error preparing NotFound: %v", err) @@ -67,7 +67,7 @@ func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - reponse, err := json.Marshal(_responses.InternalServerError(errors.New("unexpected error"))) + reponse, err := json.Marshal(responses.InternalServerError(errors.New("unexpected error"))) if err != nil { sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %v", err)) logrus.Errorf("error preparing InternalServerError: %v", err) diff --git a/api/unstable/info.go b/api/unstable/info.go index b037efdb..1ea980b9 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -10,9 +10,9 @@ import ( "github.com/disintegration/imaging" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" @@ -54,14 +54,14 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI allowRemote := r.URL.Query().Get("allow_remote") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) + return responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -74,7 +74,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } record, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{ @@ -85,22 +85,22 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI // Error handling copied from download endpoint if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } response := &MediaInfoResponse{ @@ -137,7 +137,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI if err != nil { rctx.Log.Error("Unexpected error locating media thumbnails: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } if len(thumbs) > 0 { diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go index 715d6cf6..1a427716 100644 --- a/api/unstable/local_copy.go +++ b/api/unstable/local_copy.go @@ -8,10 +8,10 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -28,14 +28,14 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI rctx.Log.Warn("This endpoint is deprecated. See https://github.com/t2bot/matrix-media-repo/issues/422") if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest(errors.New("invalid server ID")) + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) + return responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -48,11 +48,11 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } if r.Host == server { - return _responses.BadRequest(errors.New("Attempted to clone media to the same origin")) + return responses.BadRequest(errors.New("Attempted to clone media to the same origin")) } // TODO: There's a lot of room for improvement here. Instead of re-uploading media, we should just update the DB. @@ -65,33 +65,33 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI // Error handling copied from download endpoint if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } record, err = pipeline_upload.Execute(rctx, server, mediaId, stream, record.ContentType, record.UploadName, user.UserId, datastores.LocalMediaKind) // Error handling copied from upload(sync) endpoint if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("Unexpected Error")) + return responses.InternalServerError(errors.New("Unexpected Error")) } return &r0.MediaUploadedResponse{ diff --git a/api/v1/create.go b/api/v1/create.go index 1aebac0d..592212cd 100644 --- a/api/v1/create.go +++ b/api/v1/create.go @@ -5,8 +5,8 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_responses" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_create" "github.com/t2bot/matrix-media-repo/util" @@ -22,7 +22,7 @@ func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.Use if err != nil { rctx.Log.Error("Unexpected error creating media ID:", err) sentry.CaptureException(err) - return _responses.InternalServerError(errors.New("unexpected error")) + return responses.InternalServerError(errors.New("unexpected error")) } return &MediaCreatedResponse{ diff --git a/api/webserver.go b/api/webserver.go index 25c761bc..fb0a4ba9 100644 --- a/api/webserver.go +++ b/api/webserver.go @@ -14,7 +14,7 @@ import ( "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" ) @@ -37,7 +37,7 @@ func Init() *sync.WaitGroup { limiter.SetBurst(config.Get().RateLimit.BurstCount) limiter.SetMax(config.Get().RateLimit.RequestsPerSecond) - reponse, _ := json.Marshal(_responses.RateLimitReached()) + reponse, _ := json.Marshal(responses.RateLimitReached()) limiter.SetMessage(string(reponse)) limiter.SetMessageContentType("application/json") diff --git a/metrics/metrics.go b/metrics/metrics.go index 4f597ee4..93467a47 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,7 +11,7 @@ var InvalidHttpRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "media_invalid_http_requests_total", }, []string{"action", "method"}) var HttpResponses = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "media_http_responses_total", + Name: "media_httpresponses_total", }, []string{"host", "action", "method", "statusCode"}) var HttpResponseTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "media_http_response_time_seconds", From a6b6598ef67265ba97caefdaf92193b7faf19b08 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:12:07 +0100 Subject: [PATCH 17/22] move api/_routers to api/routers --- api/branched_route.go | 6 +- api/custom/datastores.go | 8 +- api/custom/exports.go | 24 ++-- api/custom/federation.go | 6 +- api/custom/imports.go | 10 +- api/custom/media_attributes.go | 14 +-- api/custom/purge.go | 16 +-- api/custom/quarantine.go | 16 +-- api/custom/tasks.go | 4 +- api/custom/usage.go | 18 +-- api/r0/download.go | 10 +- api/r0/identicon.go | 4 +- api/r0/thumbnail.go | 8 +- api/r0/upload_async.go | 6 +- api/router.go | 4 +- .../00-install-params.go | 2 +- .../01-install_metadata.go | 2 +- .../02-install-headers.go | 2 +- .../03-host_detection.go | 2 +- .../04-request-metrics.go | 2 +- .../97-optional-access-token.go | 2 +- .../97-require-access-token.go | 2 +- .../97-require-repo-admin.go | 2 +- api/{_routers => routers}/98-use-rcontext.go | 2 +- .../99-response-metrics.go | 2 +- api/routes.go | 112 +++++++++--------- api/unstable/info.go | 8 +- api/unstable/local_copy.go | 8 +- 28 files changed, 151 insertions(+), 151 deletions(-) rename api/{_routers => routers}/00-install-params.go (97%) rename api/{_routers => routers}/01-install_metadata.go (99%) rename api/{_routers => routers}/02-install-headers.go (98%) rename api/{_routers => routers}/03-host_detection.go (99%) rename api/{_routers => routers}/04-request-metrics.go (97%) rename api/{_routers => routers}/97-optional-access-token.go (99%) rename api/{_routers => routers}/97-require-access-token.go (99%) rename api/{_routers => routers}/97-require-repo-admin.go (97%) rename api/{_routers => routers}/98-use-rcontext.go (99%) rename api/{_routers => routers}/99-response-metrics.go (97%) diff --git a/api/branched_route.go b/api/branched_route.go index 04090e30..897a1c0a 100644 --- a/api/branched_route.go +++ b/api/branched_route.go @@ -4,7 +4,7 @@ import ( "net/http" "strings" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/routers" ) type branch struct { @@ -26,7 +26,7 @@ func branchedRoute(branches []branch) http.Handler { } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - catchAll := _routers.GetParam("branch", r) + catchAll := routers.GetParam("branch", r) if catchAll[0] == '/' { catchAll = catchAll[1:] } @@ -38,7 +38,7 @@ func branchedRoute(branches []branch) http.Handler { } for i, segment := range branch.segments { if segment[0] == ':' { - r = _routers.ForceSetParam(segment[1:], params[i], r) + r = routers.ForceSetParam(segment[1:], params[i], r) } } branch.handler.ServeHTTP(w, r) diff --git a/api/custom/datastores.go b/api/custom/datastores.go index f15455c2..52e76363 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -53,8 +53,8 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use } } - sourceDsId := _routers.GetParam("sourceDsId", r) - targetDsId := _routers.GetParam("targetDsId", r) + sourceDsId := routers.GetParam("sourceDsId", r) + targetDsId := routers.GetParam("targetDsId", r) rctx = rctx.LogWithFields(logrus.Fields{ "beforeTs": beforeTs, @@ -106,7 +106,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, } } - datastoreId := _routers.GetParam("datastoreId", r) + datastoreId := routers.GetParam("datastoreId", r) rctx = rctx.LogWithFields(logrus.Fields{ "beforeTs": beforeTs, diff --git a/api/custom/exports.go b/api/custom/exports.go index a65b796b..1b6e90cd 100644 --- a/api/custom/exports.go +++ b/api/custom/exports.go @@ -7,9 +7,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -50,7 +50,7 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta. s3urls := r.URL.Query().Get("s3_urls") != "false" - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) if !isAdmin && user.UserId != userId { return responses.BadRequest(errors.New("cannot export data for another user")) @@ -85,7 +85,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimet s3urls := r.URL.Query().Get("s3_urls") != "false" - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) if !isAdmin { // They might be a local admin, so check that. @@ -127,9 +127,9 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.User return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { + if !routers.ServerNameRegex.MatchString(exportId) { responses.BadRequest(errors.New("invalid export ID")) } @@ -195,9 +195,9 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apime return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { + if !routers.ServerNameRegex.MatchString(exportId) { responses.BadRequest(errors.New("invalid export ID")) } @@ -242,10 +242,10 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apim return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) - pid := _routers.GetParam("partId", r) + exportId := routers.GetParam("exportId", r) + pid := routers.GetParam("partId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { + if !routers.ServerNameRegex.MatchString(exportId) { responses.BadRequest(errors.New("invalid export ID")) } @@ -298,9 +298,9 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { + if !routers.ServerNameRegex.MatchString(exportId) { responses.BadRequest(errors.New("invalid export ID")) } diff --git a/api/custom/federation.go b/api/custom/federation.go index 5ad0522f..ae70dffd 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -6,9 +6,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -16,9 +16,9 @@ import ( ) func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } diff --git a/api/custom/imports.go b/api/custom/imports.go index 1b2e2153..700a5b1e 100644 --- a/api/custom/imports.go +++ b/api/custom/imports.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/tasks" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -57,9 +57,9 @@ func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user apimeta. return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } - importId := _routers.GetParam("importId", r) + importId := routers.GetParam("importId", r) - if !_routers.ServerNameRegex.MatchString(importId) { + if !routers.ServerNameRegex.MatchString(importId) { return responses.BadRequest(errors.New("invalid import ID")) } @@ -84,9 +84,9 @@ func StopImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.User return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } - importId := _routers.GetParam("importId", r) + importId := routers.GetParam("importId", r) - if !_routers.ServerNameRegex.MatchString(importId) { + if !routers.ServerNameRegex.MatchString(importId) { return responses.BadRequest(errors.New("invalid import ID")) } diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go index 5203db5e..6b3eb5df 100644 --- a/api/custom/media_attributes.go +++ b/api/custom/media_attributes.go @@ -7,9 +7,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/matrix" @@ -34,10 +34,10 @@ func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin s } func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - origin := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + origin := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(origin) { + if !routers.ServerNameRegex.MatchString(origin) { return responses.BadRequest(errors.New("invalid origin")) } @@ -80,10 +80,10 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.U } func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - origin := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + origin := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(origin) { + if !routers.ServerNameRegex.MatchString(origin) { return responses.BadRequest(errors.New("invalid origin")) } diff --git a/api/custom/purge.go b/api/custom/purge.go index a983cb93..59d89cd1 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -8,9 +8,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -54,10 +54,10 @@ func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user apimet func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, _, _ := getPurgeAuthContext(rctx, r, user) - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } @@ -189,7 +189,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. } } - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) rctx = rctx.LogWithFields(logrus.Fields{ "userId": userId, @@ -246,7 +246,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta. } } - roomId := _routers.GetParam("roomId", r) + roomId := routers.GetParam("roomId", r) rctx = rctx.LogWithFields(logrus.Fields{ "roomId": roomId, @@ -319,9 +319,9 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimet } } - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go index d59d10a8..22b8fab0 100644 --- a/api/custom/quarantine.go +++ b/api/custom/quarantine.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -27,7 +27,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user api return responses.AuthFailed() } - roomId := _routers.GetParam("roomId", r) + roomId := routers.GetParam("roomId", r) rctx = rctx.LogWithFields(logrus.Fields{ "roomId": roomId, @@ -58,7 +58,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user api return responses.AuthFailed() } - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) rctx = rctx.LogWithFields(logrus.Fields{ "userId": userId, @@ -95,9 +95,9 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user a return responses.AuthFailed() } - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } @@ -129,10 +129,10 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta return responses.AuthFailed() } - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } diff --git a/api/custom/tasks.go b/api/custom/tasks.go index 0b7f0ae5..eb60c70a 100644 --- a/api/custom/tasks.go +++ b/api/custom/tasks.go @@ -6,9 +6,9 @@ import ( "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/sirupsen/logrus" @@ -26,7 +26,7 @@ type TaskStatus struct { } func GetTask(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - taskIdStr := _routers.GetParam("taskId", r) + taskIdStr := routers.GetParam("taskId", r) taskId, err := strconv.Atoi(taskIdStr) if err != nil { rctx.Log.Error(err) diff --git a/api/custom/usage.go b/api/custom/usage.go index e4640d03..e14e8e6f 100644 --- a/api/custom/usage.go +++ b/api/custom/usage.go @@ -8,9 +8,9 @@ import ( "strings" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -55,9 +55,9 @@ type MediaUsageEntry struct { } func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } @@ -102,10 +102,10 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta. } func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) userIds := r.URL.Query()["user_id"] - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } @@ -161,10 +161,10 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.Us } func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) mxcs := r.URL.Query()["mxc"] - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } @@ -228,12 +228,12 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user a qs := r.URL.Query() var err error - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) if serverName == "" && strings.HasPrefix(r.URL.Path, synapse.PrefixAdminApi) { serverName = r.Host } - if !_routers.ServerNameRegex.MatchString(serverName) { + if !routers.ServerNameRegex.MatchString(serverName) { return responses.BadRequest(errors.New("invalid server name")) } diff --git a/api/r0/download.go b/api/r0/download.go index e8c3bb7f..03e6d80e 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/util" @@ -20,14 +20,14 @@ import ( ) func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) - filename := _routers.GetParam("filename", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) + filename := routers.GetParam("filename", r) allowRemote := r.URL.Query().Get("allow_remote") allowRedirect := r.URL.Query().Get("allow_redirect") timeoutMs := r.URL.Query().Get("timeout_ms") - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } diff --git a/api/r0/identicon.go b/api/r0/identicon.go index cd76c9b8..3f61ceba 100644 --- a/api/r0/identicon.go +++ b/api/r0/identicon.go @@ -11,9 +11,9 @@ import ( "github.com/cupcake/sigil/gen" "github.com/disintegration/imaging" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -22,7 +22,7 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserI return responses.NotFoundError() } - seed := _routers.GetParam("seed", r) + seed := routers.GetParam("seed", r) var err error width := 96 diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 49565dc5..17bda434 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -7,9 +7,9 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" @@ -22,13 +22,13 @@ import ( ) func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") allowRedirect := r.URL.Query().Get("allow_redirect") timeoutMs := r.URL.Query().Get("timeout_ms") - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go index ab60047f..01d50f8e 100644 --- a/api/r0/upload_async.go +++ b/api/r0/upload_async.go @@ -7,17 +7,17 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_upload" ) func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) filename := filepath.Base(r.URL.Query().Get("filename")) rctx = rctx.LogWithFields(logrus.Fields{ diff --git a/api/router.go b/api/router.go index 5405ec1f..540c4d9b 100644 --- a/api/router.go +++ b/api/router.go @@ -9,8 +9,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/util" ) @@ -21,7 +21,7 @@ func buildPrimaryRouter() *httprouter.Router { router.MethodNotAllowed = http.HandlerFunc(methodNotAllowedFn) router.NotFound = http.HandlerFunc(notFoundFn) router.HandleOPTIONS = true - router.GlobalOPTIONS = _routers.NewInstallHeadersRouter(http.HandlerFunc(finishCorsFn)) + router.GlobalOPTIONS = routers.NewInstallHeadersRouter(http.HandlerFunc(finishCorsFn)) router.PanicHandler = panicFn return router } diff --git a/api/_routers/00-install-params.go b/api/routers/00-install-params.go similarity index 97% rename from api/_routers/00-install-params.go rename to api/routers/00-install-params.go index b289ed63..6f75903c 100644 --- a/api/_routers/00-install-params.go +++ b/api/routers/00-install-params.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "context" diff --git a/api/_routers/01-install_metadata.go b/api/routers/01-install_metadata.go similarity index 99% rename from api/_routers/01-install_metadata.go rename to api/routers/01-install_metadata.go index 10ab4d04..fe5a4688 100644 --- a/api/_routers/01-install_metadata.go +++ b/api/routers/01-install_metadata.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "context" diff --git a/api/_routers/02-install-headers.go b/api/routers/02-install-headers.go similarity index 98% rename from api/_routers/02-install-headers.go rename to api/routers/02-install-headers.go index 8d3547ce..ad18b549 100644 --- a/api/_routers/02-install-headers.go +++ b/api/routers/02-install-headers.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/_routers/03-host_detection.go b/api/routers/03-host_detection.go similarity index 99% rename from api/_routers/03-host_detection.go rename to api/routers/03-host_detection.go index fdf8519d..eb6a105c 100644 --- a/api/_routers/03-host_detection.go +++ b/api/routers/03-host_detection.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "context" diff --git a/api/_routers/04-request-metrics.go b/api/routers/04-request-metrics.go similarity index 97% rename from api/_routers/04-request-metrics.go rename to api/routers/04-request-metrics.go index 59c1b24a..9de5dac1 100644 --- a/api/_routers/04-request-metrics.go +++ b/api/routers/04-request-metrics.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/_routers/97-optional-access-token.go b/api/routers/97-optional-access-token.go similarity index 99% rename from api/_routers/97-optional-access-token.go rename to api/routers/97-optional-access-token.go index a2376cea..69939f17 100644 --- a/api/_routers/97-optional-access-token.go +++ b/api/routers/97-optional-access-token.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "errors" diff --git a/api/_routers/97-require-access-token.go b/api/routers/97-require-access-token.go similarity index 99% rename from api/_routers/97-require-access-token.go rename to api/routers/97-require-access-token.go index ea1aeb6c..bbff8f37 100644 --- a/api/_routers/97-require-access-token.go +++ b/api/routers/97-require-access-token.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "errors" diff --git a/api/_routers/97-require-repo-admin.go b/api/routers/97-require-repo-admin.go similarity index 97% rename from api/_routers/97-require-repo-admin.go rename to api/routers/97-require-repo-admin.go index 95fca9ef..0b1d3916 100644 --- a/api/_routers/97-require-repo-admin.go +++ b/api/routers/97-require-repo-admin.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/_routers/98-use-rcontext.go b/api/routers/98-use-rcontext.go similarity index 99% rename from api/_routers/98-use-rcontext.go rename to api/routers/98-use-rcontext.go index 25497112..fe385e1a 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/routers/98-use-rcontext.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "bytes" diff --git a/api/_routers/99-response-metrics.go b/api/routers/99-response-metrics.go similarity index 97% rename from api/_routers/99-response-metrics.go rename to api/routers/99-response-metrics.go index 6fcc1d90..4839a09d 100644 --- a/api/_routers/99-response-metrics.go +++ b/api/routers/99-response-metrics.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/routes.go b/api/routes.go index 423a6e32..6aa4a6b1 100644 --- a/api/routes.go +++ b/api/routes.go @@ -8,9 +8,9 @@ import ( "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api/_debug" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/custom" "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/api/unstable" v1 "github.com/t2bot/matrix-media-repo/api/v1" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -20,7 +20,7 @@ const PrefixMedia = "/_matrix/media" const PrefixClient = "/_matrix/client" func buildRoutes() http.Handler { - counter := &_routers.RequestCounter{} + counter := &routers.RequestCounter{} router := buildPrimaryRouter() pprofSecret := os.Getenv("MEDIA_PPROF_SECRET_KEY") @@ -30,94 +30,94 @@ func buildRoutes() http.Handler { } // Standard (spec) features - register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter)) - register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter)) - downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMedia), "download", counter) + register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter)) + register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter)) + downloadRoute := makeRoute(routers.OptionalAccessToken(r0.DownloadMedia), "download", counter) register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute) register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute) - register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) - register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)) - register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter)) - register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter)) - register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", counter)) - register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter)) - register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.CreateMedia), "create", counter)) + register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) + register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)) + register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(routers.OptionalAccessToken(r0.Identicon), "identicon", counter)) + register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.PublicConfig), "config", counter)) + register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.Logout), "logout", counter)) + register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter)) + register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(routers.RequireAccessToken(v1.CreateMedia), "create", counter)) // Custom features - register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter)) - register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.MediaInfo), "info", counter)) - purgeOneRoute := makeRoute(_routers.RequireAccessToken(custom.PurgeIndividualRecord), "purge_individual_media", counter) + register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter)) + register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(routers.RequireAccessToken(unstable.MediaInfo), "info", counter)) + purgeOneRoute := makeRoute(routers.RequireAccessToken(custom.PurgeIndividualRecord), "purge_individual_media", counter) register([]string{"DELETE"}, PrefixMedia, "download/:server/:mediaId", mxUnstable, router, purgeOneRoute) - register([]string{"GET"}, PrefixMedia, "usage", msc4034, router, makeRoute(_routers.RequireAccessToken(unstable.PublicUsage), "usage", counter)) + register([]string{"GET"}, PrefixMedia, "usage", msc4034, router, makeRoute(routers.RequireAccessToken(unstable.PublicUsage), "usage", counter)) // Custom and top-level features - router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(_routers.OptionalAccessToken(custom.GetVersion), "get_version", counter)) - healthzRoute := makeRoute(_routers.OptionalAccessToken(custom.GetHealthz), "healthz", counter) // Note: healthz handling is special in makeRoute() + router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(routers.OptionalAccessToken(custom.GetVersion), "get_version", counter)) + healthzRoute := makeRoute(routers.OptionalAccessToken(custom.GetHealthz), "healthz", counter) // Note: healthz handling is special in makeRoute() router.Handler("GET", "/healthz", healthzRoute) router.Handler("HEAD", "/healthz", healthzRoute) // Register the Synapse admin API endpoints we're compatible with - synUserStatsRoute := makeRoute(_routers.RequireAccessToken(custom.SynGetUsersMediaStats), "users_usage_stats", counter) + synUserStatsRoute := makeRoute(routers.RequireAccessToken(custom.SynGetUsersMediaStats), "users_usage_stats", counter) register([]string{"GET"}, synapse.PrefixAdminApi, "statistics/users/media", mxV1, router, synUserStatsRoute) // All admin routes are unstable only - purgeRemoteRoute := makeRoute(_routers.RequireRepoAdmin(custom.PurgeRemoteMedia), "purge_remote_media", counter) + purgeRemoteRoute := makeRoute(routers.RequireRepoAdmin(custom.PurgeRemoteMedia), "purge_remote_media", counter) purgeBranch := branchedRoute([]branch{ {"remote", purgeRemoteRoute}, - {"old", makeRoute(_routers.RequireRepoAdmin(custom.PurgeOldMedia), "purge_old_media", counter)}, - {"quarantined", makeRoute(_routers.RequireAccessToken(custom.PurgeQuarantined), "purge_quarantined", counter)}, - {"user/:userId", makeRoute(_routers.RequireAccessToken(custom.PurgeUserMedia), "purge_user_media", counter)}, - {"room/:roomId", makeRoute(_routers.RequireAccessToken(custom.PurgeRoomMedia), "purge_room_media", counter)}, - {"server/:serverName", makeRoute(_routers.RequireAccessToken(custom.PurgeDomainMedia), "purge_domain_media", counter)}, + {"old", makeRoute(routers.RequireRepoAdmin(custom.PurgeOldMedia), "purge_old_media", counter)}, + {"quarantined", makeRoute(routers.RequireAccessToken(custom.PurgeQuarantined), "purge_quarantined", counter)}, + {"user/:userId", makeRoute(routers.RequireAccessToken(custom.PurgeUserMedia), "purge_user_media", counter)}, + {"room/:roomId", makeRoute(routers.RequireAccessToken(custom.PurgeRoomMedia), "purge_room_media", counter)}, + {"server/:serverName", makeRoute(routers.RequireAccessToken(custom.PurgeDomainMedia), "purge_domain_media", counter)}, {":server/:mediaId", purgeOneRoute}, }) register([]string{"POST"}, PrefixMedia, "admin/purge/*branch", mxUnstable, router, purgeBranch) register([]string{"POST"}, PrefixMedia, "admin/purge_remote", mxUnstable, router, purgeRemoteRoute) register([]string{"POST"}, PrefixClient, "admin/purge_media_cache", mxUnstable, router, purgeRemoteRoute) // synapse compat - quarantineRoomRoute := makeRoute(_routers.RequireAccessToken(custom.QuarantineRoomMedia), "quarantine_room", counter) + quarantineRoomRoute := makeRoute(routers.RequireAccessToken(custom.QuarantineRoomMedia), "quarantine_room", counter) quarantineBranch := branchedRoute([]branch{ {"room/:roomId", quarantineRoomRoute}, - {"user/:userId", makeRoute(_routers.RequireAccessToken(custom.QuarantineUserMedia), "quarantine_user", counter)}, - {"server/:serverName", makeRoute(_routers.RequireAccessToken(custom.QuarantineDomainMedia), "quarantine_domain", counter)}, - {":server/:mediaId", makeRoute(_routers.RequireAccessToken(custom.QuarantineMedia), "quarantine_media", counter)}, + {"user/:userId", makeRoute(routers.RequireAccessToken(custom.QuarantineUserMedia), "quarantine_user", counter)}, + {"server/:serverName", makeRoute(routers.RequireAccessToken(custom.QuarantineDomainMedia), "quarantine_domain", counter)}, + {":server/:mediaId", makeRoute(routers.RequireAccessToken(custom.QuarantineMedia), "quarantine_media", counter)}, }) register([]string{"POST"}, PrefixMedia, "admin/quarantine/*branch", mxUnstable, router, quarantineBranch) register([]string{"POST"}, PrefixClient, "admin/quarantine_media/:roomId", mxUnstable, router, quarantineRoomRoute) // synapse compat - register([]string{"GET"}, PrefixMedia, "admin/datastores/:datastoreId/size_estimate", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastoreStorageEstimate), "get_storage_estimate", counter)) - register([]string{"POST"}, PrefixMedia, "admin/datastores/:sourceDsId/transfer_to/:targetDsId", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.MigrateBetweenDatastores), "datastore_transfer", counter)) - register([]string{"GET"}, PrefixMedia, "admin/datastores", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastores), "list_datastores", counter)) - register([]string{"GET"}, PrefixMedia, "admin/federation/test/:serverName", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetFederationInfo), "federation_test", counter)) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDomainUsage), "domain_usage", counter)) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUserUsage), "user_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/datastores/:datastoreId/size_estimate", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDatastoreStorageEstimate), "get_storage_estimate", counter)) + register([]string{"POST"}, PrefixMedia, "admin/datastores/:sourceDsId/transfer_to/:targetDsId", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.MigrateBetweenDatastores), "datastore_transfer", counter)) + register([]string{"GET"}, PrefixMedia, "admin/datastores", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDatastores), "list_datastores", counter)) + register([]string{"GET"}, PrefixMedia, "admin/federation/test/:serverName", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetFederationInfo), "federation_test", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDomainUsage), "domain_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetUserUsage), "user_usage", counter)) register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users-stats", mxUnstable, router, synUserStatsRoute) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/uploads", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUploadsUsage), "uploads_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/uploads", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetUploadsUsage), "uploads_usage", counter)) tasksBranch := branchedRoute([]branch{ - {"all", makeRoute(_routers.RequireRepoAdmin(custom.ListAllTasks), "list_all_background_tasks", counter)}, - {"unfinished", makeRoute(_routers.RequireRepoAdmin(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", counter)}, - {":taskId", makeRoute(_routers.RequireRepoAdmin(custom.GetTask), "get_background_task", counter)}, + {"all", makeRoute(routers.RequireRepoAdmin(custom.ListAllTasks), "list_all_background_tasks", counter)}, + {"unfinished", makeRoute(routers.RequireRepoAdmin(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", counter)}, + {":taskId", makeRoute(routers.RequireRepoAdmin(custom.GetTask), "get_background_task", counter)}, }) register([]string{"GET"}, PrefixMedia, "admin/tasks/*branch", mxUnstable, router, tasksBranch) - register([]string{"POST"}, PrefixMedia, "admin/user/:userId/export", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.ExportUserData), "export_user_data", counter)) - register([]string{"POST"}, PrefixMedia, "admin/server/:serverName/export", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.ExportServerData), "export_server_data", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/view", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.ViewExport), "view_export", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/metadata", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.GetExportMetadata), "get_export_metadata", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/part/:partId", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.DownloadExportPart), "download_export_part", counter)) - register([]string{"DELETE"}, PrefixMedia, "admin/export/:exportId", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.DeleteExport), "delete_export", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.StartImport), "start_import", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import/:importId/part", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.AppendToImport), "append_to_import", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import/:importId/close", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.StopImport), "stop_import", counter)) - register([]string{"GET"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.GetAttributes), "get_media_attributes", counter)) - register([]string{"POST"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.SetAttributes), "set_media_attributes", counter)) + register([]string{"POST"}, PrefixMedia, "admin/user/:userId/export", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.ExportUserData), "export_user_data", counter)) + register([]string{"POST"}, PrefixMedia, "admin/server/:serverName/export", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.ExportServerData), "export_server_data", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/view", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.ViewExport), "view_export", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/metadata", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.GetExportMetadata), "get_export_metadata", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/part/:partId", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.DownloadExportPart), "download_export_part", counter)) + register([]string{"DELETE"}, PrefixMedia, "admin/export/:exportId", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.DeleteExport), "delete_export", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.StartImport), "start_import", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import/:importId/part", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.AppendToImport), "append_to_import", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import/:importId/close", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.StopImport), "stop_import", counter)) + register([]string{"GET"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.GetAttributes), "get_media_attributes", counter)) + register([]string{"POST"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.SetAttributes), "set_media_attributes", counter)) return router } -func makeRoute(generator _routers.GeneratorFn, name string, counter *_routers.RequestCounter) http.Handler { - return _routers.NewInstallMetadataRouter(name == "healthz", name, counter, - _routers.NewInstallHeadersRouter( - _routers.NewHostRouter( - _routers.NewMetricsRequestRouter( - _routers.NewRContextRouter(generator, _routers.NewMetricsResponseRouter(nil)), +func makeRoute(generator routers.GeneratorFn, name string, counter *routers.RequestCounter) http.Handler { + return routers.NewInstallMetadataRouter(name == "healthz", name, counter, + routers.NewInstallHeadersRouter( + routers.NewHostRouter( + routers.NewMetricsRequestRouter( + routers.NewRContextRouter(generator, routers.NewMetricsResponseRouter(nil)), ), ), )) diff --git a/api/unstable/info.go b/api/unstable/info.go index 1ea980b9..951b023a 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -10,9 +10,9 @@ import ( "github.com/disintegration/imaging" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" @@ -49,11 +49,11 @@ type MediaInfoResponse struct { } func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go index 1a427716..ed7b810d 100644 --- a/api/unstable/local_copy.go +++ b/api/unstable/local_copy.go @@ -8,10 +8,10 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/r0" "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -21,13 +21,13 @@ import ( ) func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") rctx.Log.Warn("This endpoint is deprecated. See https://github.com/t2bot/matrix-media-repo/issues/422") - if !_routers.ServerNameRegex.MatchString(server) { + if !routers.ServerNameRegex.MatchString(server) { return responses.BadRequest(errors.New("invalid server ID")) } From 6fc49c274645d246c1168dad8a2507f26aa35f3d Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:14:59 +0100 Subject: [PATCH 18/22] move api/_auth_cache to api/auth_cache --- api/{_auth_cache => auth_cache}/auth_cache.go | 2 +- api/r0/logout.go | 6 +++--- api/routers/97-optional-access-token.go | 4 ++-- api/routers/97-require-access-token.go | 4 ++-- cmd/workers/media_repo/reloads.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) rename api/{_auth_cache => auth_cache}/auth_cache.go (99%) diff --git a/api/_auth_cache/auth_cache.go b/api/auth_cache/auth_cache.go similarity index 99% rename from api/_auth_cache/auth_cache.go rename to api/auth_cache/auth_cache.go index f3721496..a306983a 100644 --- a/api/_auth_cache/auth_cache.go +++ b/api/auth_cache/auth_cache.go @@ -1,4 +1,4 @@ -package _auth_cache +package auth_cache import ( "errors" diff --git a/api/r0/logout.go b/api/r0/logout.go index fa284aed..3709cfca 100644 --- a/api/r0/logout.go +++ b/api/r0/logout.go @@ -8,12 +8,12 @@ import ( "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/responses" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/common/rcontext" ) func Logout(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - err := _auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId) + err := auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) @@ -23,7 +23,7 @@ func Logout(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo } func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { - err := _auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId) + err := auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) diff --git a/api/routers/97-optional-access-token.go b/api/routers/97-optional-access-token.go index 69939f17..5dc7bf6d 100644 --- a/api/routers/97-optional-access-token.go +++ b/api/routers/97-optional-access-token.go @@ -6,8 +6,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -34,7 +34,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { }) } appserviceUserId := util.GetAppserviceUserIdFromRequest(r) - userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId) + userId, err := auth_cache.GetUserId(ctx, accessToken, appserviceUserId) if err != nil { if !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) diff --git a/api/routers/97-require-access-token.go b/api/routers/97-require-access-token.go index bbff8f37..f01b6e6e 100644 --- a/api/routers/97-require-access-token.go +++ b/api/routers/97-require-access-token.go @@ -6,8 +6,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" @@ -37,7 +37,7 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { }) } appserviceUserId := util.GetAppserviceUserIdFromRequest(r) - userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId) + userId, err := auth_cache.GetUserId(ctx, accessToken, appserviceUserId) if err != nil || userId == "" { if errors.Is(err, matrix.ErrGuestToken) { return responses.GuestAuthFailed() diff --git a/cmd/workers/media_repo/reloads.go b/cmd/workers/media_repo/reloads.go index f469b718..57099dfd 100644 --- a/cmd/workers/media_repo/reloads.go +++ b/cmd/workers/media_repo/reloads.go @@ -3,7 +3,7 @@ package main import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/globals" "github.com/t2bot/matrix-media-repo/common/runtime" @@ -136,7 +136,7 @@ func reloadAccessTokensOnChan(reloadChan chan bool) { for { shouldReload := <-reloadChan if shouldReload { - _auth_cache.FlushCache() + auth_cache.FlushCache() } else { return // received stop } From dc277e9ce665a5a53d8bcab14d205512893fda0e Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:15:55 +0100 Subject: [PATCH 19/22] move api/_debug to api/debug --- api/{_debug => debug}/pprof.go | 2 +- api/routes.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename api/{_debug => debug}/pprof.go (99%) diff --git a/api/_debug/pprof.go b/api/debug/pprof.go similarity index 99% rename from api/_debug/pprof.go rename to api/debug/pprof.go index f2942dc8..e5d7ca58 100644 --- a/api/_debug/pprof.go +++ b/api/debug/pprof.go @@ -1,4 +1,4 @@ -package _debug +package debug import ( "encoding/json" diff --git a/api/routes.go b/api/routes.go index 6aa4a6b1..100ccaae 100644 --- a/api/routes.go +++ b/api/routes.go @@ -7,8 +7,8 @@ import ( "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_debug" "github.com/t2bot/matrix-media-repo/api/custom" + "github.com/t2bot/matrix-media-repo/api/debug" "github.com/t2bot/matrix-media-repo/api/r0" "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/api/unstable" @@ -26,7 +26,7 @@ func buildRoutes() http.Handler { pprofSecret := os.Getenv("MEDIA_PPROF_SECRET_KEY") if pprofSecret != "" { logrus.Warn("Enabling pprof/debug http endpoints") - _debug.BindPprofEndpoints(router, pprofSecret) + debug.BindPprofEndpoints(router, pprofSecret) } // Standard (spec) features From 1f3380e8660e2532e9521a7a846e3520c5358516 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:19:12 +0100 Subject: [PATCH 20/22] move cmd/homeserver_live_importers/_common to cmd/homeserver_live_importers/internal --- cmd/homeserver_live_importers/import_dendrite/main.go | 8 ++++---- cmd/homeserver_live_importers/import_synapse/main.go | 8 ++++---- .../{_common => internal}/download.go | 2 +- .../{_common => internal}/init_import.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) rename cmd/homeserver_live_importers/{_common => internal}/download.go (99%) rename cmd/homeserver_live_importers/{_common => internal}/init_import.go (99%) diff --git a/cmd/homeserver_live_importers/import_dendrite/main.go b/cmd/homeserver_live_importers/import_dendrite/main.go index cae20c32..bcb44112 100644 --- a/cmd/homeserver_live_importers/import_dendrite/main.go +++ b/cmd/homeserver_live_importers/import_dendrite/main.go @@ -2,13 +2,13 @@ package main import ( "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/dendrite" ) func main() { - cfg := _common.InitImportPsqlMatrixDownload("Dendrite") + cfg := internal.InitImportPsqlMatrixDownload("Dendrite") ctx := rcontext.Initial() ctx.Log.Debug("Connecting to homeserver database...") @@ -17,8 +17,8 @@ func main() { logrus.Fatalf("Failed to open database: %v", err) } - _common.PsqlMatrixDownloadCopy[dendrite.LocalMedia](ctx, cfg, hsDb, func(record *dendrite.LocalMedia) (*_common.MediaMetadata, error) { - return &_common.MediaMetadata{ + internal.PsqlMatrixDownloadCopy[dendrite.LocalMedia](ctx, cfg, hsDb, func(record *dendrite.LocalMedia) (*internal.MediaMetadata, error) { + return &internal.MediaMetadata{ MediaId: record.MediaId, ContentType: record.ContentType, FileName: record.UploadName, diff --git a/cmd/homeserver_live_importers/import_synapse/main.go b/cmd/homeserver_live_importers/import_synapse/main.go index 8edbcd69..b25c1182 100644 --- a/cmd/homeserver_live_importers/import_synapse/main.go +++ b/cmd/homeserver_live_importers/import_synapse/main.go @@ -2,13 +2,13 @@ package main import ( "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" ) func main() { - cfg := _common.InitImportPsqlMatrixDownload("Synapse") + cfg := internal.InitImportPsqlMatrixDownload("Synapse") ctx := rcontext.Initial() ctx.Log.Debug("Connecting to homeserver database...") @@ -17,8 +17,8 @@ func main() { logrus.Fatalf("Failed to open database: %v", err) } - _common.PsqlMatrixDownloadCopy[synapse.LocalMedia](ctx, cfg, hsDb, func(record *synapse.LocalMedia) (*_common.MediaMetadata, error) { - return &_common.MediaMetadata{ + internal.PsqlMatrixDownloadCopy[synapse.LocalMedia](ctx, cfg, hsDb, func(record *synapse.LocalMedia) (*internal.MediaMetadata, error) { + return &internal.MediaMetadata{ MediaId: record.MediaId, ContentType: record.ContentType, FileName: record.UploadName, diff --git a/cmd/homeserver_live_importers/_common/download.go b/cmd/homeserver_live_importers/internal/download.go similarity index 99% rename from cmd/homeserver_live_importers/_common/download.go rename to cmd/homeserver_live_importers/internal/download.go index 2618784b..5e811875 100644 --- a/cmd/homeserver_live_importers/_common/download.go +++ b/cmd/homeserver_live_importers/internal/download.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "errors" diff --git a/cmd/homeserver_live_importers/_common/init_import.go b/cmd/homeserver_live_importers/internal/init_import.go similarity index 99% rename from cmd/homeserver_live_importers/_common/init_import.go rename to cmd/homeserver_live_importers/internal/init_import.go index 7b025157..5a8ffa95 100644 --- a/cmd/homeserver_live_importers/_common/init_import.go +++ b/cmd/homeserver_live_importers/internal/init_import.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag" From a31465bd278a5df9d7382916e0ec42c79465f1df Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:22:51 +0100 Subject: [PATCH 21/22] move cmd/homeserver_offline_importers/_common to cmd/homeserver_offline_importers/internal --- .../export_dendrite_for_import/main.go | 6 +++--- .../export_synapse_for_import/main.go | 6 +++--- .../{_common => internal}/archiver.go | 2 +- .../{_common => internal}/init_export.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) rename cmd/homeserver_offline_importers/{_common => internal}/archiver.go (98%) rename cmd/homeserver_offline_importers/{_common => internal}/init_export.go (99%) diff --git a/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go b/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go index 621702a6..201df83a 100644 --- a/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go +++ b/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go @@ -6,14 +6,14 @@ import ( "path" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/dendrite" "github.com/t2bot/matrix-media-repo/util" ) func main() { - cfg := _common.InitExportPsqlFlatFile("Dendrite", "media_api.base_path") + cfg := internal.InitExportPsqlFlatFile("Dendrite", "media_api.base_path") ctx := rcontext.InitialNoConfig() ctx.Log.Debug("Connecting to homeserver database...") @@ -22,7 +22,7 @@ func main() { panic(err) } - _common.PsqlFlatFileArchive[dendrite.LocalMedia](ctx, cfg, hsDb, func(r *dendrite.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { + internal.PsqlFlatFileArchive[dendrite.LocalMedia](ctx, cfg, hsDb, func(r *dendrite.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { // For Base64Hash ABCCDD : // $importPath/A/B/CCDD/file diff --git a/cmd/homeserver_offline_importers/export_synapse_for_import/main.go b/cmd/homeserver_offline_importers/export_synapse_for_import/main.go index 6c8cfb3c..6f7839ec 100644 --- a/cmd/homeserver_offline_importers/export_synapse_for_import/main.go +++ b/cmd/homeserver_offline_importers/export_synapse_for_import/main.go @@ -7,14 +7,14 @@ import ( "strings" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" "github.com/t2bot/matrix-media-repo/util" ) func main() { - cfg := _common.InitExportPsqlFlatFile("Synapse", "media_store_path") + cfg := internal.InitExportPsqlFlatFile("Synapse", "media_store_path") ctx := rcontext.InitialNoConfig() ctx.Log.Debug("Connecting to homeserver database...") @@ -23,7 +23,7 @@ func main() { panic(err) } - _common.PsqlFlatFileArchive[synapse.LocalMedia](ctx, cfg, hsDb, func(r *synapse.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { + internal.PsqlFlatFileArchive[synapse.LocalMedia](ctx, cfg, hsDb, func(r *synapse.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { // For MediaID AABBCCDD : // $importPath/local_content/AA/BB/CCDD // diff --git a/cmd/homeserver_offline_importers/_common/archiver.go b/cmd/homeserver_offline_importers/internal/archiver.go similarity index 98% rename from cmd/homeserver_offline_importers/_common/archiver.go rename to cmd/homeserver_offline_importers/internal/archiver.go index 645730a4..6b46e2c3 100644 --- a/cmd/homeserver_offline_importers/_common/archiver.go +++ b/cmd/homeserver_offline_importers/internal/archiver.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "fmt" diff --git a/cmd/homeserver_offline_importers/_common/init_export.go b/cmd/homeserver_offline_importers/internal/init_export.go similarity index 99% rename from cmd/homeserver_offline_importers/_common/init_export.go rename to cmd/homeserver_offline_importers/internal/init_export.go index f9b3e7e8..dd0e6d32 100644 --- a/cmd/homeserver_offline_importers/_common/init_export.go +++ b/cmd/homeserver_offline_importers/internal/init_export.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag" From 0c957d8fc595be541290fe31354d4caedbbc9f6c Mon Sep 17 00:00:00 2001 From: Moritz Poldrack Date: Sun, 3 Mar 2024 15:24:43 +0100 Subject: [PATCH 22/22] move cmd/homeserver_offline_exporters/_common to cmd/homeserver_offline_exporters/internal --- cmd/homeserver_offline_exporters/import_to_synapse/main.go | 6 +++--- .../{_common => internal}/archive_reader.go | 2 +- .../{_common => internal}/init_export.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename cmd/homeserver_offline_exporters/{_common => internal}/archive_reader.go (99%) rename cmd/homeserver_offline_exporters/{_common => internal}/init_export.go (99%) diff --git a/cmd/homeserver_offline_exporters/import_to_synapse/main.go b/cmd/homeserver_offline_exporters/import_to_synapse/main.go index 3c7db1e2..850816f0 100644 --- a/cmd/homeserver_offline_exporters/import_to_synapse/main.go +++ b/cmd/homeserver_offline_exporters/import_to_synapse/main.go @@ -10,7 +10,7 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_exporters/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_exporters/internal" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -33,7 +33,7 @@ var synapseDefaultSizes = []thumbnailSize{ } func main() { - cfg := _common.InitExportPsqlFlatFile("Synapse", "media_store_path") + cfg := internal.InitExportPsqlFlatFile("Synapse", "media_store_path") ctx := rcontext.InitialNoConfig() ctx.Config.Thumbnails = config.ThumbnailsConfig{ Types: []string{ @@ -58,7 +58,7 @@ func main() { panic(err) } - err = _common.ProcessArchiveDirectory(ctx, cfg.ServerName, cfg.SourcePath, func(record *v2archive.ManifestRecord, f io.ReadCloser) error { + err = internal.ProcessArchiveDirectory(ctx, cfg.ServerName, cfg.SourcePath, func(record *v2archive.ManifestRecord, f io.ReadCloser) error { defer f.Close() mxc := util.MxcUri(record.Origin, record.MediaId) diff --git a/cmd/homeserver_offline_exporters/_common/archive_reader.go b/cmd/homeserver_offline_exporters/internal/archive_reader.go similarity index 99% rename from cmd/homeserver_offline_exporters/_common/archive_reader.go rename to cmd/homeserver_offline_exporters/internal/archive_reader.go index 3974860d..26659248 100644 --- a/cmd/homeserver_offline_exporters/_common/archive_reader.go +++ b/cmd/homeserver_offline_exporters/internal/archive_reader.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "io" diff --git a/cmd/homeserver_offline_exporters/_common/init_export.go b/cmd/homeserver_offline_exporters/internal/init_export.go similarity index 99% rename from cmd/homeserver_offline_exporters/_common/init_export.go rename to cmd/homeserver_offline_exporters/internal/init_export.go index 398abce9..ea76978c 100644 --- a/cmd/homeserver_offline_exporters/_common/init_export.go +++ b/cmd/homeserver_offline_exporters/internal/init_export.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag"