diff --git a/api/_apimeta/auth.go b/api/apimeta/auth.go similarity index 97% rename from api/_apimeta/auth.go rename to api/apimeta/auth.go index 5bd5dfd6..7690ec62 100644 --- a/api/_apimeta/auth.go +++ b/api/apimeta/auth.go @@ -1,4 +1,4 @@ -package _apimeta +package apimeta import ( "net/http" diff --git a/api/_auth_cache/auth_cache.go b/api/auth_cache/auth_cache.go similarity index 88% rename from api/_auth_cache/auth_cache.go rename to api/auth_cache/auth_cache.go index da1f2d5c..a306983a 100644 --- a/api/_auth_cache/auth_cache.go +++ b/api/auth_cache/auth_cache.go @@ -1,4 +1,4 @@ -package _auth_cache +package auth_cache import ( "errors" @@ -103,25 +103,25 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId return checkTokenWithHomeserver(ctx, accessToken, appserviceUserId, true) } - for _, r := range ctx.Config.AccessTokens.Appservices { - if r.AppserviceToken != accessToken { + for _, appSrv := range ctx.Config.AccessTokens.Appservices { + if appSrv.AppserviceToken != accessToken { continue } - if r.SenderUserId != "" && (r.SenderUserId == appserviceUserId || appserviceUserId == "") { - ctx.Log.Debugf("Access token belongs to appservice (sender user ID): %s", r.Id) - cacheToken(ctx, accessToken, appserviceUserId, r.SenderUserId, nil) - return r.SenderUserId, nil + if appSrv.SenderUserId != "" && (appSrv.SenderUserId == appserviceUserId || appserviceUserId == "") { + ctx.Log.Debugf("Access token belongs to appservice (sender user ID): %s", appSrv.Id) + cacheToken(ctx, accessToken, appserviceUserId, appSrv.SenderUserId, nil) + return appSrv.SenderUserId, nil } - for _, n := range r.UserNamespaces { + for _, n := range appSrv.UserNamespaces { regex, ok := regexCache[n.Regex] if !ok { regex = regexp.MustCompile(n.Regex) regexCache[n.Regex] = regex } if regex.MatchString(appserviceUserId) { - ctx.Log.Debugf("Access token belongs to appservice: %s", r.Id) + ctx.Log.Debugf("Access token belongs to appservice: %s", appSrv.Id) cacheToken(ctx, accessToken, appserviceUserId, appserviceUserId, nil) return appserviceUserId, nil } @@ -133,13 +133,13 @@ func GetUserId(ctx rcontext.RequestContext, accessToken string, appserviceUserId } func cacheToken(ctx rcontext.RequestContext, accessToken string, appserviceUserId string, userId string, err error) { - v := cachedToken{ + token := cachedToken{ userId: userId, err: err, } t := time.Duration(ctx.Config.AccessTokens.MaxCacheTimeSeconds) * time.Second rwLock.Lock() - tokenCache.Set(cacheKey(accessToken, appserviceUserId), v, t) + tokenCache.Set(cacheKey(accessToken, appserviceUserId), token, t) rwLock.Unlock() } diff --git a/api/branched_route.go b/api/branched_route.go index 6dc9c98f..897a1c0a 100644 --- a/api/branched_route.go +++ b/api/branched_route.go @@ -4,7 +4,7 @@ import ( "net/http" "strings" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/routers" ) type branch struct { @@ -19,29 +19,29 @@ type splitBranch struct { func branchedRoute(branches []branch) http.Handler { sbranches := make([]splitBranch, len(branches)) - for i, b := range branches { + for i, branch := range branches { sbranches[i] = splitBranch{ - segments: strings.Split(b.string, "/"), - handler: b.Handler, + segments: strings.Split(branch.string, "/"), + handler: branch.Handler, } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - catchAll := _routers.GetParam("branch", r) + catchAll := routers.GetParam("branch", r) if catchAll[0] == '/' { catchAll = catchAll[1:] } params := strings.Split(catchAll, "/") - for _, b := range sbranches { - if b.segments[0][0] == ':' || b.segments[0] == params[0] { - if len(b.segments) != len(params) { + for _, branch := range sbranches { + if branch.segments[0][0] == ':' || branch.segments[0] == params[0] { + if len(branch.segments) != len(params) { continue } - for i, s := range b.segments { - if s[0] == ':' { - r = _routers.ForceSetParam(s[1:], params[i], r) + for i, segment := range branch.segments { + if segment[0] == ':' { + r = routers.ForceSetParam(segment[1:], params[i], r) } } - b.handler.ServeHTTP(w, r) + branch.handler.ServeHTTP(w, r) return } } diff --git a/api/custom/datastores.go b/api/custom/datastores.go index 5985611d..52e76363 100644 --- a/api/custom/datastores.go +++ b/api/custom/datastores.go @@ -1,20 +1,22 @@ package custom import ( + "errors" + "fmt" + "net/http" + "strconv" + "time" + "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" - "net/http" - "strconv" - "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DatastoreMigration struct { @@ -22,37 +24,37 @@ type DatastoreMigration struct { TaskID int `json:"task_id"` } -func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { response := make(map[string]interface{}) - for _, ds := range config.UniqueDatastores() { - uri, err := datastores.GetUri(ds) + for _, store := range config.UniqueDatastores() { + uri, err := datastores.GetUri(store) if err != nil { sentry.CaptureException(err) rctx.Log.Error("Error getting datastore URI: ", err) - return _responses.InternalServerError("unexpected error getting datastore information") + return responses.InternalServerError(errors.New("unexpected error getting datastore information")) } - dsMap := make(map[string]interface{}) - dsMap["type"] = ds.Type - dsMap["uri"] = uri - response[ds.Id] = dsMap + dataStoreMap := make(map[string]interface{}) + dataStoreMap["type"] = store.Type + dataStoreMap["uri"] = uri + response[store.Id] = dataStoreMap } - return &_responses.DoNotCacheResponse{Payload: response} + return &responses.DoNotCacheResponse{Payload: response} } -func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } - sourceDsId := _routers.GetParam("sourceDsId", r) - targetDsId := _routers.GetParam("targetDsId", r) + sourceDsId := routers.GetParam("sourceDsId", r) + targetDsId := routers.GetParam("targetDsId", r) rctx = rctx.LogWithFields(logrus.Fields{ "beforeTs": beforeTs, @@ -61,20 +63,20 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use }) if sourceDsId == targetDsId { - return _responses.BadRequest("Source and target datastore cannot be the same") + return responses.BadRequest(errors.New("Source and target datastore cannot be the same")) } if _, ok := datastores.Get(rctx, sourceDsId); !ok { - return _responses.BadRequest("Source datastore does not appear to exist") + return responses.BadRequest(errors.New("Source datastore does not appear to exist")) } if _, ok := datastores.Get(rctx, targetDsId); !ok { - return _responses.BadRequest("Target datastore does not appear to exist") + return responses.BadRequest(errors.New("Target datastore does not appear to exist")) } estimate, err := datastores.SizeOfDsIdWithAge(rctx, sourceDsId, beforeTs) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error getting storage estimate") + return responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } rctx.Log.Infof("User %s has started a datastore media transfer", user.UserId) @@ -82,7 +84,7 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error starting migration") + return responses.InternalServerError(errors.New("Unexpected error starting migration")) } migration := &DatastoreMigration{ @@ -90,21 +92,21 @@ func MigrateBetweenDatastores(r *http.Request, rctx rcontext.RequestContext, use TaskID: task.TaskId, } - return &_responses.DoNotCacheResponse{Payload: migration} + return &responses.DoNotCacheResponse{Payload: migration} } -func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() var err error if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } - datastoreId := _routers.GetParam("datastoreId", r) + datastoreId := routers.GetParam("datastoreId", r) rctx = rctx.LogWithFields(logrus.Fields{ "beforeTs": beforeTs, @@ -115,7 +117,7 @@ func GetDatastoreStorageEstimate(r *http.Request, rctx rcontext.RequestContext, if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected error getting storage estimate") + return responses.InternalServerError(errors.New("Unexpected error getting storage estimate")) } - return &_responses.DoNotCacheResponse{Payload: result} + return &responses.DoNotCacheResponse{Payload: result} } diff --git a/api/custom/exports.go b/api/custom/exports.go index 59a89af0..1b6e90cd 100644 --- a/api/custom/exports.go +++ b/api/custom/exports.go @@ -2,13 +2,14 @@ package custom import ( "bytes" + "errors" "net/http" "strconv" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/tasks" @@ -37,22 +38,22 @@ type ExportMetadata struct { Parts []*ExportPartMetadata `json:"parts"` } -func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if !rctx.Config.Archiving.SelfService && !isAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } s3urls := r.URL.Query().Get("s3_urls") != "false" - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) if !isAdmin && user.UserId != userId { - return _responses.BadRequest("cannot export data for another user") + return responses.BadRequest(errors.New("cannot export data for another user")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -63,35 +64,35 @@ func ExportUserData(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting export") + return responses.InternalServerError(errors.New("fatal error starting export")) } - return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ + return &responses.DoNotCacheResponse{Payload: &ExportStarted{ TaskID: task.TaskId, ExportID: exportId, }} } -func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } isAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if !rctx.Config.Archiving.SelfService && !isAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } s3urls := r.URL.Query().Get("s3_urls") != "false" - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) if !isAdmin { // They might be a local admin, so check that. // We won't be able to check unless we know about the homeserver though if !util.IsServerOurs(serverName) { - return _responses.BadRequest("cannot export data for another server") + return responses.BadRequest(errors.New("cannot export data for another server")) } isLocalAdmin, err := matrix.IsUserAdmin(rctx, serverName, user.AccessToken, r.RemoteAddr) @@ -100,7 +101,7 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime isLocalAdmin = false } if !isLocalAdmin { - return _responses.BadRequest("cannot export data for another server") + return responses.BadRequest(errors.New("cannot export data for another server")) } } @@ -112,24 +113,24 @@ func ExportServerData(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting export") + return responses.InternalServerError(errors.New("fatal error starting export")) } - return &_responses.DoNotCacheResponse{Payload: &ExportStarted{ + return &responses.DoNotCacheResponse{Payload: &ExportStarted{ TaskID: task.TaskId, ExportID: exportId, }} } -func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ViewExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + if !routers.ServerNameRegex.MatchString(exportId) { + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -143,24 +144,24 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get entity for export ID") + return responses.InternalServerError(errors.New("failed to get entity for export ID")) } if entityId == "" { - return _responses.NotFoundError() + return responses.NotFoundError() } parts, err := partsDb.GetForExport(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return responses.InternalServerError(errors.New("failed to get export parts")) } template, err := templating.GetTemplate("view_export") if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get template") + return responses.InternalServerError(errors.New("failed to get template")) } model := &templating.ViewExportModel{ @@ -183,21 +184,21 @@ func ViewExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to render template") + return responses.InternalServerError(errors.New("failed to render template")) } - return &_responses.HtmlResponse{HTML: html.String()} + return &responses.HtmlResponse{HTML: html.String()} } -func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + if !routers.ServerNameRegex.MatchString(exportId) { + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -211,14 +212,14 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get entity for export ID") + return responses.InternalServerError(errors.New("failed to get entity for export ID")) } parts, err := partsDb.GetForExport(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return responses.InternalServerError(errors.New("failed to get export parts")) } metadata := &ExportMetadata{ @@ -233,25 +234,25 @@ func GetExportMetadata(r *http.Request, rctx rcontext.RequestContext, user _apim }) } - return &_responses.DoNotCacheResponse{Payload: metadata} + return &responses.DoNotCacheResponse{Payload: metadata} } -func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) - pid := _routers.GetParam("partId", r) + exportId := routers.GetParam("exportId", r) + pid := routers.GetParam("partId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + if !routers.ServerNameRegex.MatchString(exportId) { + responses.BadRequest(errors.New("invalid export ID")) } partId, err := strconv.Atoi(pid) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest("invalid part index") + return responses.BadRequest(errors.New("invalid part index")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -264,26 +265,26 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get part") + return responses.InternalServerError(errors.New("failed to get part")) } if part == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } dsConf, ok := datastores.Get(rctx, part.DatastoreId) if !ok { sentry.CaptureMessage("failed to locate datastore") - return _responses.InternalServerError("failed to locate datastore") + return responses.InternalServerError(errors.New("failed to locate datastore")) } s, err := datastores.Download(rctx, dsConf, part.Location) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to start download") + return responses.InternalServerError(errors.New("failed to start download")) } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: "application/gzip", // TODO: We should be detecting type rather than assuming SizeBytes: part.SizeBytes, Data: s, @@ -292,15 +293,15 @@ func DownloadExportPart(r *http.Request, rctx rcontext.RequestContext, user _api } } -func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } - exportId := _routers.GetParam("exportId", r) + exportId := routers.GetParam("exportId", r) - if !_routers.ServerNameRegex.MatchString(exportId) { - _responses.BadRequest("invalid export ID") + if !routers.ServerNameRegex.MatchString(exportId) { + responses.BadRequest(errors.New("invalid export ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -315,7 +316,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get export parts") + return responses.InternalServerError(errors.New("failed to get export parts")) } for _, part := range parts { @@ -324,7 +325,7 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export part") + return responses.InternalServerError(errors.New("failed to delete export part")) } } @@ -333,14 +334,14 @@ func DeleteExport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export parts") + return responses.InternalServerError(errors.New("failed to delete export parts")) } err = exportDb.Delete(exportId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to delete export record") + return responses.InternalServerError(errors.New("failed to delete export record")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } diff --git a/api/custom/federation.go b/api/custom/federation.go index 48a0fb92..ae70dffd 100644 --- a/api/custom/federation.go +++ b/api/custom/federation.go @@ -2,23 +2,24 @@ package custom import ( "encoding/json" + "errors" "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/matrix" ) -func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) +func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -29,7 +30,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return responses.InternalServerError(err) } versionUrl := url + "/_matrix/federation/v1/version" @@ -37,7 +38,7 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return responses.InternalServerError(err) } decoder := json.NewDecoder(versionResponse.Body) @@ -46,12 +47,12 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError(err.Error()) + return responses.InternalServerError(err) } resp := make(map[string]interface{}) resp["base_url"] = url resp["hostname"] = hostname resp["versions_response"] = out - return &_responses.DoNotCacheResponse{Payload: resp} + return &responses.DoNotCacheResponse{Payload: resp} } diff --git a/api/custom/health.go b/api/custom/health.go index ac4780d8..3ca4154c 100644 --- a/api/custom/health.go +++ b/api/custom/health.go @@ -3,8 +3,8 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -13,8 +13,8 @@ type HealthzResponse struct { Status string `json:"status"` } -func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - return &_responses.DoNotCacheResponse{ +func GetHealthz(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + return &responses.DoNotCacheResponse{ Payload: &HealthzResponse{ OK: true, Status: "Probably not dead", diff --git a/api/custom/imports.go b/api/custom/imports.go index a667834a..700a5b1e 100644 --- a/api/custom/imports.go +++ b/api/custom/imports.go @@ -5,9 +5,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/tasks" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -21,84 +21,84 @@ type ImportStarted struct { TaskID int `json:"task_id"` } -func StartImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func StartImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } task, importId, err := tasks.RunImport(rctx) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("fatal error starting import") + return responses.InternalServerError(errors.New("fatal error starting import")) } err = task_runner.AppendImportFile(rctx, importId, r.Body) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error appending first file to import") + return responses.InternalServerError(errors.New("error appending first file to import")) } - return &_responses.DoNotCacheResponse{Payload: &ImportStarted{ + return &responses.DoNotCacheResponse{Payload: &ImportStarted{ TaskID: task.TaskId, ImportID: importId, }} } -func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func AppendToImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } - importId := _routers.GetParam("importId", r) + importId := routers.GetParam("importId", r) - if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest("invalid import ID") + if !routers.ServerNameRegex.MatchString(importId) { + return responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.AppendImportFile(rctx, importId, r.Body) if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error appending to import") + return responses.InternalServerError(errors.New("error appending to import")) } - return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} + return &responses.DoNotCacheResponse{Payload: &responses.EmptyResponse{}} } -func StopImport(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func StopImport(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Archiving.Enabled { - return _responses.BadRequest("archiving is not enabled") + return responses.BadRequest(errors.New("archiving is not enabled")) } if ids.GetMachineId() != tasks.ExecutingMachineId { - return _responses.BadRequest("archival import can only be done on the background tasks worker") + return responses.BadRequest(errors.New("archival import can only be done on the background tasks worker")) } - importId := _routers.GetParam("importId", r) + importId := routers.GetParam("importId", r) - if !_routers.ServerNameRegex.MatchString(importId) { - return _responses.BadRequest("invalid import ID") + if !routers.ServerNameRegex.MatchString(importId) { + return responses.BadRequest(errors.New("invalid import ID")) } err := task_runner.FinishImport(rctx, importId) if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error stopping import") + return responses.InternalServerError(errors.New("error stopping import")) } - return &_responses.DoNotCacheResponse{Payload: &_responses.EmptyResponse{}} + return &responses.DoNotCacheResponse{Payload: &responses.EmptyResponse{}} } diff --git a/api/custom/media_attributes.go b/api/custom/media_attributes.go index 8f9abda1..6b3eb5df 100644 --- a/api/custom/media_attributes.go +++ b/api/custom/media_attributes.go @@ -2,13 +2,14 @@ package custom import ( "encoding/json" + "errors" "net/http" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/matrix" @@ -19,7 +20,7 @@ type Attributes struct { Purpose database.Purpose `json:"purpose"` } -func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user _apimeta.UserInfo) bool { +func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin string, user apimeta.UserInfo) bool { isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared if isGlobalAdmin { return true @@ -32,12 +33,12 @@ func canChangeAttributes(rctx rcontext.RequestContext, r *http.Request, origin s return isLocalAdmin } -func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - origin := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + origin := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest("invalid origin") + if !routers.ServerNameRegex.MatchString(origin) { + return responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -46,7 +47,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. }) if !canChangeAttributes(rctx, r, origin, user) { - return _responses.AuthFailed() + return responses.AuthFailed() } // Check to see if the media exists @@ -55,10 +56,10 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get media record") + return responses.InternalServerError(errors.New("failed to get media record")) } if media == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } attrDb := database.GetInstance().MediaAttributes.Prepare(rctx) @@ -66,7 +67,7 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get attributes record") + return responses.InternalServerError(errors.New("failed to get attributes record")) } retAttrs := &Attributes{ Purpose: database.PurposeNone, @@ -75,15 +76,15 @@ func GetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. retAttrs.Purpose = attrs.Purpose } - return &_responses.DoNotCacheResponse{Payload: retAttrs} + return &responses.DoNotCacheResponse{Payload: retAttrs} } -func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - origin := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + origin := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(origin) { - return _responses.BadRequest("invalid origin") + if !routers.ServerNameRegex.MatchString(origin) { + return responses.BadRequest(errors.New("invalid origin")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -92,7 +93,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. }) if !canChangeAttributes(rctx, r, origin, user) { - return _responses.AuthFailed() + return responses.AuthFailed() } defer r.Body.Close() @@ -102,7 +103,7 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to read attributes") + return responses.InternalServerError(errors.New("failed to read attributes")) } attrDb := database.GetInstance().MediaAttributes.Prepare(rctx) @@ -110,20 +111,20 @@ func SetAttributes(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get attributes") + return responses.InternalServerError(errors.New("failed to get attributes")) } if attrs == nil || attrs.Purpose != newAttrs.Purpose { if !database.IsPurpose(newAttrs.Purpose) { - return _responses.BadRequest("unknown purpose") + return responses.BadRequest(errors.New("unknown purpose")) } err = attrDb.UpsertPurpose(origin, mediaId, newAttrs.Purpose) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to update attributes: purpose") + return responses.InternalServerError(errors.New("failed to update attributes: purpose")) } } - return &_responses.DoNotCacheResponse{Payload: newAttrs} + return &responses.DoNotCacheResponse{Payload: newAttrs} } diff --git a/api/custom/purge.go b/api/custom/purge.go index cc8ef012..59d89cd1 100644 --- a/api/custom/purge.go +++ b/api/custom/purge.go @@ -2,13 +2,15 @@ package custom import ( "errors" + "fmt" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -23,39 +25,40 @@ type MediaPurgedResponse struct { NumRemoved int `json:"total_removed"` } -func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeRemoteMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr == "" { - return _responses.BadRequest("Missing before_ts argument") + return responses.BadRequest(errors.New("Missing before_ts argument")) } beforeTs, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } + before := time.UnixMilli(beforeTs) rctx = rctx.LogWithFields(logrus.Fields{ - "beforeTs": beforeTs, + "beforeTs": before, }) // We don't bother clearing the cache because it's still probably useful there - removed, err := task_runner.PurgeRemoteMediaBefore(rctx, beforeTs) + removed, err := task_runner.PurgeRemoteMediaBefore(rctx, before) if err != nil { rctx.Log.Error("Error purging remote media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Error purging remote media") + return responses.InternalServerError(errors.New("Error purging remote media")) } - return &_responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} + return &responses.DoNotCacheResponse{Payload: &MediaPurgedResponse{NumRemoved: removed}} } -func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, _, _ := getPurgeAuthContext(rctx, r, user) - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -71,17 +74,17 @@ func PurgeIndividualRecord(r *http.Request, rctx rcontext.RequestContext, user _ }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true}} } -func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) var affected []*database.DbMedia @@ -93,12 +96,12 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime } else if isLocalAdmin { affected, err = mediaDb.GetByOriginQuarantine(r.Host) } else { - return _responses.AuthFailed() + return responses.AuthFailed() } if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -106,25 +109,26 @@ func PurgeQuarantined(r *http.Request, rctx rcontext.RequestContext, user _apime }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { var err error - beforeTs := util.NowMillis() + before := time.Now() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { - beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) + beforeTS, err := strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } + before = time.UnixMilli(beforeTS) } includeLocal := false @@ -132,12 +136,12 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if includeLocalStr != "" { includeLocal, err = strconv.ParseBool(includeLocalStr) if err != nil { - return _responses.BadRequest("Error parsing include_local: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing include_local: %w", err)) } } rctx = rctx.LogWithFields(logrus.Fields{ - "before_ts": beforeTs, + "before_ts": before, "include_local": includeLocal, }) @@ -147,11 +151,11 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. } mediaDb := database.GetInstance().Media.Prepare(rctx) - records, err := mediaDb.GetOldExcluding(domains, beforeTs) + records, err := mediaDb.GetOldExcluding(domains, before) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, &task_runner.PurgeAuthContext{}, &task_runner.QuarantineThis{ @@ -159,33 +163,33 @@ func PurgeOldMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) rctx = rctx.LogWithFields(logrus.Fields{ "userId": userId, @@ -194,13 +198,13 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta _, userDomain, err := util.SplitUserId(userId) if err != nil { - rctx.Log.Error("Error parsing user ID ("+userId+"): ", err) + rctx.Log.Errorf("Error parsing user ID (%s): %v", userId, err) sentry.CaptureException(err) - return _responses.InternalServerError("error parsing user ID") + return responses.InternalServerError(errors.New("error parsing user ID")) } if !isGlobalAdmin && userDomain != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } mediaDb := database.GetInstance().Media.Prepare(rctx) @@ -208,7 +212,7 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -216,33 +220,33 @@ func PurgeUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %w", err)) } } - roomId := _routers.GetParam("roomId", r) + roomId := routers.GetParam("roomId", r) rctx = rctx.LogWithFields(logrus.Fields{ "roomId": roomId, @@ -253,7 +257,7 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media in room") + return responses.InternalServerError(errors.New("error retrieving media in room")) } mxcs := make([]string, 0) @@ -289,36 +293,36 @@ func PurgeRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs2}} } -func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { authCtx, isGlobalAdmin, isLocalAdmin := getPurgeAuthContext(rctx, r, user) if !isGlobalAdmin && !isLocalAdmin { - return _responses.AuthFailed() + return responses.AuthFailed() } var err error - beforeTs := util.NowMillis() + beforeTs := time.Now().UnixMilli() beforeTsStr := r.URL.Query().Get("before_ts") if beforeTsStr != "" { beforeTs, err = strconv.ParseInt(beforeTsStr, 10, 64) if err != nil { - return _responses.BadRequest("Error parsing before_ts: " + err.Error()) + return responses.BadRequest(fmt.Errorf("Error parsing before_ts: %f", err)) } } - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -327,7 +331,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime }) if !isGlobalAdmin && serverName != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } mediaDb := database.GetInstance().Media.Prepare(rctx) @@ -335,7 +339,7 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error fetching media records") + return responses.InternalServerError(errors.New("error fetching media records")) } mxcs, err := task_runner.PurgeMedia(rctx, authCtx, &task_runner.QuarantineThis{ @@ -343,18 +347,18 @@ func PurgeDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apime }) if err != nil { if errors.Is(err, common.ErrWrongUser) { - return _responses.AuthFailed() + return responses.AuthFailed() } rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } - return &_responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} + return &responses.DoNotCacheResponse{Payload: map[string]interface{}{"purged": true, "affected": mxcs}} } -func getPurgeAuthContext(ctx rcontext.RequestContext, r *http.Request, user _apimeta.UserInfo) (*task_runner.PurgeAuthContext, bool, bool) { - globalAdmin, localAdmin := _apimeta.GetRequestUserAdminStatus(r, ctx, user) +func getPurgeAuthContext(ctx rcontext.RequestContext, r *http.Request, user apimeta.UserInfo) (*task_runner.PurgeAuthContext, bool, bool) { + globalAdmin, localAdmin := apimeta.GetRequestUserAdminStatus(r, ctx, user) if globalAdmin { return &task_runner.PurgeAuthContext{}, true, localAdmin } diff --git a/api/custom/quarantine.go b/api/custom/quarantine.go index 26954935..22b8fab0 100644 --- a/api/custom/quarantine.go +++ b/api/custom/quarantine.go @@ -1,12 +1,13 @@ package custom import ( + "errors" "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/tasks/task_runner" @@ -20,13 +21,13 @@ type MediaQuarantinedResponse struct { NumQuarantined int64 `json:"num_quarantined"` } -func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } - roomId := _routers.GetParam("roomId", r) + roomId := routers.GetParam("roomId", r) rctx = rctx.LogWithFields(logrus.Fields{ "roomId": roomId, @@ -37,7 +38,7 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error while listing media in the room: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media in room") + return responses.InternalServerError(errors.New("error retrieving media in room")) } var mxcs []string @@ -51,13 +52,13 @@ func QuarantineRoomMedia(r *http.Request, rctx rcontext.RequestContext, user _ap }) } -func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } - userId := _routers.GetParam("userId", r) + userId := routers.GetParam("userId", r) rctx = rctx.LogWithFields(logrus.Fields{ "userId": userId, @@ -68,11 +69,11 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error parsing user ID ("+userId+"): ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error parsing user ID") + return responses.InternalServerError(errors.New("error parsing user ID")) } if !allowOtherHosts && userDomain != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } db := database.GetInstance().Media.Prepare(rctx) @@ -80,7 +81,7 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap if err != nil { rctx.Log.Error("Error while listing media for the user: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media for user") + return responses.InternalServerError(errors.New("error retrieving media for user")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -88,16 +89,16 @@ func QuarantineUserMedia(r *http.Request, rctx rcontext.RequestContext, user _ap }) } -func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -106,7 +107,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ }) if !allowOtherHosts && serverName != r.Host { - return _responses.AuthFailed() + return responses.AuthFailed() } db := database.GetInstance().Media.Prepare(rctx) @@ -114,7 +115,7 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ if err != nil { rctx.Log.Error("Error while listing media for the server: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("error retrieving media for server") + return responses.InternalServerError(errors.New("error retrieving media for server")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -122,17 +123,17 @@ func QuarantineDomainMedia(r *http.Request, rctx rcontext.RequestContext, user _ }) } -func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { canQuarantine, allowOtherHosts, isLocalAdmin := getQuarantineRequestInfo(r, rctx, user) if !canQuarantine { - return _responses.AuthFailed() + return responses.AuthFailed() } - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -142,7 +143,7 @@ func QuarantineMedia(r *http.Request, rctx rcontext.RequestContext, user _apimet }) if !allowOtherHosts && r.Host != server { - return _responses.BadRequest("unable to quarantine media on other homeservers") + return responses.BadRequest(errors.New("unable to quarantine media on other homeservers")) } return performQuarantineRequest(rctx, r.Host, allowOtherHosts, &task_runner.QuarantineThis{ @@ -163,13 +164,13 @@ func performQuarantineRequest(ctx rcontext.RequestContext, host string, allowOth if err != nil { ctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("error quarantining media") + return responses.InternalServerError(errors.New("error quarantining media")) } - return &_responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} + return &responses.DoNotCacheResponse{Payload: &MediaQuarantinedResponse{NumQuarantined: total}} } -func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) (bool, bool, bool) { +func getQuarantineRequestInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) (bool, bool, bool) { isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared canQuarantine := isGlobalAdmin allowOtherHosts := isGlobalAdmin diff --git a/api/custom/tasks.go b/api/custom/tasks.go index 2002ab9f..eb60c70a 100644 --- a/api/custom/tasks.go +++ b/api/custom/tasks.go @@ -1,15 +1,16 @@ package custom import ( - "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" - "github.com/t2bot/matrix-media-repo/database" - + "errors" "net/http" "strconv" + "github.com/getsentry/sentry-go" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" + "github.com/t2bot/matrix-media-repo/database" + "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -24,12 +25,12 @@ type TaskStatus struct { Error string `json:"error_message"` } -func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - taskIdStr := _routers.GetParam("taskId", r) +func GetTask(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + taskIdStr := routers.GetParam("taskId", r) taskId, err := strconv.Atoi(taskIdStr) if err != nil { rctx.Log.Error(err) - return _responses.BadRequest("invalid task ID") + return responses.BadRequest(errors.New("invalid task ID")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -42,13 +43,13 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserIn if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("failed to get task information") + return responses.InternalServerError(errors.New("failed to get task information")) } if task == nil { - return _responses.NotFoundError() + return responses.NotFoundError() } - return &_responses.DoNotCacheResponse{Payload: &TaskStatus{ + return &responses.DoNotCacheResponse{Payload: &TaskStatus{ TaskID: task.TaskId, Name: task.Name, Params: task.Params, @@ -59,14 +60,14 @@ func GetTask(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserIn }} } -func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { db := database.GetInstance().Tasks.Prepare(rctx) tasks, err := db.GetAll(true) if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get background tasks") + return responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) @@ -82,17 +83,17 @@ func ListAllTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U }) } - return &_responses.DoNotCacheResponse{Payload: statusObjs} + return &responses.DoNotCacheResponse{Payload: statusObjs} } -func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { db := database.GetInstance().Tasks.Prepare(rctx) tasks, err := db.GetAll(false) if err != nil { logrus.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get background tasks") + return responses.InternalServerError(errors.New("Failed to get background tasks")) } statusObjs := make([]*TaskStatus, 0) @@ -108,5 +109,5 @@ func ListUnfinishedTasks(r *http.Request, rctx rcontext.RequestContext, user _ap }) } - return &_responses.DoNotCacheResponse{Payload: statusObjs} + return &responses.DoNotCacheResponse{Payload: statusObjs} } diff --git a/api/custom/usage.go b/api/custom/usage.go index 3025ebdb..e14e8e6f 100644 --- a/api/custom/usage.go +++ b/api/custom/usage.go @@ -1,14 +1,16 @@ package custom import ( + "errors" + "fmt" "net/http" "strconv" "strings" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -52,11 +54,11 @@ type MediaUsageEntry struct { CreatedTs int64 `json:"created_ts"` } -func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) +func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + serverName := routers.GetParam("serverName", r) - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -69,17 +71,17 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get byte usage for server") + return responses.InternalServerError(errors.New("Failed to get byte usage for server")) } mediaCount, thumbCount, err := db.CountUsageForServer(serverName) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get count usage for server") + return responses.InternalServerError(errors.New("Failed to get count usage for server")) } - return &_responses.DoNotCacheResponse{ + return &responses.DoNotCacheResponse{ Payload: &CountsUsageResponse{ RawBytes: &UsageInfo{ MinimalUsageInfo: &MinimalUsageInfo{ @@ -99,12 +101,12 @@ func GetDomainUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta } } -func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) +func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + serverName := routers.GetParam("serverName", r) userIds := r.URL.Query()["user_id"] - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -124,7 +126,7 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get media records for users") + return responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*UserUsageEntry) @@ -155,15 +157,15 @@ func GetUserUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.U entry.UploadedMxcs = append(entry.UploadedMxcs, util.MxcUri(media.Origin, media.MediaId)) } - return &_responses.DoNotCacheResponse{Payload: parsed} + return &responses.DoNotCacheResponse{Payload: parsed} } -func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - serverName := _routers.GetParam("serverName", r) +func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + serverName := routers.GetParam("serverName", r) mxcs := r.URL.Query()["mxc"] - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -183,11 +185,11 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Error parsing MXC " + mxc) + return responses.InternalServerError(fmt.Errorf("Error parsing MXC %s", mxc)) } if o != serverName { - return _responses.BadRequest("MXC URIs must match the requested server") + return responses.BadRequest(errors.New("MXC URIs must match the requested server")) } split = append(split, i) @@ -198,7 +200,7 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get media records for users") + return responses.InternalServerError(errors.New("Failed to get media records for users")) } parsed := make(map[string]*MediaUsageEntry) @@ -217,27 +219,27 @@ func GetUploadsUsage(r *http.Request, rctx rcontext.RequestContext, user _apimet } } - return &_responses.DoNotCacheResponse{Payload: parsed} + return &responses.DoNotCacheResponse{Payload: parsed} } // SynGetUsersMediaStats attempts to provide a loose equivalent to this Synapse admin endpoint: // https://matrix-org.github.io/synapse/v1.88/admin_api/statistics.html#users-media-usage-statistics -func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { qs := r.URL.Query() var err error - serverName := _routers.GetParam("serverName", r) + serverName := routers.GetParam("serverName", r) if serverName == "" && strings.HasPrefix(r.URL.Path, synapse.PrefixAdminApi) { serverName = r.Host } - if !_routers.ServerNameRegex.MatchString(serverName) { - return _responses.BadRequest("invalid server name") + if !routers.ServerNameRegex.MatchString(serverName) { + return responses.BadRequest(errors.New("invalid server name")) } - isGlobalAdmin, isLocalAdmin := _apimeta.GetRequestUserAdminStatus(r, rctx, user) + isGlobalAdmin, isLocalAdmin := apimeta.GetRequestUserAdminStatus(r, rctx, user) if !isGlobalAdmin && (serverName != r.Host || !isLocalAdmin) { - return _responses.AuthFailed() + return responses.AuthFailed() } orderBy := database.SynStatUserOrderBy(qs.Get("order_by")) @@ -245,14 +247,14 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ orderBy = database.DefaultSynStatUserOrderBy } if !database.IsSynStatUserOrderBy(orderBy) { - return _responses.BadRequest("Query parameter 'order_by' must be one of the accepted values") + return responses.BadRequest(errors.New("Query parameter 'order_by' must be one of the accepted values")) } var start int64 = 0 if len(qs["from"]) > 0 { start, err = strconv.ParseInt(qs.Get("from"), 10, 64) if err != nil || start < 0 { - return _responses.BadRequest("Query parameter 'from' must be a non-negative integer") + return responses.BadRequest(errors.New("Query parameter 'from' must be a non-negative integer")) } } @@ -260,7 +262,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["limit"]) > 0 { limit, err = strconv.ParseInt(qs.Get("limit"), 10, 64) if err != nil || limit < 0 { - return _responses.BadRequest("Query parameter 'limit' must be a non-negative integer") + return responses.BadRequest(errors.New("Query parameter 'limit' must be a non-negative integer")) } } if limit > 50 { @@ -272,7 +274,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["from_ts"]) > 0 { fromTS, err = strconv.ParseInt(qs.Get("from_ts"), 10, 64) if err != nil || fromTS < 0 { - return _responses.BadRequest("Query parameter 'from_ts' must be a non-negative integer") + return responses.BadRequest(errors.New("Query parameter 'from_ts' must be a non-negative integer")) } } @@ -280,15 +282,15 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if len(qs["until_ts"]) > 0 { untilTS, err = strconv.ParseInt(qs.Get("until_ts"), 10, 64) if err != nil || untilTS < 0 { - return _responses.BadRequest("Query parameter 'until_ts' must be a non-negative integer") + return responses.BadRequest(errors.New("Query parameter 'until_ts' must be a non-negative integer")) } else if untilTS <= fromTS { - return _responses.BadRequest("Query parameter 'until_ts' must be greater than 'from_ts'") + return responses.BadRequest(errors.New("Query parameter 'until_ts' must be greater than 'from_ts'")) } } searchTerm := qs.Get("search_term") if searchTerm == "" && len(qs["search_term"]) > 0 { - return _responses.BadRequest("Query parameter 'search_term' cannot be an empty string") + return responses.BadRequest(errors.New("Query parameter 'search_term' cannot be an empty string")) } isAscendingOrder := true @@ -296,7 +298,7 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ if direction == "b" && len(qs["dir"]) > 0 { isAscendingOrder = false } else { - return _responses.BadRequest("Query parameter 'dir' must be one of ['f', 'b']") + return responses.BadRequest(errors.New("Query parameter 'dir' must be one of ['f', 'b']")) } rctx = rctx.LogWithFields(logrus.Fields{ @@ -313,11 +315,10 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ db := database.GetInstance().MetadataView.Prepare(rctx) stats, totalCount, err := db.UnoptimizedSynapseUserStatsPage(serverName, orderBy, start, limit, fromTS, untilTS, searchTerm, isAscendingOrder) - if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("Failed to get users' usage stats on specified server") + return responses.InternalServerError(errors.New("Failed to get users' usage stats on specified server")) } result := &synapse.SynUserStatsResponse{ @@ -338,5 +339,5 @@ func SynGetUsersMediaStats(r *http.Request, rctx rcontext.RequestContext, user _ result.NextToken = start + int64(len(stats)) } - return &_responses.DoNotCacheResponse{Payload: result} + return &responses.DoNotCacheResponse{Payload: result} } diff --git a/api/custom/version.go b/api/custom/version.go index 11ff3b51..1ebb5c19 100644 --- a/api/custom/version.go +++ b/api/custom/version.go @@ -3,16 +3,16 @@ package custom import ( "net/http" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/common/version" ) -func GetVersion(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func GetVersion(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { unstableFeatures := make(map[string]bool) - return &_responses.DoNotCacheResponse{ + return &responses.DoNotCacheResponse{ Payload: map[string]interface{}{ "Version": version.Version, "GitCommit": version.GitCommit, diff --git a/api/_debug/pprof.go b/api/debug/pprof.go similarity index 99% rename from api/_debug/pprof.go rename to api/debug/pprof.go index f2942dc8..e5d7ca58 100644 --- a/api/_debug/pprof.go +++ b/api/debug/pprof.go @@ -1,4 +1,4 @@ -package _debug +package debug import ( "encoding/json" diff --git a/api/r0/download.go b/api/r0/download.go index ab13d4a8..03e6d80e 100644 --- a/api/r0/download.go +++ b/api/r0/download.go @@ -4,11 +4,12 @@ import ( "errors" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/util" @@ -18,23 +19,23 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) - filename := _routers.GetParam("filename", r) +func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) + filename := routers.GetParam("filename", r) allowRemote := r.URL.Query().Get("allow_remote") allowRedirect := r.URL.Query().Get("allow_redirect") timeoutMs := r.URL.Query().Get("timeout_ms") - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest("allow_remote flag does not appear to be a boolean") + return responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -43,14 +44,18 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest("allow_redirect flag does not appear to be a boolean") + return responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } - blockFor, err := util.CalcBlockForDuration(timeoutMs) + timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { - return _responses.BadRequest("timeout_ms does not appear to be an integer") + return responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) + } + timeout := time.Duration(timeoutMS) * time.Millisecond + if timeout > time.Minute { + timeout = time.Minute } rctx = rctx.LogWithFields(logrus.Fields{ @@ -63,42 +68,42 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta. if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } media, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{ FetchRemoteIfNeeded: downloadRemote, - BlockForReadUntil: blockFor, + BlockForReadUntil: timeout, CanRedirect: canRedirect, }) if err != nil { var redirect datastores.RedirectError if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return responses.Redirect(redirect.RedirectUrl) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } if filename == "" { filename = media.UploadName } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: media.ContentType, Filename: filename, SizeBytes: media.SizeBytes, diff --git a/api/r0/identicon.go b/api/r0/identicon.go index eff16d22..3f61ceba 100644 --- a/api/r0/identicon.go +++ b/api/r0/identicon.go @@ -2,6 +2,7 @@ package r0 import ( "crypto/md5" + "fmt" "image/color" "io" "net/http" @@ -10,18 +11,18 @@ import ( "github.com/cupcake/sigil/gen" "github.com/disintegration/imaging" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func Identicon(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.Identicons.Enabled { - return _responses.NotFoundError() + return responses.NotFoundError() } - seed := _routers.GetParam("seed", r) + seed := routers.GetParam("seed", r) var err error width := 96 @@ -32,14 +33,14 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if widthStr != "" { width, err = strconv.Atoi(widthStr) if err != nil { - return _responses.InternalServerError("Error parsing width: " + err.Error()) + return responses.InternalServerError(fmt.Errorf("Error parsing width: %w", err)) } height = width } if heightStr != "" { height, err = strconv.Atoi(heightStr) if err != nil { - return _responses.InternalServerError("Error parsing height: " + err.Error()) + return responses.InternalServerError(fmt.Errorf("Error parsing height: %w", err)) } } @@ -98,7 +99,7 @@ func Identicon(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } }() - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: "image/png", Filename: string(hashed) + ".png", SizeBytes: 0, diff --git a/api/r0/logout.go b/api/r0/logout.go index 7a15ef47..3709cfca 100644 --- a/api/r0/logout.go +++ b/api/r0/logout.go @@ -1,32 +1,33 @@ package r0 import ( - "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - + "errors" "net/http" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" + "github.com/getsentry/sentry-go" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func Logout(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - err := _auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId) +func Logout(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + err := auth_cache.InvalidateToken(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unable to logout") + return responses.InternalServerError(errors.New("unable to logout")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } -func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - err := _auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId) +func LogoutAll(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + err := auth_cache.InvalidateAllTokens(rctx, user.AccessToken, user.UserId) if err != nil { rctx.Log.Error(err) sentry.CaptureException(err) - return _responses.InternalServerError("unable to logout") + return responses.InternalServerError(errors.New("unable to logout")) } - return _responses.EmptyResponse{} + return responses.EmptyResponse{} } diff --git a/api/r0/preview_url.go b/api/r0/preview_url.go index e6fe8f8a..ef98a8ef 100644 --- a/api/r0/preview_url.go +++ b/api/r0/preview_url.go @@ -5,15 +5,15 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_preview" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type MatrixOpenGraph struct { @@ -29,9 +29,9 @@ type MatrixOpenGraph struct { ImageHeight int `json:"og:image:height,omitempty"` } -func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if !rctx.Config.UrlPreviews.Enabled { - return _responses.NotFoundError() + return responses.NotFoundError() } params := r.URL.Query() @@ -39,23 +39,27 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use // Parse the parameters urlStr := params.Get("url") tsStr := params.Get("ts") - ts := util.NowMillis() + + var ts int64 var err error if tsStr != "" { ts, err = strconv.ParseInt(tsStr, 10, 64) if err != nil { rctx.Log.Error("Error parsing ts: ", err) - return _responses.BadRequest(err.Error()) + return responses.BadRequest(err) } } + if ts == 0 { + ts = time.Now().UnixMilli() + } // Validate the URL if urlStr == "" { - return _responses.BadRequest("No url provided") + return responses.BadRequest(errors.New("No url provided")) } //goland:noinspection HttpUrlsUsage if strings.Index(urlStr, "http://") != 0 && strings.Index(urlStr, "https://") != 0 { - return _responses.BadRequest("Scheme not accepted") + return responses.BadRequest(errors.New("Scheme not accepted")) } languageHeader := rctx.Config.UrlPreviews.DefaultLanguage @@ -78,13 +82,13 @@ func PreviewUrl(r *http.Request, rctx rcontext.RequestContext, user _apimeta.Use } if err != nil { if errors.Is(err, common.ErrMediaNotFound) || errors.Is(err, common.ErrHostNotFound) { - return _responses.NotFoundError() - } else if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) { - return _responses.BadRequest(err.Error()) - } else { - sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.NotFoundError() + } + if errors.Is(err, common.ErrInvalidHost) || errors.Is(err, common.ErrHostNotAllowed) { + return responses.BadRequest(err) } + sentry.CaptureException(err) + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MatrixOpenGraph{ diff --git a/api/r0/public_config.go b/api/r0/public_config.go index 55715565..9a34fe66 100644 --- a/api/r0/public_config.go +++ b/api/r0/public_config.go @@ -4,9 +4,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) type PublicConfigResponse struct { @@ -15,7 +15,7 @@ type PublicConfigResponse struct { StorageMaxFiles int64 `json:"org.matrix.msc4034.storage.max_files,omitempty"` } -func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PublicConfig(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { uploadSize := rctx.Config.Uploads.ReportedMaxSizeBytes if uploadSize == 0 { uploadSize = rctx.Config.Uploads.MaxSizeBytes diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go index 50c88ec4..17bda434 100644 --- a/api/r0/thumbnail.go +++ b/api/r0/thumbnail.go @@ -4,11 +4,12 @@ import ( "errors" "net/http" "strconv" + "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" @@ -20,22 +21,22 @@ import ( "github.com/t2bot/matrix-media-repo/common/rcontext" ) -func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") allowRedirect := r.URL.Query().Get("allow_redirect") timeoutMs := r.URL.Query().Get("timeout_ms") - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.BadRequest("allow_remote flag does not appear to be a boolean") + return responses.BadRequest(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -44,14 +45,18 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if allowRedirect != "" { parsedFlag, err := strconv.ParseBool(allowRedirect) if err != nil { - return _responses.BadRequest("allow_redirect flag does not appear to be a boolean") + return responses.BadRequest(errors.New("allow_redirect flag does not appear to be a boolean")) } canRedirect = parsedFlag } - blockFor, err := util.CalcBlockForDuration(timeoutMs) + timeoutMS, err := strconv.ParseInt(timeoutMs, 10, 64) if err != nil { - return _responses.BadRequest("timeout_ms does not appear to be an integer") + return responses.BadRequest(errors.New("timeout_ms does not appear to be an integer")) + } + timeout := time.Duration(timeoutMS) * time.Millisecond + if timeout > time.Minute { + timeout = time.Minute } rctx = rctx.LogWithFields(logrus.Fields{ @@ -63,7 +68,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } widthStr := r.URL.Query().Get("width") @@ -75,7 +80,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } if widthStr == "" || heightStr == "" { - return _responses.BadRequest("Width and height are required") + return responses.BadRequest(errors.New("Width and height are required")) } width := 0 @@ -85,21 +90,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if widthStr != "" { parsedWidth, err := strconv.Atoi(widthStr) if err != nil { - return _responses.BadRequest("Width does not appear to be an integer") + return responses.BadRequest(errors.New("Width does not appear to be an integer")) } width = parsedWidth } if heightStr != "" { parsedHeight, err := strconv.Atoi(heightStr) if err != nil { - return _responses.BadRequest("Height does not appear to be an integer") + return responses.BadRequest(errors.New("Height does not appear to be an integer")) } height = parsedHeight } if animatedStr != "" { parsedFlag, err := strconv.ParseBool(animatedStr) if err != nil { - return _responses.BadRequest("Animated flag does not appear to be a boolean") + return responses.BadRequest(errors.New("Animated flag does not appear to be a boolean")) } animated = parsedFlag } @@ -115,13 +120,13 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta }) if width <= 0 || height <= 0 { - return _responses.BadRequest("Width and height must be greater than zero") + return responses.BadRequest(errors.New("Width and height must be greater than zero")) } thumbnail, stream, err := pipeline_thumbnail.Execute(rctx, server, mediaId, pipeline_thumbnail.ThumbnailOpts{ DownloadOpts: pipeline_download.DownloadOpts{ FetchRemoteIfNeeded: downloadRemote, - BlockForReadUntil: blockFor, + BlockForReadUntil: timeout, RecordOnly: false, // overridden CanRedirect: canRedirect, }, @@ -133,21 +138,21 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { var redirect datastores.RedirectError if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } else if errors.Is(err, common.ErrMediaDimensionsTooSmall) { if stream == nil { - return _responses.NotFoundError() // something went wrong so just 404 the thumbnail + return responses.NotFoundError() // something went wrong so just 404 the thumbnail } // We have a stream, and an error about image size, so we know there should be a media record @@ -156,9 +161,9 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta if err != nil { rctx.Log.Error("Unexpected error locating media record: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } else { - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: record.ContentType, Filename: record.UploadName, SizeBytes: record.SizeBytes, @@ -167,14 +172,14 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta } } } else if errors.As(err, &redirect) { - return _responses.Redirect(redirect.RedirectUrl) + return responses.Redirect(redirect.RedirectUrl) } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } - return &_responses.DownloadResponse{ + return &responses.DownloadResponse{ ContentType: thumbnail.ContentType, Filename: "thumbnail" + util.ExtensionForContentType(thumbnail.ContentType), SizeBytes: thumbnail.SizeBytes, diff --git a/api/r0/upload_async.go b/api/r0/upload_async.go index 30800652..01d50f8e 100644 --- a/api/r0/upload_async.go +++ b/api/r0/upload_async.go @@ -7,17 +7,17 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_upload" ) -func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) filename := filepath.Base(r.URL.Query().Get("filename")) rctx = rctx.LogWithFields(logrus.Fields{ @@ -27,7 +27,7 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apime }) if r.Host != server { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeNotFound, Message: "Upload request is for another domain.", InternalCode: common.ErrCodeForbidden, @@ -48,21 +48,21 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apime _, err := pipeline_upload.ExecutePut(rctx, server, mediaId, r.Body, contentType, filename, user.UserId) if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } else if errors.Is(err, common.ErrAlreadyUploaded) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeCannotOverwrite, Message: "This media has already been uploaded.", InternalCode: common.ErrCodeCannotOverwrite, } } else if errors.Is(err, common.ErrWrongUser) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeForbidden, Message: "You do not have permission to upload this media.", InternalCode: common.ErrCodeForbidden, } } else if errors.Is(err, common.ErrExpired) { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeNotFound, Message: "Media expired or not found.", InternalCode: common.ErrCodeNotFound, @@ -70,10 +70,10 @@ func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apime } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ - //ContentUri: util.MxcUri(media.Origin, media.MediaId), // This endpoint doesn't return a URI + // ContentUri: util.MxcUri(media.Origin, media.MediaId), // This endpoint doesn't return a URI } } diff --git a/api/r0/upload_sync.go b/api/r0/upload_sync.go index 87f81beb..daf8df5b 100644 --- a/api/r0/upload_sync.go +++ b/api/r0/upload_sync.go @@ -8,8 +8,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -21,7 +21,7 @@ type MediaUploadedResponse struct { ContentUri string `json:"content_uri,omitempty"` } -func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { filename := filepath.Base(r.URL.Query().Get("filename")) rctx = rctx.LogWithFields(logrus.Fields{ @@ -42,11 +42,11 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimet media, err := pipeline_upload.Execute(rctx, r.Host, "", r.Body, contentType, filename, user.UserId, datastores.LocalMediaKind) if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } return &MediaUploadedResponse{ @@ -54,26 +54,26 @@ func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimet } } -func uploadRequestSizeCheck(rctx rcontext.RequestContext, r *http.Request) *_responses.ErrorResponse { +func uploadRequestSizeCheck(rctx rcontext.RequestContext, r *http.Request) *responses.ErrorResponse { maxSize := rctx.Config.Uploads.MaxSizeBytes minSize := rctx.Config.Uploads.MinSizeBytes if maxSize > 0 || minSize > 0 { if r.ContentLength > 0 { if maxSize > 0 && maxSize < r.ContentLength { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } if minSize > 0 && minSize > r.ContentLength { - return _responses.RequestTooSmall() + return responses.RequestTooSmall() } } else { header := r.Header.Get("Content-Length") if header != "" { parsed, _ := strconv.ParseInt(header, 10, 64) if maxSize > 0 && maxSize < parsed { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } if minSize > 0 && minSize > parsed { - return _responses.RequestTooSmall() + return responses.RequestTooSmall() } } } diff --git a/api/_responses/content.go b/api/responses/content.go similarity index 96% rename from api/_responses/content.go rename to api/responses/content.go index 735e4b1b..775ef330 100644 --- a/api/_responses/content.go +++ b/api/responses/content.go @@ -1,4 +1,4 @@ -package _responses +package responses import "io" diff --git a/api/_responses/errors.go b/api/responses/errors.go similarity index 76% rename from api/_responses/errors.go rename to api/responses/errors.go index f22b33d1..38006e27 100644 --- a/api/_responses/errors.go +++ b/api/responses/errors.go @@ -1,6 +1,10 @@ -package _responses +package responses -import "github.com/t2bot/matrix-media-repo/common" +import ( + "fmt" + + "github.com/t2bot/matrix-media-repo/common" +) type ErrorResponse struct { Code string `json:"errcode"` @@ -8,12 +12,12 @@ type ErrorResponse struct { InternalCode string `json:"mr_errcode"` } -func InternalServerError(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown} +func InternalServerError(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeUnknown} } -func BadGatewayError(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeUnknown} +func BadGatewayError(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeUnknown} } func MethodNotAllowed() *ErrorResponse { @@ -48,8 +52,8 @@ func GuestAuthFailed() *ErrorResponse { return &ErrorResponse{common.ErrCodeNoGuests, "Guests cannot use this endpoint", common.ErrCodeNoGuests} } -func BadRequest(message string) *ErrorResponse { - return &ErrorResponse{common.ErrCodeUnknown, message, common.ErrCodeBadRequest} +func BadRequest(err error) *ErrorResponse { + return &ErrorResponse{common.ErrCodeUnknown, fmt.Sprint(err), common.ErrCodeBadRequest} } func QuotaExceeded() *ErrorResponse { diff --git a/api/_responses/meta.go b/api/responses/meta.go similarity index 75% rename from api/_responses/meta.go rename to api/responses/meta.go index 0971bd72..cbb7febb 100644 --- a/api/_responses/meta.go +++ b/api/responses/meta.go @@ -1,4 +1,4 @@ -package _responses +package responses type DoNotCacheResponse struct { Payload interface{} diff --git a/api/_responses/redirect.go b/api/responses/redirect.go similarity index 87% rename from api/_responses/redirect.go rename to api/responses/redirect.go index 1e8c66c4..b94e738c 100644 --- a/api/_responses/redirect.go +++ b/api/responses/redirect.go @@ -1,4 +1,4 @@ -package _responses +package responses type RedirectResponse struct { ToUrl string diff --git a/api/router.go b/api/router.go index 2f84be5b..540c4d9b 100644 --- a/api/router.go +++ b/api/router.go @@ -9,8 +9,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/util" ) @@ -21,7 +21,7 @@ func buildPrimaryRouter() *httprouter.Router { router.MethodNotAllowed = http.HandlerFunc(methodNotAllowedFn) router.NotFound = http.HandlerFunc(notFoundFn) router.HandleOPTIONS = true - router.GlobalOPTIONS = _routers.NewInstallHeadersRouter(http.HandlerFunc(finishCorsFn)) + router.GlobalOPTIONS = routers.NewInstallHeadersRouter(http.HandlerFunc(finishCorsFn)) router.PanicHandler = panicFn return router } @@ -29,25 +29,25 @@ func buildPrimaryRouter() *httprouter.Router { func methodNotAllowedFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusMethodNotAllowed) - if b, err := json.Marshal(_responses.MethodNotAllowed()); err != nil { - panic(errors.New("error preparing MethodNotAllowed: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending MethodNotAllowed: " + err.Error())) - } + reponse, err := json.Marshal(responses.MethodNotAllowed()) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing MethodNotAllowed: %v", err)) + logrus.Errorf("error preparing MethodNotAllowed: %v", err) + return } + w.Write(reponse) } func notFoundFn(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) - if b, err := json.Marshal(_responses.NotFoundError()); err != nil { - panic(errors.New("error preparing NotFound: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending NotFound: " + err.Error())) - } + reponse, err := json.Marshal(responses.NotFoundError()) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing NotFound: %v", err)) + logrus.Errorf("error preparing NotFound: %v", err) + return } + w.Write(reponse) } func finishCorsFn(w http.ResponseWriter, r *http.Request) { @@ -66,11 +66,12 @@ func panicFn(w http.ResponseWriter, r *http.Request, i interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusInternalServerError) - if b, err := json.Marshal(_responses.InternalServerError("unexpected error")); err != nil { - panic(errors.New("error preparing InternalServerError: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending InternalServerError: " + err.Error())) - } + + reponse, err := json.Marshal(responses.InternalServerError(errors.New("unexpected error"))) + if err != nil { + sentry.CaptureException(fmt.Errorf("error preparing InternalServerError: %v", err)) + logrus.Errorf("error preparing InternalServerError: %v", err) + return } + w.Write(reponse) } diff --git a/api/_routers/00-install-params.go b/api/routers/00-install-params.go similarity index 58% rename from api/_routers/00-install-params.go rename to api/routers/00-install-params.go index 11b42f33..6f75903c 100644 --- a/api/_routers/00-install-params.go +++ b/api/routers/00-install-params.go @@ -1,32 +1,22 @@ -package _routers +package routers import ( "context" - "errors" "net/http" "regexp" "github.com/julienschmidt/httprouter" ) -func localCompile(expr string) *regexp.Regexp { - r, err := regexp.Compile(expr) - if err != nil { - panic(errors.New("error compiling expression: " + expr + " | " + err.Error())) - } - return r -} - -var ServerNameRegex = localCompile("[a-zA-Z0-9.:\\-_]+") - -//var NumericIdRegex = localCompile("[0-9]+") +var ServerNameRegex = regexp.MustCompile("[a-zA-Z0-9.:\\-_]+") +// var NumericIdRegex = regexp.MustCompile("[0-9]+") func GetParam(name string, r *http.Request) string { - p := httprouter.ParamsFromContext(r.Context()) - if p == nil { + parameter := httprouter.ParamsFromContext(r.Context()) + if parameter == nil { return "" } - return p.ByName(name) + return parameter.ByName(name) } func ForceSetParam(name string, val string, r *http.Request) *http.Request { diff --git a/api/_routers/01-install_metadata.go b/api/routers/01-install_metadata.go similarity index 66% rename from api/_routers/01-install_metadata.go rename to api/routers/01-install_metadata.go index db3fa1d0..fe5a4688 100644 --- a/api/_routers/01-install_metadata.go +++ b/api/routers/01-install_metadata.go @@ -1,9 +1,10 @@ -package _routers +package routers import ( "context" "net/http" "strconv" + "time" "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common" @@ -37,8 +38,8 @@ func NewInstallMetadataRouter(ignoreHost bool, actionName string, counter *Reque } } -func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { - requestId := i.counter.NextId() +func (router *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + requestId := router.counter.NextId() logger := logrus.WithFields(logrus.Fields{ "method": r.Method, "host": r.Host, @@ -52,46 +53,46 @@ func (i *InstallMetadataRouter) ServeHTTP(w http.ResponseWriter, r *http.Request }) ctx := r.Context() - ctx = context.WithValue(ctx, common.ContextRequestStartTime, util.NowMillis()) + ctx = context.WithValue(ctx, common.ContextRequestStartTime, time.Now()) ctx = context.WithValue(ctx, common.ContextRequestId, requestId) - ctx = context.WithValue(ctx, common.ContextAction, i.actionName) - ctx = context.WithValue(ctx, common.ContextIgnoreHost, i.ignoreHost) + ctx = context.WithValue(ctx, common.ContextAction, router.actionName) + ctx = context.WithValue(ctx, common.ContextIgnoreHost, router.ignoreHost) ctx = context.WithValue(ctx, common.ContextLogger, logger) r = r.WithContext(ctx) - if i.next != nil { - i.next.ServeHTTP(w, r) + if router.next != nil { + router.next.ServeHTTP(w, r) } } func GetActionName(r *http.Request) string { - x, ok := r.Context().Value(common.ContextAction).(string) + action, ok := r.Context().Value(common.ContextAction).(string) if !ok { return "" } - return x + return action } func ShouldIgnoreHost(r *http.Request) bool { - x, ok := r.Context().Value(common.ContextIgnoreHost).(bool) + ignoreHost, ok := r.Context().Value(common.ContextIgnoreHost).(bool) if !ok { return false } - return x + return ignoreHost } func GetLogger(r *http.Request) *logrus.Entry { - x, ok := r.Context().Value(common.ContextLogger).(*logrus.Entry) + log, ok := r.Context().Value(common.ContextLogger).(*logrus.Entry) if !ok { return nil } - return x + return log } func GetRequestDuration(r *http.Request) float64 { - x, ok := r.Context().Value(common.ContextRequestStartTime).(int64) + duration, ok := r.Context().Value(common.ContextRequestStartTime).(time.Time) if !ok { return -1 } - return float64(util.NowMillis()-x) / 1000.0 + return float64(time.Since(duration).Milliseconds()) } diff --git a/api/_routers/02-install-headers.go b/api/routers/02-install-headers.go similarity index 98% rename from api/_routers/02-install-headers.go rename to api/routers/02-install-headers.go index 8d3547ce..ad18b549 100644 --- a/api/_routers/02-install-headers.go +++ b/api/routers/02-install-headers.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/_routers/03-host_detection.go b/api/routers/03-host_detection.go similarity index 85% rename from api/_routers/03-host_detection.go rename to api/routers/03-host_detection.go index 830e48f0..eb6a105c 100644 --- a/api/_routers/03-host_detection.go +++ b/api/routers/03-host_detection.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "context" @@ -12,7 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sebest/xff" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/metrics" @@ -60,13 +60,14 @@ func (h *HostRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { }).Inc() logger.Warnf("The server name provided ('%s') in the Host header is not configured, or the request was made directly to the media repo. Please specify a Host header and check your reverse proxy configuration. The request is being rejected.", r.Host) w.WriteHeader(http.StatusBadGateway) - if b, err := json.Marshal(_responses.BadGatewayError("Review server logs to continue")); err != nil { - panic(errors.New("error preparing BadGatewayError: " + err.Error())) - } else { - if _, err = w.Write(b); err != nil { - panic(errors.New("error sending BadGatewayError: " + err.Error())) - } + + var b []byte + if b, err = json.Marshal(responses.BadGatewayError(errors.New("Review server logs to continue"))); err != nil { + logger.Errorf("Error preparing BadGateway: %v", err) + sentry.CaptureException(err) + return } + w.Write(b) return // don't call next handler } diff --git a/api/_routers/04-request-metrics.go b/api/routers/04-request-metrics.go similarity index 97% rename from api/_routers/04-request-metrics.go rename to api/routers/04-request-metrics.go index 59c1b24a..9de5dac1 100644 --- a/api/_routers/04-request-metrics.go +++ b/api/routers/04-request-metrics.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/_routers/97-optional-access-token.go b/api/routers/97-optional-access-token.go similarity index 72% rename from api/_routers/97-optional-access-token.go rename to api/routers/97-optional-access-token.go index 9950b5bc..5dc7bf6d 100644 --- a/api/_routers/97-optional-access-token.go +++ b/api/routers/97-optional-access-token.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "errors" @@ -6,9 +6,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/auth_cache" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/matrix" @@ -19,7 +19,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { accessToken := util.GetAccessTokenFromRequest(r) if accessToken == "" { - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "", AccessToken: "", IsShared: false, @@ -27,19 +27,19 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { } if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token { ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true, }) } appserviceUserId := util.GetAppserviceUserIdFromRequest(r) - userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId) + userId, err := auth_cache.GetUserId(ctx, accessToken, appserviceUserId) if err != nil { if !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError("unexpected error validating access token") + return responses.InternalServerError(errors.New("unexpected error validating access token")) } ctx.Log.Warn("Failed to verify token (non-fatal): ", err) @@ -47,7 +47,7 @@ func OptionalAccessToken(generator GeneratorWithUserFn) GeneratorFn { } ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: userId, AccessToken: accessToken, IsShared: false, diff --git a/api/_routers/97-require-access-token.go b/api/routers/97-require-access-token.go similarity index 71% rename from api/_routers/97-require-access-token.go rename to api/routers/97-require-access-token.go index ba7dda5e..f01b6e6e 100644 --- a/api/_routers/97-require-access-token.go +++ b/api/routers/97-require-access-token.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "errors" @@ -6,9 +6,9 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/auth_cache" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -16,13 +16,13 @@ import ( "github.com/t2bot/matrix-media-repo/util" ) -type GeneratorWithUserFn = func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} +type GeneratorWithUserFn = func(r *http.Request, ctx rcontext.RequestContext, user apimeta.UserInfo) interface{} func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { accessToken := util.GetAccessTokenFromRequest(r) if accessToken == "" { - return &_responses.ErrorResponse{ + return &responses.ErrorResponse{ Code: common.ErrCodeMissingToken, Message: "no token provided (required)", InternalCode: common.ErrCodeMissingToken, @@ -30,28 +30,28 @@ func RequireAccessToken(generator GeneratorWithUserFn) GeneratorFn { } if config.Get().SharedSecret.Enabled && accessToken == config.Get().SharedSecret.Token { ctx = ctx.LogWithFields(logrus.Fields{"sharedSecretAuth": true}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: "@sharedsecret", AccessToken: accessToken, IsShared: true, }) } appserviceUserId := util.GetAppserviceUserIdFromRequest(r) - userId, err := _auth_cache.GetUserId(ctx, accessToken, appserviceUserId) + userId, err := auth_cache.GetUserId(ctx, accessToken, appserviceUserId) if err != nil || userId == "" { if errors.Is(err, matrix.ErrGuestToken) { - return _responses.GuestAuthFailed() + return responses.GuestAuthFailed() } if err != nil && !errors.Is(err, matrix.ErrInvalidToken) { sentry.CaptureException(err) ctx.Log.Error("Error verifying token: ", err) - return _responses.InternalServerError("unexpected error validating access token") + return responses.InternalServerError(errors.New("unexpected error validating access token")) } - return _responses.AuthFailed() + return responses.AuthFailed() } ctx = ctx.LogWithFields(logrus.Fields{"authUserId": userId}) - return generator(r, ctx, _apimeta.UserInfo{ + return generator(r, ctx, apimeta.UserInfo{ UserId: userId, AccessToken: accessToken, IsShared: false, diff --git a/api/_routers/97-require-repo-admin.go b/api/routers/97-require-repo-admin.go similarity index 57% rename from api/_routers/97-require-repo-admin.go rename to api/routers/97-require-repo-admin.go index cac1c38a..0b1d3916 100644 --- a/api/_routers/97-require-repo-admin.go +++ b/api/routers/97-require-repo-admin.go @@ -1,24 +1,25 @@ -package _routers +package routers import ( - "errors" "net/http" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/sirupsen/logrus" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" ) func RequireRepoAdmin(generator GeneratorWithUserFn) GeneratorFn { return func(r *http.Request, ctx rcontext.RequestContext) interface{} { - return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { + return RequireAccessToken(func(r *http.Request, ctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { if user.UserId == "" { - panic(errors.New("safety check failed: Repo admin access check received empty user ID")) + logrus.Error("safety check failed: Repo admin access check received empty user ID") + return responses.AuthFailed() } if !user.IsShared && !util.IsGlobalAdmin(user.UserId) { - return _responses.AuthFailed() + return responses.AuthFailed() } return generator(r, ctx, user) diff --git a/api/_routers/98-use-rcontext.go b/api/routers/98-use-rcontext.go similarity index 86% rename from api/_routers/98-use-rcontext.go rename to api/routers/98-use-rcontext.go index 35e36369..fe385e1a 100644 --- a/api/_routers/98-use-rcontext.go +++ b/api/routers/98-use-rcontext.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "bytes" @@ -16,7 +16,7 @@ import ( "github.com/alioygur/is" "github.com/getsentry/sentry-go" "github.com/t2bot/gotd-contrib/http_range" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/util" @@ -52,11 +52,11 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { var res interface{} res = c.generatorFn(r, rctx) if res == nil { - res = &_responses.EmptyResponse{} + res = &responses.EmptyResponse{} } shouldCache := true - wrappedRes, isNoCache := res.(*_responses.DoNotCacheResponse) + wrappedRes, isNoCache := res.(*responses.DoNotCacheResponse) if isNoCache { shouldCache = false res = wrappedRes.Payload @@ -65,7 +65,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { headers := w.Header() // Check for redirection early - if redirect, isRedirect := res.(*_responses.RedirectResponse); isRedirect { + if redirect, isRedirect := res.(*responses.RedirectResponse); isRedirect { log.Infof("Replying with result: %T <%s>", res, redirect.ToUrl) headers.Set("Location", redirect.ToUrl) r = writeStatusCode(w, r, http.StatusTemporaryRedirect) @@ -73,7 +73,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // Check for HTML response and reply accordingly - if htmlRes, isHtml := res.(*_responses.HtmlResponse); isHtml { + if htmlRes, isHtml := res.(*responses.HtmlResponse); isHtml { log.Infof("Replying with result: %T <%d chars of html>", res, len(htmlRes.HTML)) // Write out HTML here, now that we know it's happening @@ -88,7 +88,7 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { r = writeStatusCode(w, r, http.StatusOK) if _, err := w.Write([]byte(htmlRes.HTML)); err != nil { - panic(errors.New("error sending HtmlResponse: " + err.Error())) + panic(fmt.Errorf("error sending HtmlResponse: %w", err)) } return // don't continue } @@ -100,20 +100,20 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { var contentType string beforeParseDownload: log.Infof("Replying with result: %T %+v", res, res) - if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload { + if downloadRes, isDownload := res.(*responses.DownloadResponse); isDownload { ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes) if errors.Is(err, http_range.ErrInvalid) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("invalid range header") + res = responses.BadRequest(errors.New("invalid range header")) goto beforeParseDownload // reprocess `res` } else if errors.Is(err, http_range.ErrNoOverlap) { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("out of range") + res = responses.BadRequest(errors.New("out of range")) goto beforeParseDownload // reprocess `res` } if len(ranges) > 1 { proposedStatusCode = http.StatusRequestedRangeNotSatisfiable - res = _responses.BadRequest("only 1 range is supported") + res = responses.BadRequest(errors.New("only 1 range is supported")) goto beforeParseDownload // reprocess `res` } @@ -182,10 +182,10 @@ beforeParseDownload: } // Try to find a suitable error code, if one is needed - if errRes, isError := res.(_responses.ErrorResponse); isError { + if errRes, isError := res.(responses.ErrorResponse); isError { res = &errRes // just fix it } - if errRes, isError := res.(*_responses.ErrorResponse); isError && proposedStatusCode == http.StatusOK { + if errRes, isError := res.(*responses.ErrorResponse); isError && proposedStatusCode == http.StatusOK { switch errRes.InternalCode { case common.ErrCodeMissingToken: proposedStatusCode = http.StatusUnauthorized @@ -225,7 +225,9 @@ beforeParseDownload: contentType = "application/json" b, err := json.Marshal(res) if err != nil { - panic(err) // blow up this request + sentry.CaptureException(err) + log.Errorf("Failed to marshal response: %v", err) + return } stream = io.NopCloser(bytes.NewReader(b)) expectedBytes = int64(len(b)) @@ -253,10 +255,11 @@ beforeParseDownload: defer stream.Close() written, err := io.Copy(w, stream) if err != nil { - panic(err) // blow up this request + log.Errorf("Failed to write response: %v", err) } if expectedBytes > 0 && written != expectedBytes { - panic(errors.New(fmt.Sprintf("mismatch transfer size: %d expected, %d sent", expectedBytes, written))) + sentry.CaptureException(fmt.Errorf("expected %d bytes, but only wrote %d bytes for %q", expectedBytes, written, r.URL.Path)) + log.Warnf("Expected %d bytes, but only wrote %d bytes", expectedBytes, written) } } diff --git a/api/_routers/99-response-metrics.go b/api/routers/99-response-metrics.go similarity index 97% rename from api/_routers/99-response-metrics.go rename to api/routers/99-response-metrics.go index 6fcc1d90..4839a09d 100644 --- a/api/_routers/99-response-metrics.go +++ b/api/routers/99-response-metrics.go @@ -1,4 +1,4 @@ -package _routers +package routers import ( "net/http" diff --git a/api/routes.go b/api/routes.go index 423a6e32..100ccaae 100644 --- a/api/routes.go +++ b/api/routes.go @@ -7,10 +7,10 @@ import ( "github.com/julienschmidt/httprouter" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_debug" - "github.com/t2bot/matrix-media-repo/api/_routers" "github.com/t2bot/matrix-media-repo/api/custom" + "github.com/t2bot/matrix-media-repo/api/debug" "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/api/unstable" v1 "github.com/t2bot/matrix-media-repo/api/v1" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -20,104 +20,104 @@ const PrefixMedia = "/_matrix/media" const PrefixClient = "/_matrix/client" func buildRoutes() http.Handler { - counter := &_routers.RequestCounter{} + counter := &routers.RequestCounter{} router := buildPrimaryRouter() pprofSecret := os.Getenv("MEDIA_PPROF_SECRET_KEY") if pprofSecret != "" { logrus.Warn("Enabling pprof/debug http endpoints") - _debug.BindPprofEndpoints(router, pprofSecret) + debug.BindPprofEndpoints(router, pprofSecret) } // Standard (spec) features - register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter)) - register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter)) - downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMedia), "download", counter) + register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter)) + register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter)) + downloadRoute := makeRoute(routers.OptionalAccessToken(r0.DownloadMedia), "download", counter) register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute) register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute) - register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) - register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)) - register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter)) - register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter)) - register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", counter)) - register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter)) - register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.CreateMedia), "create", counter)) + register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter)) + register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)) + register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(routers.OptionalAccessToken(r0.Identicon), "identicon", counter)) + register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.PublicConfig), "config", counter)) + register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.Logout), "logout", counter)) + register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter)) + register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(routers.RequireAccessToken(v1.CreateMedia), "create", counter)) // Custom features - register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter)) - register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.MediaInfo), "info", counter)) - purgeOneRoute := makeRoute(_routers.RequireAccessToken(custom.PurgeIndividualRecord), "purge_individual_media", counter) + register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter)) + register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(routers.RequireAccessToken(unstable.MediaInfo), "info", counter)) + purgeOneRoute := makeRoute(routers.RequireAccessToken(custom.PurgeIndividualRecord), "purge_individual_media", counter) register([]string{"DELETE"}, PrefixMedia, "download/:server/:mediaId", mxUnstable, router, purgeOneRoute) - register([]string{"GET"}, PrefixMedia, "usage", msc4034, router, makeRoute(_routers.RequireAccessToken(unstable.PublicUsage), "usage", counter)) + register([]string{"GET"}, PrefixMedia, "usage", msc4034, router, makeRoute(routers.RequireAccessToken(unstable.PublicUsage), "usage", counter)) // Custom and top-level features - router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(_routers.OptionalAccessToken(custom.GetVersion), "get_version", counter)) - healthzRoute := makeRoute(_routers.OptionalAccessToken(custom.GetHealthz), "healthz", counter) // Note: healthz handling is special in makeRoute() + router.Handler("GET", fmt.Sprintf("%s/version", PrefixMedia), makeRoute(routers.OptionalAccessToken(custom.GetVersion), "get_version", counter)) + healthzRoute := makeRoute(routers.OptionalAccessToken(custom.GetHealthz), "healthz", counter) // Note: healthz handling is special in makeRoute() router.Handler("GET", "/healthz", healthzRoute) router.Handler("HEAD", "/healthz", healthzRoute) // Register the Synapse admin API endpoints we're compatible with - synUserStatsRoute := makeRoute(_routers.RequireAccessToken(custom.SynGetUsersMediaStats), "users_usage_stats", counter) + synUserStatsRoute := makeRoute(routers.RequireAccessToken(custom.SynGetUsersMediaStats), "users_usage_stats", counter) register([]string{"GET"}, synapse.PrefixAdminApi, "statistics/users/media", mxV1, router, synUserStatsRoute) // All admin routes are unstable only - purgeRemoteRoute := makeRoute(_routers.RequireRepoAdmin(custom.PurgeRemoteMedia), "purge_remote_media", counter) + purgeRemoteRoute := makeRoute(routers.RequireRepoAdmin(custom.PurgeRemoteMedia), "purge_remote_media", counter) purgeBranch := branchedRoute([]branch{ {"remote", purgeRemoteRoute}, - {"old", makeRoute(_routers.RequireRepoAdmin(custom.PurgeOldMedia), "purge_old_media", counter)}, - {"quarantined", makeRoute(_routers.RequireAccessToken(custom.PurgeQuarantined), "purge_quarantined", counter)}, - {"user/:userId", makeRoute(_routers.RequireAccessToken(custom.PurgeUserMedia), "purge_user_media", counter)}, - {"room/:roomId", makeRoute(_routers.RequireAccessToken(custom.PurgeRoomMedia), "purge_room_media", counter)}, - {"server/:serverName", makeRoute(_routers.RequireAccessToken(custom.PurgeDomainMedia), "purge_domain_media", counter)}, + {"old", makeRoute(routers.RequireRepoAdmin(custom.PurgeOldMedia), "purge_old_media", counter)}, + {"quarantined", makeRoute(routers.RequireAccessToken(custom.PurgeQuarantined), "purge_quarantined", counter)}, + {"user/:userId", makeRoute(routers.RequireAccessToken(custom.PurgeUserMedia), "purge_user_media", counter)}, + {"room/:roomId", makeRoute(routers.RequireAccessToken(custom.PurgeRoomMedia), "purge_room_media", counter)}, + {"server/:serverName", makeRoute(routers.RequireAccessToken(custom.PurgeDomainMedia), "purge_domain_media", counter)}, {":server/:mediaId", purgeOneRoute}, }) register([]string{"POST"}, PrefixMedia, "admin/purge/*branch", mxUnstable, router, purgeBranch) register([]string{"POST"}, PrefixMedia, "admin/purge_remote", mxUnstable, router, purgeRemoteRoute) register([]string{"POST"}, PrefixClient, "admin/purge_media_cache", mxUnstable, router, purgeRemoteRoute) // synapse compat - quarantineRoomRoute := makeRoute(_routers.RequireAccessToken(custom.QuarantineRoomMedia), "quarantine_room", counter) + quarantineRoomRoute := makeRoute(routers.RequireAccessToken(custom.QuarantineRoomMedia), "quarantine_room", counter) quarantineBranch := branchedRoute([]branch{ {"room/:roomId", quarantineRoomRoute}, - {"user/:userId", makeRoute(_routers.RequireAccessToken(custom.QuarantineUserMedia), "quarantine_user", counter)}, - {"server/:serverName", makeRoute(_routers.RequireAccessToken(custom.QuarantineDomainMedia), "quarantine_domain", counter)}, - {":server/:mediaId", makeRoute(_routers.RequireAccessToken(custom.QuarantineMedia), "quarantine_media", counter)}, + {"user/:userId", makeRoute(routers.RequireAccessToken(custom.QuarantineUserMedia), "quarantine_user", counter)}, + {"server/:serverName", makeRoute(routers.RequireAccessToken(custom.QuarantineDomainMedia), "quarantine_domain", counter)}, + {":server/:mediaId", makeRoute(routers.RequireAccessToken(custom.QuarantineMedia), "quarantine_media", counter)}, }) register([]string{"POST"}, PrefixMedia, "admin/quarantine/*branch", mxUnstable, router, quarantineBranch) register([]string{"POST"}, PrefixClient, "admin/quarantine_media/:roomId", mxUnstable, router, quarantineRoomRoute) // synapse compat - register([]string{"GET"}, PrefixMedia, "admin/datastores/:datastoreId/size_estimate", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastoreStorageEstimate), "get_storage_estimate", counter)) - register([]string{"POST"}, PrefixMedia, "admin/datastores/:sourceDsId/transfer_to/:targetDsId", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.MigrateBetweenDatastores), "datastore_transfer", counter)) - register([]string{"GET"}, PrefixMedia, "admin/datastores", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDatastores), "list_datastores", counter)) - register([]string{"GET"}, PrefixMedia, "admin/federation/test/:serverName", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetFederationInfo), "federation_test", counter)) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetDomainUsage), "domain_usage", counter)) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUserUsage), "user_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/datastores/:datastoreId/size_estimate", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDatastoreStorageEstimate), "get_storage_estimate", counter)) + register([]string{"POST"}, PrefixMedia, "admin/datastores/:sourceDsId/transfer_to/:targetDsId", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.MigrateBetweenDatastores), "datastore_transfer", counter)) + register([]string{"GET"}, PrefixMedia, "admin/datastores", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDatastores), "list_datastores", counter)) + register([]string{"GET"}, PrefixMedia, "admin/federation/test/:serverName", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetFederationInfo), "federation_test", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetDomainUsage), "domain_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetUserUsage), "user_usage", counter)) register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/users-stats", mxUnstable, router, synUserStatsRoute) - register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/uploads", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.GetUploadsUsage), "uploads_usage", counter)) + register([]string{"GET"}, PrefixMedia, "admin/usage/:serverName/uploads", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.GetUploadsUsage), "uploads_usage", counter)) tasksBranch := branchedRoute([]branch{ - {"all", makeRoute(_routers.RequireRepoAdmin(custom.ListAllTasks), "list_all_background_tasks", counter)}, - {"unfinished", makeRoute(_routers.RequireRepoAdmin(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", counter)}, - {":taskId", makeRoute(_routers.RequireRepoAdmin(custom.GetTask), "get_background_task", counter)}, + {"all", makeRoute(routers.RequireRepoAdmin(custom.ListAllTasks), "list_all_background_tasks", counter)}, + {"unfinished", makeRoute(routers.RequireRepoAdmin(custom.ListUnfinishedTasks), "list_unfinished_background_tasks", counter)}, + {":taskId", makeRoute(routers.RequireRepoAdmin(custom.GetTask), "get_background_task", counter)}, }) register([]string{"GET"}, PrefixMedia, "admin/tasks/*branch", mxUnstable, router, tasksBranch) - register([]string{"POST"}, PrefixMedia, "admin/user/:userId/export", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.ExportUserData), "export_user_data", counter)) - register([]string{"POST"}, PrefixMedia, "admin/server/:serverName/export", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.ExportServerData), "export_server_data", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/view", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.ViewExport), "view_export", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/metadata", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.GetExportMetadata), "get_export_metadata", counter)) - register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/part/:partId", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.DownloadExportPart), "download_export_part", counter)) - register([]string{"DELETE"}, PrefixMedia, "admin/export/:exportId", mxUnstable, router, makeRoute(_routers.OptionalAccessToken(custom.DeleteExport), "delete_export", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.StartImport), "start_import", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import/:importId/part", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.AppendToImport), "append_to_import", counter)) - register([]string{"POST"}, PrefixMedia, "admin/import/:importId/close", mxUnstable, router, makeRoute(_routers.RequireRepoAdmin(custom.StopImport), "stop_import", counter)) - register([]string{"GET"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.GetAttributes), "get_media_attributes", counter)) - register([]string{"POST"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(_routers.RequireAccessToken(custom.SetAttributes), "set_media_attributes", counter)) + register([]string{"POST"}, PrefixMedia, "admin/user/:userId/export", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.ExportUserData), "export_user_data", counter)) + register([]string{"POST"}, PrefixMedia, "admin/server/:serverName/export", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.ExportServerData), "export_server_data", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/view", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.ViewExport), "view_export", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/metadata", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.GetExportMetadata), "get_export_metadata", counter)) + register([]string{"GET"}, PrefixMedia, "admin/export/:exportId/part/:partId", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.DownloadExportPart), "download_export_part", counter)) + register([]string{"DELETE"}, PrefixMedia, "admin/export/:exportId", mxUnstable, router, makeRoute(routers.OptionalAccessToken(custom.DeleteExport), "delete_export", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.StartImport), "start_import", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import/:importId/part", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.AppendToImport), "append_to_import", counter)) + register([]string{"POST"}, PrefixMedia, "admin/import/:importId/close", mxUnstable, router, makeRoute(routers.RequireRepoAdmin(custom.StopImport), "stop_import", counter)) + register([]string{"GET"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.GetAttributes), "get_media_attributes", counter)) + register([]string{"POST"}, PrefixMedia, "admin/media/:server/:mediaId/attributes", mxUnstable, router, makeRoute(routers.RequireAccessToken(custom.SetAttributes), "set_media_attributes", counter)) return router } -func makeRoute(generator _routers.GeneratorFn, name string, counter *_routers.RequestCounter) http.Handler { - return _routers.NewInstallMetadataRouter(name == "healthz", name, counter, - _routers.NewInstallHeadersRouter( - _routers.NewHostRouter( - _routers.NewMetricsRequestRouter( - _routers.NewRContextRouter(generator, _routers.NewMetricsResponseRouter(nil)), +func makeRoute(generator routers.GeneratorFn, name string, counter *routers.RequestCounter) http.Handler { + return routers.NewInstallMetadataRouter(name == "healthz", name, counter, + routers.NewInstallHeadersRouter( + routers.NewHostRouter( + routers.NewMetricsRequestRouter( + routers.NewRContextRouter(generator, routers.NewMetricsResponseRouter(nil)), ), ), )) diff --git a/api/unstable/info.go b/api/unstable/info.go index eb245991..951b023a 100644 --- a/api/unstable/info.go +++ b/api/unstable/info.go @@ -10,15 +10,15 @@ import ( "github.com/disintegration/imaging" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" "github.com/t2bot/matrix-media-repo/thumbnailing" - "github.com/t2bot/matrix-media-repo/thumbnailing/i" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/util" ) @@ -48,20 +48,20 @@ type MediaInfoResponse struct { NumChannels int `json:"num_channels,omitempty"` } -func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError("allow_remote flag does not appear to be a boolean") + return responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -74,7 +74,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } record, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{ @@ -85,22 +85,22 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User // Error handling copied from download endpoint if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } response := &MediaInfoResponse{ @@ -121,7 +121,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User } else if strings.HasPrefix(response.ContentType, "audio/") { generator, reconstructed, err := thumbnailing.GetGenerator(stream, response.ContentType, false) if err == nil { - if audiogenerator, ok := generator.(i.AudioGenerator); ok { + if audiogenerator, ok := generator.(preview.AudioGenerator); ok { audioInfo, err := audiogenerator.GetAudioData(reconstructed, 768, rctx) if err == nil { response.KeySamples = audioInfo.KeySamples @@ -137,7 +137,7 @@ func MediaInfo(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if err != nil { rctx.Log.Error("Unexpected error locating media thumbnails: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } if len(thumbs) > 0 { diff --git a/api/unstable/local_copy.go b/api/unstable/local_copy.go index f10c1adc..ed7b810d 100644 --- a/api/unstable/local_copy.go +++ b/api/unstable/local_copy.go @@ -8,10 +8,10 @@ import ( "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" - "github.com/t2bot/matrix-media-repo/api/_routers" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/api/r0" + "github.com/t2bot/matrix-media-repo/api/responses" + "github.com/t2bot/matrix-media-repo/api/routers" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/datastores" @@ -20,22 +20,22 @@ import ( "github.com/t2bot/matrix-media-repo/util" ) -func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { - server := _routers.GetParam("server", r) - mediaId := _routers.GetParam("mediaId", r) +func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { + server := routers.GetParam("server", r) + mediaId := routers.GetParam("mediaId", r) allowRemote := r.URL.Query().Get("allow_remote") rctx.Log.Warn("This endpoint is deprecated. See https://github.com/t2bot/matrix-media-repo/issues/422") - if !_routers.ServerNameRegex.MatchString(server) { - return _responses.BadRequest("invalid server ID") + if !routers.ServerNameRegex.MatchString(server) { + return responses.BadRequest(errors.New("invalid server ID")) } downloadRemote := true if allowRemote != "" { parsedFlag, err := strconv.ParseBool(allowRemote) if err != nil { - return _responses.InternalServerError("allow_remote flag does not appear to be a boolean") + return responses.InternalServerError(errors.New("allow_remote flag does not appear to be a boolean")) } downloadRemote = parsedFlag } @@ -48,11 +48,11 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) { rctx.Log.Warn("Request blocked due to domain being ignored.") - return _responses.MediaBlocked() + return responses.MediaBlocked() } if r.Host == server { - return _responses.BadRequest("Attempted to clone media to the same origin") + return responses.BadRequest(errors.New("Attempted to clone media to the same origin")) } // TODO: There's a lot of room for improvement here. Instead of re-uploading media, we should just update the DB. @@ -65,33 +65,33 @@ func LocalCopy(r *http.Request, rctx rcontext.RequestContext, user _apimeta.User // Error handling copied from download endpoint if err != nil { if errors.Is(err, common.ErrMediaNotFound) { - return _responses.NotFoundError() + return responses.NotFoundError() } else if errors.Is(err, common.ErrMediaTooLarge) { - return _responses.RequestTooLarge() + return responses.RequestTooLarge() } else if errors.Is(err, common.ErrMediaQuarantined) { rctx.Log.Debug("Quarantined media accessed. Has stream? ", stream != nil) if stream != nil { - return _responses.MakeQuarantinedImageResponse(stream) + return responses.MakeQuarantinedImageResponse(stream) } else { - return _responses.NotFoundError() // We lie for security + return responses.NotFoundError() // We lie for security } } else if errors.Is(err, common.ErrMediaNotYetUploaded) { - return _responses.NotYetUploaded() + return responses.NotYetUploaded() } rctx.Log.Error("Unexpected error locating media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } record, err = pipeline_upload.Execute(rctx, server, mediaId, stream, record.ContentType, record.UploadName, user.UserId, datastores.LocalMediaKind) // Error handling copied from upload(sync) endpoint if err != nil { if errors.Is(err, common.ErrQuotaExceeded) { - return _responses.QuotaExceeded() + return responses.QuotaExceeded() } rctx.Log.Error("Unexpected error uploading media: ", err) sentry.CaptureException(err) - return _responses.InternalServerError("Unexpected Error") + return responses.InternalServerError(errors.New("Unexpected Error")) } return &r0.MediaUploadedResponse{ diff --git a/api/unstable/public_usage.go b/api/unstable/public_usage.go index 7a6e2a96..d6380e2f 100644 --- a/api/unstable/public_usage.go +++ b/api/unstable/public_usage.go @@ -4,9 +4,9 @@ import ( "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" + "github.com/t2bot/matrix-media-repo/api/apimeta" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" ) type PublicUsageResponse struct { @@ -14,7 +14,7 @@ type PublicUsageResponse struct { StorageFiles int64 `json:"org.matrix.msc4034.storage.files,omitempty"` } -func PublicUsage(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func PublicUsage(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { storageUsed := int64(0) current, err := quota.Current(rctx, user.UserId, quota.MaxBytes) if err != nil { diff --git a/api/v1/create.go b/api/v1/create.go index d1f40932..592212cd 100644 --- a/api/v1/create.go +++ b/api/v1/create.go @@ -1,11 +1,12 @@ package v1 import ( + "errors" "net/http" "github.com/getsentry/sentry-go" - "github.com/t2bot/matrix-media-repo/api/_apimeta" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/apimeta" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_create" "github.com/t2bot/matrix-media-repo/util" @@ -16,12 +17,12 @@ type MediaCreatedResponse struct { ExpiresTs int64 `json:"unused_expires_at"` } -func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} { +func CreateMedia(r *http.Request, rctx rcontext.RequestContext, user apimeta.UserInfo) interface{} { id, err := pipeline_create.Execute(rctx, r.Host, user.UserId, pipeline_create.DefaultExpirationTime) if err != nil { rctx.Log.Error("Unexpected error creating media ID:", err) sentry.CaptureException(err) - return _responses.InternalServerError("unexpected error") + return responses.InternalServerError(errors.New("unexpected error")) } return &MediaCreatedResponse{ diff --git a/api/webserver.go b/api/webserver.go index 1877b9a7..fb0a4ba9 100644 --- a/api/webserver.go +++ b/api/webserver.go @@ -14,23 +14,19 @@ import ( "github.com/getsentry/sentry-go" sentryhttp "github.com/getsentry/sentry-go/http" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/api/_responses" + "github.com/t2bot/matrix-media-repo/api/responses" "github.com/t2bot/matrix-media-repo/common/config" ) -var srv *http.Server -var waitGroup = &sync.WaitGroup{} -var reload = false +var ( + srv *http.Server + waitGroup = &sync.WaitGroup{} + reload = false +) func Init() *sync.WaitGroup { address := net.JoinHostPort(config.Get().General.BindAddress, strconv.Itoa(config.Get().General.Port)) - //defer func() { - // if err := recover(); err != nil { - // logrus.Fatal(err) - // } - //}() - handler := buildRoutes() if config.Get().RateLimit.Enabled { @@ -41,8 +37,8 @@ func Init() *sync.WaitGroup { limiter.SetBurst(config.Get().RateLimit.BurstCount) limiter.SetMax(config.Get().RateLimit.RequestsPerSecond) - b, _ := json.Marshal(_responses.RateLimitReached()) - limiter.SetMessage(string(b)) + reponse, _ := json.Marshal(responses.RateLimitReached()) + limiter.SetMessage(string(reponse)) limiter.SetMessageContentType("application/json") handler = tollbooth.LimitHandler(limiter, handler) @@ -86,7 +82,7 @@ func Stop() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := srv.Shutdown(ctx); err != nil { - panic(err) + logrus.Fatalf("Could not gracefully shutdown the server: %v", err) } } } diff --git a/archival/v2archive/writer.go b/archival/v2archive/writer.go index 4eef0d8c..bb3c1e7d 100644 --- a/archival/v2archive/writer.go +++ b/archival/v2archive/writer.go @@ -150,7 +150,7 @@ func (w *ArchiveWriter) AppendMedia(file io.ReadCloser, info MediaInfo) (string, } internalName := fmt.Sprintf("%s__%s%s", info.Origin, info.MediaId, mime.Extension()) - createTime := util.FromMillis(info.CreationTs) + createTime := time.UnixMilli(info.CreationTs) size, sha256hash, err := w.putFile(br.GetRewoundReader(), internalName, createTime) if err != nil { @@ -210,7 +210,7 @@ func (w *ArchiveWriter) putFile(r io.Reader, name string, creationTime time.Time hasher := sha256.New() header := &tar.Header{ Name: name, - Mode: int64(0644), + Mode: int64(0o644), ModTime: creationTime, Size: i1, } @@ -242,7 +242,7 @@ func (w *ArchiveWriter) Finish() error { manifest := &Manifest{ Version: ManifestVersion, EntityId: w.entity, - CreatedTs: util.NowMillis(), + CreatedTs: time.Now().UnixMilli(), Media: w.mediaManifest, } pr, pw := io.Pipe() diff --git a/cmd/homeserver_live_importers/import_dendrite/main.go b/cmd/homeserver_live_importers/import_dendrite/main.go index b434c9fe..bcb44112 100644 --- a/cmd/homeserver_live_importers/import_dendrite/main.go +++ b/cmd/homeserver_live_importers/import_dendrite/main.go @@ -1,23 +1,24 @@ package main import ( - "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" + "github.com/sirupsen/logrus" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/dendrite" ) func main() { - cfg := _common.InitImportPsqlMatrixDownload("Dendrite") + cfg := internal.InitImportPsqlMatrixDownload("Dendrite") ctx := rcontext.Initial() ctx.Log.Debug("Connecting to homeserver database...") hsDb, err := dendrite.OpenDatabase(cfg.ConnectionString, cfg.ServerName) if err != nil { - panic(err) + logrus.Fatalf("Failed to open database: %v", err) } - _common.PsqlMatrixDownloadCopy[dendrite.LocalMedia](ctx, cfg, hsDb, func(record *dendrite.LocalMedia) (*_common.MediaMetadata, error) { - return &_common.MediaMetadata{ + internal.PsqlMatrixDownloadCopy[dendrite.LocalMedia](ctx, cfg, hsDb, func(record *dendrite.LocalMedia) (*internal.MediaMetadata, error) { + return &internal.MediaMetadata{ MediaId: record.MediaId, ContentType: record.ContentType, FileName: record.UploadName, diff --git a/cmd/homeserver_live_importers/import_synapse/main.go b/cmd/homeserver_live_importers/import_synapse/main.go index 03a9f442..b25c1182 100644 --- a/cmd/homeserver_live_importers/import_synapse/main.go +++ b/cmd/homeserver_live_importers/import_synapse/main.go @@ -1,23 +1,24 @@ package main import ( - "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/_common" + "github.com/sirupsen/logrus" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_live_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" ) func main() { - cfg := _common.InitImportPsqlMatrixDownload("Synapse") + cfg := internal.InitImportPsqlMatrixDownload("Synapse") ctx := rcontext.Initial() ctx.Log.Debug("Connecting to homeserver database...") hsDb, err := synapse.OpenDatabase(cfg.ConnectionString) if err != nil { - panic(err) + logrus.Fatalf("Failed to open database: %v", err) } - _common.PsqlMatrixDownloadCopy[synapse.LocalMedia](ctx, cfg, hsDb, func(record *synapse.LocalMedia) (*_common.MediaMetadata, error) { - return &_common.MediaMetadata{ + internal.PsqlMatrixDownloadCopy[synapse.LocalMedia](ctx, cfg, hsDb, func(record *synapse.LocalMedia) (*internal.MediaMetadata, error) { + return &internal.MediaMetadata{ MediaId: record.MediaId, ContentType: record.ContentType, FileName: record.UploadName, diff --git a/cmd/homeserver_live_importers/_common/download.go b/cmd/homeserver_live_importers/internal/download.go similarity index 99% rename from cmd/homeserver_live_importers/_common/download.go rename to cmd/homeserver_live_importers/internal/download.go index 2618784b..5e811875 100644 --- a/cmd/homeserver_live_importers/_common/download.go +++ b/cmd/homeserver_live_importers/internal/download.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "errors" diff --git a/cmd/homeserver_live_importers/_common/init_import.go b/cmd/homeserver_live_importers/internal/init_import.go similarity index 99% rename from cmd/homeserver_live_importers/_common/init_import.go rename to cmd/homeserver_live_importers/internal/init_import.go index 7b025157..5a8ffa95 100644 --- a/cmd/homeserver_live_importers/_common/init_import.go +++ b/cmd/homeserver_live_importers/internal/init_import.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag" diff --git a/cmd/homeserver_offline_exporters/import_to_synapse/main.go b/cmd/homeserver_offline_exporters/import_to_synapse/main.go index 3c7db1e2..850816f0 100644 --- a/cmd/homeserver_offline_exporters/import_to_synapse/main.go +++ b/cmd/homeserver_offline_exporters/import_to_synapse/main.go @@ -10,7 +10,7 @@ import ( "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_exporters/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_exporters/internal" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" @@ -33,7 +33,7 @@ var synapseDefaultSizes = []thumbnailSize{ } func main() { - cfg := _common.InitExportPsqlFlatFile("Synapse", "media_store_path") + cfg := internal.InitExportPsqlFlatFile("Synapse", "media_store_path") ctx := rcontext.InitialNoConfig() ctx.Config.Thumbnails = config.ThumbnailsConfig{ Types: []string{ @@ -58,7 +58,7 @@ func main() { panic(err) } - err = _common.ProcessArchiveDirectory(ctx, cfg.ServerName, cfg.SourcePath, func(record *v2archive.ManifestRecord, f io.ReadCloser) error { + err = internal.ProcessArchiveDirectory(ctx, cfg.ServerName, cfg.SourcePath, func(record *v2archive.ManifestRecord, f io.ReadCloser) error { defer f.Close() mxc := util.MxcUri(record.Origin, record.MediaId) diff --git a/cmd/homeserver_offline_exporters/_common/archive_reader.go b/cmd/homeserver_offline_exporters/internal/archive_reader.go similarity index 99% rename from cmd/homeserver_offline_exporters/_common/archive_reader.go rename to cmd/homeserver_offline_exporters/internal/archive_reader.go index 3974860d..26659248 100644 --- a/cmd/homeserver_offline_exporters/_common/archive_reader.go +++ b/cmd/homeserver_offline_exporters/internal/archive_reader.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "io" diff --git a/cmd/homeserver_offline_exporters/_common/init_export.go b/cmd/homeserver_offline_exporters/internal/init_export.go similarity index 99% rename from cmd/homeserver_offline_exporters/_common/init_export.go rename to cmd/homeserver_offline_exporters/internal/init_export.go index 398abce9..ea76978c 100644 --- a/cmd/homeserver_offline_exporters/_common/init_export.go +++ b/cmd/homeserver_offline_exporters/internal/init_export.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag" diff --git a/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go b/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go index 621702a6..201df83a 100644 --- a/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go +++ b/cmd/homeserver_offline_importers/export_dendrite_for_import/main.go @@ -6,14 +6,14 @@ import ( "path" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/dendrite" "github.com/t2bot/matrix-media-repo/util" ) func main() { - cfg := _common.InitExportPsqlFlatFile("Dendrite", "media_api.base_path") + cfg := internal.InitExportPsqlFlatFile("Dendrite", "media_api.base_path") ctx := rcontext.InitialNoConfig() ctx.Log.Debug("Connecting to homeserver database...") @@ -22,7 +22,7 @@ func main() { panic(err) } - _common.PsqlFlatFileArchive[dendrite.LocalMedia](ctx, cfg, hsDb, func(r *dendrite.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { + internal.PsqlFlatFileArchive[dendrite.LocalMedia](ctx, cfg, hsDb, func(r *dendrite.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { // For Base64Hash ABCCDD : // $importPath/A/B/CCDD/file diff --git a/cmd/homeserver_offline_importers/export_synapse_for_import/main.go b/cmd/homeserver_offline_importers/export_synapse_for_import/main.go index 6c8cfb3c..6f7839ec 100644 --- a/cmd/homeserver_offline_importers/export_synapse_for_import/main.go +++ b/cmd/homeserver_offline_importers/export_synapse_for_import/main.go @@ -7,14 +7,14 @@ import ( "strings" "github.com/t2bot/matrix-media-repo/archival/v2archive" - "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/_common" + "github.com/t2bot/matrix-media-repo/cmd/homeserver_offline_importers/internal" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/homeserver_interop/synapse" "github.com/t2bot/matrix-media-repo/util" ) func main() { - cfg := _common.InitExportPsqlFlatFile("Synapse", "media_store_path") + cfg := internal.InitExportPsqlFlatFile("Synapse", "media_store_path") ctx := rcontext.InitialNoConfig() ctx.Log.Debug("Connecting to homeserver database...") @@ -23,7 +23,7 @@ func main() { panic(err) } - _common.PsqlFlatFileArchive[synapse.LocalMedia](ctx, cfg, hsDb, func(r *synapse.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { + internal.PsqlFlatFileArchive[synapse.LocalMedia](ctx, cfg, hsDb, func(r *synapse.LocalMedia) (v2archive.MediaInfo, io.ReadCloser, error) { // For MediaID AABBCCDD : // $importPath/local_content/AA/BB/CCDD // diff --git a/cmd/homeserver_offline_importers/_common/archiver.go b/cmd/homeserver_offline_importers/internal/archiver.go similarity index 98% rename from cmd/homeserver_offline_importers/_common/archiver.go rename to cmd/homeserver_offline_importers/internal/archiver.go index 645730a4..6b46e2c3 100644 --- a/cmd/homeserver_offline_importers/_common/archiver.go +++ b/cmd/homeserver_offline_importers/internal/archiver.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "fmt" diff --git a/cmd/homeserver_offline_importers/_common/init_export.go b/cmd/homeserver_offline_importers/internal/init_export.go similarity index 99% rename from cmd/homeserver_offline_importers/_common/init_export.go rename to cmd/homeserver_offline_importers/internal/init_export.go index f9b3e7e8..dd0e6d32 100644 --- a/cmd/homeserver_offline_importers/_common/init_export.go +++ b/cmd/homeserver_offline_importers/internal/init_export.go @@ -1,4 +1,4 @@ -package _common +package internal import ( "flag" diff --git a/cmd/plugins/plugin_antispam_ocr/main.go b/cmd/plugins/plugin_antispam_ocr/main.go index d641488a..4b16d284 100644 --- a/cmd/plugins/plugin_antispam_ocr/main.go +++ b/cmd/plugins/plugin_antispam_ocr/main.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "regexp" + "slices" "strings" "time" @@ -77,7 +78,7 @@ func (a *AntispamOCR) CheckForSpam(b64 string, filename string, contentType stri if len(b) < a.minSize || len(b) > a.maxSize { return false, nil } - if !util.ArrayContains(a.contentTypes, contentType) { + if !slices.Contains(a.contentTypes, contentType) { return false, nil } if !a.userIdRegex.MatchString(userId) { diff --git a/cmd/utilities/combine_signing_keys/main.go b/cmd/utilities/combine_signing_keys/main.go index b6026cf4..6ed97a5e 100644 --- a/cmd/utilities/combine_signing_keys/main.go +++ b/cmd/utilities/combine_signing_keys/main.go @@ -5,7 +5,7 @@ import ( "os" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/utilities/_common" + "github.com/t2bot/matrix-media-repo/cmd/utilities/common" "github.com/t2bot/matrix-media-repo/homeserver_interop" "github.com/t2bot/matrix-media-repo/homeserver_interop/any_server" "github.com/t2bot/matrix-media-repo/util" @@ -36,7 +36,7 @@ func main() { } } - _common.EncodeSigningKeys(keysArray, *outputFormat, *outputFile) + common.EncodeSigningKeys(keysArray, *outputFormat, *outputFile) } func decodeKeys(fileName string) ([]*homeserver_interop.SigningKey, error) { diff --git a/cmd/utilities/_common/signing_key_export.go b/cmd/utilities/common/signing_key_export.go similarity index 98% rename from cmd/utilities/_common/signing_key_export.go rename to cmd/utilities/common/signing_key_export.go index e8a5d5a1..092a861e 100644 --- a/cmd/utilities/_common/signing_key_export.go +++ b/cmd/utilities/common/signing_key_export.go @@ -1,4 +1,4 @@ -package _common +package common import ( "os" diff --git a/cmd/utilities/generate_signing_key/main.go b/cmd/utilities/generate_signing_key/main.go index 3367164d..1853452e 100644 --- a/cmd/utilities/generate_signing_key/main.go +++ b/cmd/utilities/generate_signing_key/main.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/cmd/utilities/_common" + "github.com/t2bot/matrix-media-repo/cmd/utilities/common" "github.com/t2bot/matrix-media-repo/homeserver_interop" "github.com/t2bot/matrix-media-repo/homeserver_interop/any_server" ) @@ -44,7 +44,7 @@ func main() { logrus.Infof("Key ID will be 'ed25519:%s'", key.KeyVersion) - _common.EncodeSigningKeys([]*homeserver_interop.SigningKey{key}, *outputFormat, *outputFile) + common.EncodeSigningKeys([]*homeserver_interop.SigningKey{key}, *outputFormat, *outputFile) } func makeKeyVersion() string { @@ -53,7 +53,6 @@ func makeKeyVersion() string { for i := 0; i < len(chars); i++ { sort.Slice(chars, func(i int, j int) bool { c, err := rand.Read(buf) - // "should never happen" clauses if err != nil { panic(err) diff --git a/cmd/workers/media_repo/reloads.go b/cmd/workers/media_repo/reloads.go index f469b718..57099dfd 100644 --- a/cmd/workers/media_repo/reloads.go +++ b/cmd/workers/media_repo/reloads.go @@ -3,7 +3,7 @@ package main import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/api" - "github.com/t2bot/matrix-media-repo/api/_auth_cache" + "github.com/t2bot/matrix-media-repo/api/auth_cache" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/globals" "github.com/t2bot/matrix-media-repo/common/runtime" @@ -136,7 +136,7 @@ func reloadAccessTokensOnChan(reloadChan chan bool) { for { shouldReload := <-reloadChan if shouldReload { - _auth_cache.FlushCache() + auth_cache.FlushCache() } else { return // received stop } diff --git a/common/assets/process.go b/common/assets/process.go index 1ab0851c..10e294c6 100644 --- a/common/assets/process.go +++ b/common/assets/process.go @@ -24,7 +24,7 @@ func SetupMigrations(givenMigrationsPath string) { if !exists { tempMigrations, err = os.MkdirTemp(os.TempDir(), "media-repo-migrations") if err != nil { - panic(err) + logrus.Fatalf("Failed to create temporary migrations directory: %v", err) } logrus.Info("Migrations path doesn't exist - attempting to unpack from compiled data") extractPrefixTo("migrations", tempMigrations) diff --git a/database/db.go b/database/db.go index 5931a481..8b6f4d98 100644 --- a/database/db.go +++ b/database/db.go @@ -2,7 +2,7 @@ package database import ( "database/sql" - "errors" + "fmt" "sync" "github.com/DavidHuie/gomigrate" @@ -70,7 +70,7 @@ func openDatabase(connectionString string, maxConns int, maxIdleConns int) error var err error if d.conn, err = sql.Open("postgres", connectionString); err != nil { - return errors.New("error connecting to db: " + err.Error()) + return fmt.Errorf("error connecting to db: %w", err) } d.conn.SetMaxOpenConns(maxConns) d.conn.SetMaxIdleConns(maxIdleConns) @@ -78,51 +78,51 @@ func openDatabase(connectionString string, maxConns int, maxIdleConns int) error // Run migrations var migrator *gomigrate.Migrator if migrator, err = gomigrate.NewMigratorWithLogger(d.conn, gomigrate.Postgres{}, config.Runtime.MigrationsPath, &logging.SendToDebugLogger{}); err != nil { - return errors.New("error setting up migrator: " + err.Error()) + return fmt.Errorf("error setting up migrator: %w", err) } if err = migrator.Migrate(); err != nil { - return errors.New("error running migrations: " + err.Error()) + return fmt.Errorf("error running migrations: %w", err) } // Prepare the table accessors if d.Media, err = prepareMediaTables(d.conn); err != nil { - return errors.New("failed to create media table accessor: " + err.Error()) + return fmt.Errorf("failed to create media table accessor: %w", err) } if d.ExpiringMedia, err = prepareExpiringMediaTables(d.conn); err != nil { - return errors.New("failed to create expiring media table accessor: " + err.Error()) + return fmt.Errorf("failed to create expiring media table accessor: %w", err) } if d.UserStats, err = prepareUserStatsTables(d.conn); err != nil { - return errors.New("failed to create user stats table accessor: " + err.Error()) + return fmt.Errorf("failed to create user stats table accessor: %w", err) } if d.ReservedMedia, err = prepareReservedMediaTables(d.conn); err != nil { - return errors.New("failed to create reserved media table accessor: " + err.Error()) + return fmt.Errorf("failed to create reserved media table accessor: %w", err) } if d.MetadataView, err = prepareMetadataVirtualTables(d.conn); err != nil { - return errors.New("failed to create metadata virtual table accessor: " + err.Error()) + return fmt.Errorf("failed to create metadata virtual table accessor: %w", err) } if d.HeldMedia, err = prepareHeldMediaTables(d.conn); err != nil { - return errors.New("failed to create held media table accessor: " + err.Error()) + return fmt.Errorf("failed to create held media table accessor: %w", err) } if d.Thumbnails, err = prepareThumbnailsTables(d.conn); err != nil { - return errors.New("failed to create thumbnails table accessor: " + err.Error()) + return fmt.Errorf("failed to create thumbnails table accessor: %w", err) } if d.LastAccess, err = prepareLastAccessTables(d.conn); err != nil { - return errors.New("failed to create last access table accessor: " + err.Error()) + return fmt.Errorf("failed to create last access table accessor: %w", err) } if d.UrlPreviews, err = prepareUrlPreviewsTables(d.conn); err != nil { - return errors.New("failed to create url previews table accessor: " + err.Error()) + return fmt.Errorf("failed to create url previews table accessor: %w", err) } if d.MediaAttributes, err = prepareMediaAttributesTables(d.conn); err != nil { - return errors.New("failed to create media attributes table accessor: " + err.Error()) + return fmt.Errorf("failed to create media attributes table accessor: %w", err) } if d.Tasks, err = prepareTasksTables(d.conn); err != nil { - return errors.New("failed to create tasks table accessor: " + err.Error()) + return fmt.Errorf("failed to create tasks table accessor: %w", err) } if d.Exports, err = prepareExportsTables(d.conn); err != nil { - return errors.New("failed to create exports table accessor: " + err.Error()) + return fmt.Errorf("failed to create exports table accessor: %w", err) } if d.ExportParts, err = prepareExportPartsTables(d.conn); err != nil { - return errors.New("failed to create export parts table accessor: " + err.Error()) + return fmt.Errorf("failed to create export parts table accessor: %w", err) } instance = d diff --git a/database/table_expiring_media.go b/database/table_expiring_media.go index c33579af..4fd218cd 100644 --- a/database/table_expiring_media.go +++ b/database/table_expiring_media.go @@ -3,9 +3,10 @@ package database import ( "database/sql" "errors" + "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbExpiringMedia struct { @@ -16,7 +17,8 @@ type DbExpiringMedia struct { } func (r *DbExpiringMedia) IsExpired() bool { - return r.ExpiresTs < util.NowMillis() + expiresTs := time.UnixMilli(r.ExpiresTs) + return expiresTs.Before(time.Now()) } const insertExpiringMedia = "INSERT INTO expiring_media (origin, media_id, user_id, expires_ts) VALUES ($1, $2, $3, $4);" @@ -40,19 +42,19 @@ type expiringMediaTableWithContext struct { func prepareExpiringMediaTables(db *sql.DB) (*expiringMediaTableStatements, error) { var err error - var stmts = &expiringMediaTableStatements{} + stmts := &expiringMediaTableStatements{} if stmts.insertExpiringMedia, err = db.Prepare(insertExpiringMedia); err != nil { - return nil, errors.New("error preparing insertExpiringMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExpiringMedia: %w", err) } if stmts.selectExpiringMediaByUserCount, err = db.Prepare(selectExpiringMediaByUserCount); err != nil { - return nil, errors.New("error preparing selectExpiringMediaByUserCount: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExpiringMediaByUserCount: %w", err) } if stmts.selectExpiringMediaById, err = db.Prepare(selectExpiringMediaById); err != nil { - return nil, errors.New("error preparing selectExpiringMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExpiringMediaById: %w", err) } if stmts.deleteExpiringMediaById, err = db.Prepare(deleteExpiringMediaById); err != nil { - return nil, errors.New("error preparing deleteExpiringMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExpiringMediaById: %w", err) } return stmts, nil @@ -71,7 +73,7 @@ func (s *expiringMediaTableWithContext) Insert(origin string, mediaId string, us } func (s *expiringMediaTableWithContext) ByUserCount(userId string) (int64, error) { - row := s.statements.selectExpiringMediaByUserCount.QueryRowContext(s.ctx, userId, util.NowMillis()) + row := s.statements.selectExpiringMediaByUserCount.QueryRowContext(s.ctx, userId, time.Now().UnixMilli()) val := int64(0) err := row.Scan(&val) if errors.Is(err, sql.ErrNoRows) { diff --git a/database/table_export_parts.go b/database/table_export_parts.go index 71d882ce..05e36971 100644 --- a/database/table_export_parts.go +++ b/database/table_export_parts.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -35,19 +36,19 @@ type exportPartsTableWithContext struct { func prepareExportPartsTables(db *sql.DB) (*exportPartsTableStatements, error) { var err error - var stmts = &exportPartsTableStatements{} + stmts := &exportPartsTableStatements{} if stmts.insertExportPart, err = db.Prepare(insertExportPart); err != nil { - return nil, errors.New("error preparing insertExportPart: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExportPart: %w", err) } if stmts.deleteExportPartsById, err = db.Prepare(deleteExportPartsById); err != nil { - return nil, errors.New("error preparing deleteExportPartsById: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExportPartsById: %w", err) } if stmts.selectExportPartsById, err = db.Prepare(selectExportPartsById); err != nil { - return nil, errors.New("error preparing selectExportPartsById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportPartsById: %w", err) } if stmts.selectExportPartById, err = db.Prepare(selectExportPartById); err != nil { - return nil, errors.New("error preparing selectExportPartById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportPartById: %w", err) } return stmts, nil diff --git a/database/table_exports.go b/database/table_exports.go index 6495475a..25f06054 100644 --- a/database/table_exports.go +++ b/database/table_exports.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -24,16 +25,16 @@ type exportsTableWithContext struct { func prepareExportsTables(db *sql.DB) (*exportsTableStatements, error) { var err error - var stmts = &exportsTableStatements{} + stmts := &exportsTableStatements{} if stmts.insertExport, err = db.Prepare(insertExport); err != nil { - return nil, errors.New("error preparing insertExport: " + err.Error()) + return nil, fmt.Errorf("error preparing insertExport: %w", err) } if stmts.selectExportEntity, err = db.Prepare(selectExportEntity); err != nil { - return nil, errors.New("error preparing selectExportEntity: " + err.Error()) + return nil, fmt.Errorf("error preparing selectExportEntity: %w", err) } if stmts.deleteExport, err = db.Prepare(deleteExport); err != nil { - return nil, errors.New("error preparing deleteExport: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteExport: %w", err) } return stmts, nil diff --git a/database/table_last_access.go b/database/table_last_access.go index eca9826a..e38cc975 100644 --- a/database/table_last_access.go +++ b/database/table_last_access.go @@ -2,7 +2,7 @@ package database import ( "database/sql" - "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -25,10 +25,10 @@ type lastAccessTableWithContext struct { func prepareLastAccessTables(db *sql.DB) (*lastAccessTableStatements, error) { var err error - var stmts = &lastAccessTableStatements{} + stmts := &lastAccessTableStatements{} if stmts.upsertLastAccess, err = db.Prepare(upsertLastAccess); err != nil { - return nil, errors.New("error preparing upsertLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing upsertLastAccess: %w", err) } return stmts, nil diff --git a/database/table_media.go b/database/table_media.go index e54f9368..f0dfd620 100644 --- a/database/table_media.go +++ b/database/table_media.go @@ -3,6 +3,8 @@ package database import ( "database/sql" "errors" + "fmt" + "time" "github.com/lib/pq" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -80,67 +82,67 @@ type MediaTableWithContext struct { func prepareMediaTables(db *sql.DB) (*mediaTableStatements, error) { var err error - var stmts = &mediaTableStatements{} + stmts := &mediaTableStatements{} if stmts.selectDistinctMediaDatastoreIds, err = db.Prepare(selectDistinctMediaDatastoreIds); err != nil { - return nil, errors.New("error preparing selectDistinctMediaDatastoreIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectDistinctMediaDatastoreIds: %w", err) } if stmts.selectMediaIsQuarantinedByHash, err = db.Prepare(selectMediaIsQuarantinedByHash); err != nil { - return nil, errors.New("error preparing selectMediaIsQuarantinedByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaIsQuarantinedByHash: %w", err) } if stmts.selectMediaByHash, err = db.Prepare(selectMediaByHash); err != nil { - return nil, errors.New("error preparing selectMediaByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByHash: %w", err) } if stmts.insertMedia, err = db.Prepare(insertMedia); err != nil { - return nil, errors.New("error preparing insertMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertMedia: %w", err) } if stmts.selectMediaExists, err = db.Prepare(selectMediaExists); err != nil { - return nil, errors.New("error preparing selectMediaExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaExists: %w", err) } if stmts.selectMediaById, err = db.Prepare(selectMediaById); err != nil { - return nil, errors.New("error preparing selectMediaById: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaById: %w", err) } if stmts.selectMediaByUserId, err = db.Prepare(selectMediaByUserId); err != nil { - return nil, errors.New("error preparing selectMediaByUserId: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByUserId: %w", err) } if stmts.selectOldMediaByUserId, err = db.Prepare(selectOldMediaByUserId); err != nil { - return nil, errors.New("error preparing selectOldMediaByUserId: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaByUserId: %w", err) } if stmts.selectMediaByOrigin, err = db.Prepare(selectMediaByOrigin); err != nil { - return nil, errors.New("error preparing selectMediaByOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOrigin: %w", err) } if stmts.selectOldMediaByOrigin, err = db.Prepare(selectOldMediaByOrigin); err != nil { - return nil, errors.New("error preparing selectOldMediaByOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaByOrigin: %w", err) } if stmts.selectMediaByLocationExists, err = db.Prepare(selectMediaByLocationExists); err != nil { - return nil, errors.New("error preparing selectMediaByLocationExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByLocationExists: %w", err) } if stmts.selectMediaByUserCount, err = db.Prepare(selectMediaByUserCount); err != nil { - return nil, errors.New("error preparing selectMediaByUserCount: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByUserCount: %w", err) } if stmts.selectMediaByOriginAndUserIds, err = db.Prepare(selectMediaByOriginAndUserIds); err != nil { - return nil, errors.New("error preparing selectMediaByOriginAndUserIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOriginAndUserIds: %w", err) } if stmts.selectMediaByOriginAndIds, err = db.Prepare(selectMediaByOriginAndIds); err != nil { - return nil, errors.New("error preparing selectMediaByOriginAndIds: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByOriginAndIds: %w", err) } if stmts.selectOldMediaExcludingDomains, err = db.Prepare(selectOldMediaExcludingDomains); err != nil { - return nil, errors.New("error preparing selectOldMediaExcludingDomains: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldMediaExcludingDomains: %w", err) } if stmts.deleteMedia, err = db.Prepare(deleteMedia); err != nil { - return nil, errors.New("error preparing deleteMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteMedia: %w", err) } if stmts.updateMediaLocation, err = db.Prepare(updateMediaLocation); err != nil { - return nil, errors.New("error preparing updateMediaLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing updateMediaLocation: %w", err) } if stmts.selectMediaByLocation, err = db.Prepare(selectMediaByLocation); err != nil { - return nil, errors.New("error preparing selectMediaByLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByLocation: %w", err) } if stmts.selectMediaByQuarantine, err = db.Prepare(selectMediaByQuarantine); err != nil { - return nil, errors.New("error preparing selectMediaByQuarantine: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByQuarantine: %w", err) } if stmts.selectMediaByQuarantineAndOrigin, err = db.Prepare(selectMediaByQuarantineAndOrigin); err != nil { - return nil, errors.New("error preparing selectMediaByQuarantineAndOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaByQuarantineAndOrigin: %w", err) } return stmts, nil @@ -233,8 +235,8 @@ func (s *MediaTableWithContext) GetByIds(origin string, mediaIds []string) ([]*D return s.scanRows(s.statements.selectMediaByOriginAndIds.QueryContext(s.ctx, origin, pq.Array(mediaIds))) } -func (s *MediaTableWithContext) GetOldExcluding(origins []string, beforeTs int64) ([]*DbMedia, error) { - return s.scanRows(s.statements.selectOldMediaExcludingDomains.QueryContext(s.ctx, pq.Array(origins), beforeTs)) +func (s *MediaTableWithContext) GetOldExcluding(origins []string, beforeTs time.Time) ([]*DbMedia, error) { + return s.scanRows(s.statements.selectOldMediaExcludingDomains.QueryContext(s.ctx, pq.Array(origins), beforeTs.UnixMilli())) } func (s *MediaTableWithContext) GetByLocation(datastoreId string, location string) ([]*DbMedia, error) { diff --git a/database/table_media_attributes.go b/database/table_media_attributes.go index 7532ce73..d5838a0c 100644 --- a/database/table_media_attributes.go +++ b/database/table_media_attributes.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -39,13 +40,13 @@ type mediaAttributesTableWithContext struct { func prepareMediaAttributesTables(db *sql.DB) (*mediaAttributesTableStatements, error) { var err error - var stmts = &mediaAttributesTableStatements{} + stmts := &mediaAttributesTableStatements{} if stmts.selectMediaAttributes, err = db.Prepare(selectMediaAttributes); err != nil { - return nil, errors.New("error preparing selectMediaAttributes: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaAttributes: %w", err) } if stmts.upsertMediaPurpose, err = db.Prepare(upsertMediaPurpose); err != nil { - return nil, errors.New("error preparing upsertMediaPurpose: " + err.Error()) + return nil, fmt.Errorf("error preparing upsertMediaPurpose: %w", err) } return stmts, nil diff --git a/database/table_media_hold.go b/database/table_media_hold.go index 05bb3a5c..df094476 100644 --- a/database/table_media_hold.go +++ b/database/table_media_hold.go @@ -2,10 +2,10 @@ package database import ( "database/sql" - "errors" + "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbHeldMedia struct { @@ -35,13 +35,13 @@ type heldMediaTableWithContext struct { func prepareHeldMediaTables(db *sql.DB) (*heldMediaTableStatements, error) { var err error - var stmts = &heldMediaTableStatements{} + stmts := &heldMediaTableStatements{} if stmts.insertHeldMedia, err = db.Prepare(insertHeldMedia); err != nil { - return nil, errors.New("error preparing insertHeldMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing insertHeldMedia: %w", err) } if stmts.deleteHeldMedia, err = db.Prepare(deleteHeldMedia); err != nil { - return nil, errors.New("error preparing deleteHeldMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteHeldMedia: %w", err) } return stmts, nil @@ -55,11 +55,11 @@ func (s *heldMediaTableStatements) Prepare(ctx rcontext.RequestContext) *heldMed } func (s *heldMediaTableWithContext) TryInsert(origin string, mediaId string, reason HeldReason) error { - _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, util.NowMillis()) + _, err := s.statements.insertHeldMedia.ExecContext(s.ctx, origin, mediaId, reason, time.Now().UnixMilli()) return err } -func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThanTs int64) error { - _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThanTs) +func (s *heldMediaTableWithContext) DeleteOlderThan(reason HeldReason, olderThan time.Time) error { + _, err := s.statements.deleteHeldMedia.ExecContext(s.ctx, reason, olderThan.UnixMilli()) return err } diff --git a/database/table_reserved_media.go b/database/table_reserved_media.go index 776d0cf2..4c351cd4 100644 --- a/database/table_reserved_media.go +++ b/database/table_reserved_media.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -28,13 +29,13 @@ type reservedMediaTableWithContext struct { func prepareReservedMediaTables(db *sql.DB) (*reservedMediaTableStatements, error) { var err error - var stmts = &reservedMediaTableStatements{} + stmts := &reservedMediaTableStatements{} if stmts.insertReservedMediaNoConflict, err = db.Prepare(insertReservedMediaNoConflict); err != nil { - return nil, errors.New("error preparing insertReservedMediaNoConflict: " + err.Error()) + return nil, fmt.Errorf("error preparing insertReservedMediaNoConflict: %w", err) } if stmts.selectReservedMediaExists, err = db.Prepare(selectReservedMediaExists); err != nil { - return nil, errors.New("error preparing selectReservedMediaExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectReservedMediaExists: %w", err) } return stmts, nil diff --git a/database/table_tasks.go b/database/table_tasks.go index 8b2b5f67..742b5472 100644 --- a/database/table_tasks.go +++ b/database/table_tasks.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -39,25 +40,25 @@ type tasksTableWithContext struct { func prepareTasksTables(db *sql.DB) (*tasksTableStatements, error) { var err error - var stmts = &tasksTableStatements{} + stmts := &tasksTableStatements{} if stmts.selectTask, err = db.Prepare(selectTask); err != nil { - return nil, errors.New("error preparing selectTask: " + err.Error()) + return nil, fmt.Errorf("error preparing selectTask: %w", err) } if stmts.insertTask, err = db.Prepare(insertTask); err != nil { - return nil, errors.New("error preparing insertTask: " + err.Error()) + return nil, fmt.Errorf("error preparing insertTask: %w", err) } if stmts.selectAllTasks, err = db.Prepare(selectAllTasks); err != nil { - return nil, errors.New("error preparing selectAllTasks: " + err.Error()) + return nil, fmt.Errorf("error preparing selectAllTasks: %w", err) } if stmts.selectIncompleteTasks, err = db.Prepare(selectIncompleteTasks); err != nil { - return nil, errors.New("error preparing selectIncompleteTasks: " + err.Error()) + return nil, fmt.Errorf("error preparing selectIncompleteTasks: %w", err) } if stmts.updateTaskEndTime, err = db.Prepare(updateTaskEndTime); err != nil { - return nil, errors.New("error preparing updateTaskEndTime: " + err.Error()) + return nil, fmt.Errorf("error preparing updateTaskEndTime: %w", err) } if stmts.updateTaskError, err = db.Prepare(updateTaskError); err != nil { - return nil, errors.New("error preparing updateTaskError: " + err.Error()) + return nil, fmt.Errorf("error preparing updateTaskError: %w", err) } return stmts, nil diff --git a/database/table_thumbnails.go b/database/table_thumbnails.go index 5358103b..9033e735 100644 --- a/database/table_thumbnails.go +++ b/database/table_thumbnails.go @@ -3,6 +3,8 @@ package database import ( "database/sql" "errors" + "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -52,31 +54,31 @@ type thumbnailsTableWithContext struct { func prepareThumbnailsTables(db *sql.DB) (*thumbnailsTableStatements, error) { var err error - var stmts = &thumbnailsTableStatements{} + stmts := &thumbnailsTableStatements{} if stmts.selectThumbnailByParams, err = db.Prepare(selectThumbnailByParams); err != nil { - return nil, errors.New("error preparing selectThumbnailByParams: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailByParams: %w", err) } if stmts.insertThumbnail, err = db.Prepare(insertThumbnail); err != nil { - return nil, errors.New("error preparing insertThumbnail: " + err.Error()) + return nil, fmt.Errorf("error preparing insertThumbnail: %w", err) } if stmts.selectThumbnailByLocationExists, err = db.Prepare(selectThumbnailByLocationExists); err != nil { - return nil, errors.New("error preparing selectThumbnailByLocationExists: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailByLocationExists: %w", err) } if stmts.selectThumbnailsForMedia, err = db.Prepare(selectThumbnailsForMedia); err != nil { - return nil, errors.New("error preparing selectThumbnailsForMedia: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsForMedia: %w", err) } if stmts.selectOldThumbnails, err = db.Prepare(selectOldThumbnails); err != nil { - return nil, errors.New("error preparing selectOldThumbnails: " + err.Error()) + return nil, fmt.Errorf("error preparing selectOldThumbnails: %w", err) } if stmts.deleteThumbnail, err = db.Prepare(deleteThumbnail); err != nil { - return nil, errors.New("error preparing deleteThumbnail: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteThumbnail: %w", err) } if stmts.updateThumbnailLocation, err = db.Prepare(updateThumbnailLocation); err != nil { - return nil, errors.New("error preparing updateThumbnailLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing updateThumbnailLocation: %w", err) } if stmts.selectThumbnailsByLocation, err = db.Prepare(selectThumbnailsByLocation); err != nil { - return nil, errors.New("error preparing selectThumbnailsByLocation: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsByLocation: %w", err) } return stmts, nil @@ -123,8 +125,8 @@ func (s *thumbnailsTableWithContext) GetForMedia(origin string, mediaId string) return s.scanRows(s.statements.selectThumbnailsForMedia.QueryContext(s.ctx, origin, mediaId)) } -func (s *thumbnailsTableWithContext) GetOlderThan(ts int64) ([]*DbThumbnail, error) { - return s.scanRows(s.statements.selectOldThumbnails.QueryContext(s.ctx, ts)) +func (s *thumbnailsTableWithContext) GetOlderThan(ts time.Time) ([]*DbThumbnail, error) { + return s.scanRows(s.statements.selectOldThumbnails.QueryContext(s.ctx, ts.UnixMilli())) } func (s *thumbnailsTableWithContext) GetByLocation(datastoreId string, location string) ([]*DbThumbnail, error) { diff --git a/database/table_url_previews.go b/database/table_url_previews.go index 55c6bde5..44c651e1 100644 --- a/database/table_url_previews.go +++ b/database/table_url_previews.go @@ -3,9 +3,10 @@ package database import ( "database/sql" "errors" + "fmt" + "time" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/util" ) type DbUrlPreview struct { @@ -42,16 +43,16 @@ type urlPreviewsTableWithContext struct { func prepareUrlPreviewsTables(db *sql.DB) (*urlPreviewsTableStatements, error) { var err error - var stmts = &urlPreviewsTableStatements{} + stmts := &urlPreviewsTableStatements{} if stmts.selectUrlPreview, err = db.Prepare(selectUrlPreview); err != nil { - return nil, errors.New("error preparing selectUrlPreview: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUrlPreview: %w", err) } if stmts.insertUrlPreview, err = db.Prepare(insertUrlPreview); err != nil { - return nil, errors.New("error preparing insertUrlPreview: " + err.Error()) + return nil, fmt.Errorf("error preparing insertUrlPreview: %w", err) } if stmts.deleteOldUrlPreviews, err = db.Prepare(deleteOldUrlPreviews); err != nil { - return nil, errors.New("error preparing deleteOldUrlPreviews: " + err.Error()) + return nil, fmt.Errorf("error preparing deleteOldUrlPreviews: %w", err) } return stmts, nil @@ -64,8 +65,8 @@ func (s *urlPreviewsTableStatements) Prepare(ctx rcontext.RequestContext) *urlPr } } -func (s *urlPreviewsTableWithContext) Get(url string, ts int64, languageHeader string) (*DbUrlPreview, error) { - row := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, ts, languageHeader) +func (s *urlPreviewsTableWithContext) Get(url string, timestamp int64, languageHeader string) (*DbUrlPreview, error) { + row := s.statements.selectUrlPreview.QueryRowContext(s.ctx, url, timestamp, languageHeader) val := &DbUrlPreview{} err := row.Scan(&val.Url, &val.ErrorCode, &val.BucketTs, &val.SiteUrl, &val.SiteName, &val.ResourceType, &val.Description, &val.Title, &val.ImageMxc, &val.ImageType, &val.ImageSize, &val.ImageWidth, &val.ImageHeight, &val.LanguageHeader) if errors.Is(err, sql.ErrNoRows) { @@ -74,8 +75,8 @@ func (s *urlPreviewsTableWithContext) Get(url string, ts int64, languageHeader s return val, err } -func (s *urlPreviewsTableWithContext) Insert(p *DbUrlPreview) error { - _, err := s.statements.insertUrlPreview.ExecContext(s.ctx, p.Url, p.ErrorCode, p.BucketTs, p.SiteUrl, p.SiteName, p.ResourceType, p.Description, p.Title, p.ImageMxc, p.ImageType, p.ImageSize, p.ImageWidth, p.ImageHeight, p.LanguageHeader) +func (s *urlPreviewsTableWithContext) Insert(preview *DbUrlPreview) error { + _, err := s.statements.insertUrlPreview.ExecContext(s.ctx, preview.Url, preview.ErrorCode, preview.BucketTs, preview.SiteUrl, preview.SiteName, preview.ResourceType, preview.Description, preview.Title, preview.ImageMxc, preview.ImageType, preview.ImageSize, preview.ImageWidth, preview.ImageHeight, preview.LanguageHeader) return err } @@ -83,12 +84,12 @@ func (s *urlPreviewsTableWithContext) InsertError(url string, errorCode string) _ = s.Insert(&DbUrlPreview{ Url: url, ErrorCode: errorCode, - BucketTs: util.GetHourBucket(util.NowMillis()), + BucketTs: time.Now().UnixMilli(), // remainder of fields don't matter }) } -func (s *urlPreviewsTableWithContext) DeleteOlderThan(ts int64) error { - _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, ts) +func (s *urlPreviewsTableWithContext) DeleteOlderThan(ts time.Time) error { + _, err := s.statements.deleteOldUrlPreviews.ExecContext(s.ctx, ts.UnixMilli()) return err } diff --git a/database/table_user_stats.go b/database/table_user_stats.go index ad36acd7..be1084e8 100644 --- a/database/table_user_stats.go +++ b/database/table_user_stats.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "errors" + "fmt" "github.com/t2bot/matrix-media-repo/common/rcontext" ) @@ -25,10 +26,10 @@ type userStatsTableWithContext struct { func prepareUserStatsTables(db *sql.DB) (*userStatsTableStatements, error) { var err error - var stmts = &userStatsTableStatements{} + stmts := &userStatsTableStatements{} if stmts.selectUserStatsUploadedBytes, err = db.Prepare(selectUserStatsUploadedBytes); err != nil { - return nil, errors.New("error preparing selectUserStatsUploadedBytes: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUserStatsUploadedBytes: %w", err) } return stmts, nil diff --git a/database/virtualtable_metadata.go b/database/virtualtable_metadata.go index f99d256f..71b4846a 100644 --- a/database/virtualtable_metadata.go +++ b/database/virtualtable_metadata.go @@ -64,30 +64,30 @@ type metadataVirtualTableWithContext struct { func prepareMetadataVirtualTables(db *sql.DB) (*metadataVirtualTableStatements, error) { var err error - var stmts = &metadataVirtualTableStatements{ + stmts := &metadataVirtualTableStatements{ db: db, } if stmts.selectEstimatedDatastoreSize, err = db.Prepare(selectEstimatedDatastoreSize); err != nil { - return nil, errors.New("error preparing selectEstimatedDatastoreSize: " + err.Error()) + return nil, fmt.Errorf("error preparing selectEstimatedDatastoreSize: %w", err) } if stmts.selectUploadSizesForServer, err = db.Prepare(selectUploadSizesForServer); err != nil { - return nil, errors.New("error preparing selectUploadSizesForServer: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUploadSizesForServer: %w", err) } if stmts.selectUploadCountsForServer, err = db.Prepare(selectUploadCountsForServer); err != nil { - return nil, errors.New("error preparing selectUploadCountsForServer: " + err.Error()) + return nil, fmt.Errorf("error preparing selectUploadCountsForServer: %w", err) } if stmts.selectMediaForDatastoreWithLastAccess, err = db.Prepare(selectMediaForDatastoreWithLastAccess); err != nil { - return nil, errors.New("error preparing selectMediaForDatastoreWithLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing selectMediaForDatastoreWithLastAccess: %w", err) } if stmts.selectThumbnailsForDatastoreWithLastAccess, err = db.Prepare(selectThumbnailsForDatastoreWithLastAccess); err != nil { - return nil, errors.New("error preparing selectThumbnailsForDatastoreWithLastAccess: " + err.Error()) + return nil, fmt.Errorf("error preparing selectThumbnailsForDatastoreWithLastAccess: %w", err) } if stmts.updateQuarantineByHash, err = db.Prepare(updateQuarantineByHash); err != nil { - return nil, errors.New("error preparing updateQuarantineByHash: " + err.Error()) + return nil, fmt.Errorf("error preparing updateQuarantineByHash: %w", err) } if stmts.updateQuarantineByHashAndOrigin, err = db.Prepare(updateQuarantineByHashAndOrigin); err != nil { - return nil, errors.New("error preparing updateQuarantineByHashAndOrigin: " + err.Error()) + return nil, fmt.Errorf("error preparing updateQuarantineByHashAndOrigin: %w", err) } return stmts, nil diff --git a/datastores/buffer.go b/datastores/buffer.go index 4a0d71dd..37b629cf 100644 --- a/datastores/buffer.go +++ b/datastores/buffer.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "os" @@ -21,7 +22,7 @@ func BufferTemp(datastore config.DatastoreConfig, contents io.ReadCloser) (strin } else if datastore.Type == "file" { fpath, err = os.MkdirTemp(os.TempDir(), "mmr") if err != nil { - return "", 0, nil, errors.New("error generating temporary directory: " + err.Error()) + return "", 0, nil, fmt.Errorf("error generating temporary directory: %w", err) } } else { return "", 0, nil, errors.New("unknown datastore type - contact developer") @@ -32,14 +33,14 @@ func BufferTemp(datastore config.DatastoreConfig, contents io.ReadCloser) (strin logrus.Warnf("Datastore %s does not have a valid temporary path configured. This will lead to increased memory usage.", datastore.Id) target = &bytes.Buffer{} } else { - err = os.Mkdir(fpath, os.ModeDir|0700) + err = os.Mkdir(fpath, os.ModeDir|0o700) if err != nil && !os.IsExist(err) { - return "", 0, nil, errors.New("error creating temp path: " + err.Error()) + return "", 0, nil, fmt.Errorf("error creating temp path: %w", err) } var file *os.File file, err = os.CreateTemp(fpath, "mmr") if err != nil { - return "", 0, nil, errors.New("error generating temporary file: " + err.Error()) + return "", 0, nil, fmt.Errorf("error generating temporary file: %w", err) } target = file } diff --git a/matrix/server_discovery.go b/matrix/server_discovery.go index 02e5ce37..2d85869f 100644 --- a/matrix/server_discovery.go +++ b/matrix/server_discovery.go @@ -2,6 +2,7 @@ package matrix import ( "encoding/json" + "errors" "fmt" "net" "net/http" @@ -15,8 +16,10 @@ import ( "github.com/sirupsen/logrus" ) -var apiUrlCacheInstance *cache.Cache -var apiUrlSingletonLock = &sync.Once{} +var ( + apiUrlCacheInstance *cache.Cache + apiUrlSingletonLock = &sync.Once{} +) type cachedServer struct { url string @@ -45,20 +48,23 @@ func GetServerApiUrl(hostname string) (string, string, error) { return server.url, server.hostname, nil } - h, p, err := net.SplitHostPort(hostname) + addrErr := &net.AddrError{} + host, port, err := net.SplitHostPort(hostname) defPort := false - if err != nil && strings.HasSuffix(err.Error(), "missing port in address") { - h, p, err = net.SplitHostPort(hostname + ":8448") - defPort = true - } - if err != nil { - return "", "", err + switch { + case errors.As(err, &addrErr) && addrErr.Err == "missing port in address": + host, port, err = net.SplitHostPort(hostname + ":8448") + if err != nil { + return "", "", fmt.Errorf("failed to parse hostname: %w", err) + } + case err != nil: + return "", "", fmt.Errorf("failed to parse hostname: %w", err) } // Step 1 of the discovery process: if the hostname is an IP, use that with explicit or default port - logrus.Debug("Testing if " + h + " is an IP address") - if is.IP(h) { - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) + logrus.Debug("Testing if " + host + " is an IP address") + if is.IP(host) { + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) server := cachedServer{url, hostname} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) logrus.Debug("Server API URL for " + hostname + " is " + url + " (IP address)") @@ -68,17 +74,17 @@ func GetServerApiUrl(hostname string) (string, string, error) { // Step 2: if the hostname is not an IP address, and an explicit port is given, use that logrus.Debug("Testing if a default port was used. Using default = ", defPort) if !defPort { - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) - server := cachedServer{url, h} + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (explicit port)") - return url, h, nil + logrus.Debugf("Server API URL for %s is %s (explicit port)", hostname, url) + return url, host, nil } // Step 3: if the hostname is not an IP address and no explicit port is given, do .well-known // Note that we have sprawling branches here because we need to fall through to step 4 if parsing fails - logrus.Debug("Doing .well-known lookup on " + h) - r, err := http.Get(fmt.Sprintf("https://%s/.well-known/matrix/server", h)) + logrus.Debug("Doing .well-known lookup on " + host) + r, err := http.Get(fmt.Sprintf("https://%s/.well-known/matrix/server", host)) if r != nil { defer r.Body.Close() } @@ -101,7 +107,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(wkHost, wkPort)) server := cachedServer{url, wk.ServerAddr} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; IP address)") + logrus.Debugf("Server API URL for %s is %s (WK; IP address)", hostname, url) return url, wk.ServerAddr, nil } @@ -112,7 +118,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", wkHost) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; explicit port)") + logrus.Debugf("Server API URL for %s is %s (WK; explicit port)", hostname, url) return url, wkHost, nil } @@ -129,7 +135,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; SRV)") + logrus.Debugf("Server API URL for %s is %s (WK; SRV)", hostname, url) return url, wkHost, nil } @@ -147,7 +153,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; SRV-Deprecated)") + logrus.Debugf("Server API URL for %s is %s (WK; SRV-Deprecated)", hostname, url) return url, wkHost, nil } @@ -156,7 +162,7 @@ func GetServerApiUrl(hostname string) (string, string, error) { url := fmt.Sprintf("https://%s", net.JoinHostPort(wkHost, wkPort)) server := cachedServer{url, wkHost} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (WK; fallback)") + logrus.Debugf("Server API URL for %s is %s (WK; fallback)", hostname, url) return url, wkHost, nil } } @@ -177,10 +183,10 @@ func GetServerApiUrl(hostname string) (string, string, error) { realAddr = realAddr[0 : len(realAddr)-1] } url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) - server := cachedServer{url, h} + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (SRV)") - return url, h, nil + logrus.Debugf("Server API URL for %s is %s (SRV)", hostname, url) + return url, host, nil } // Step 5: try resolving a hostname using DEPRECATED SRV records and use it @@ -194,17 +200,17 @@ func GetServerApiUrl(hostname string) (string, string, error) { realAddr = realAddr[0 : len(realAddr)-1] } url := fmt.Sprintf("https://%s", net.JoinHostPort(realAddr, strconv.Itoa(int(addrs[0].Port)))) - server := cachedServer{url, h} + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (SRV-Deprecated)") - return url, h, nil + logrus.Debugf("Server API URL for %s is %s (SRV-Deprecated)", hostname, url) + return url, host, nil } // Step 6: use the target host as-is logrus.Debug("Using host as-is: ", hostname) - url := fmt.Sprintf("https://%s", net.JoinHostPort(h, p)) - server := cachedServer{url, h} + url := fmt.Sprintf("https://%s", net.JoinHostPort(host, port)) + server := cachedServer{url, host} apiUrlCacheInstance.Set(hostname, server, cache.DefaultExpiration) - logrus.Debug("Server API URL for " + hostname + " is " + url + " (fallback)") - return url, h, nil + logrus.Debugf("Server API URL for %s is %s (fallback)", hostname, url) + return url, host, nil } diff --git a/metrics/metrics.go b/metrics/metrics.go index 4f597ee4..93467a47 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,7 +11,7 @@ var InvalidHttpRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "media_invalid_http_requests_total", }, []string{"action", "method"}) var HttpResponses = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "media_http_responses_total", + Name: "media_httpresponses_total", }, []string{"host", "action", "method", "statusCode"}) var HttpResponseTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "media_http_response_time_seconds", diff --git a/pipelines/pipeline_create/pipeline.go b/pipelines/pipeline_create/pipeline.go index ea7d14ac..daf12ae2 100644 --- a/pipelines/pipeline_create/pipeline.go +++ b/pipelines/pipeline_create/pipeline.go @@ -1,11 +1,12 @@ package pipeline_create import ( + "time" + "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" - "github.com/t2bot/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/upload" ) const DefaultExpirationTime = 0 @@ -24,10 +25,11 @@ func Execute(ctx rcontext.RequestContext, origin string, userId string, expirati // Step 3: Insert record of expiration if expirationTime == DefaultExpirationTime { - expirationTime = ctx.Config.Uploads.MaxAgeSeconds * 1000 + expirationTime = ctx.Config.Uploads.MaxAgeSeconds } - expiresTs := util.NowMillis() + expirationTime - if err = database.GetInstance().ExpiringMedia.Prepare(ctx).Insert(origin, mediaId, userId, expiresTs); err != nil { + expires := time.Now().Add(time.Duration(expirationTime) * time.Second) + expiresTS := expires.UnixMilli() + if err = database.GetInstance().ExpiringMedia.Prepare(ctx).Insert(origin, mediaId, userId, expiresTS); err != nil { return nil, err } @@ -36,6 +38,6 @@ func Execute(ctx rcontext.RequestContext, origin string, userId string, expirati Origin: origin, MediaId: mediaId, UserId: userId, - ExpiresTs: expiresTs, + ExpiresTs: expiresTS, }, nil } diff --git a/pipelines/pipeline_download/pipeline.go b/pipelines/pipeline_download/pipeline.go index a20eaf23..b8f5c396 100644 --- a/pipelines/pipeline_download/pipeline.go +++ b/pipelines/pipeline_download/pipeline.go @@ -8,13 +8,13 @@ import ( "time" "github.com/getsentry/sentry-go" - "github.com/t2bot/go-singleflight-streams" + sfstreams "github.com/t2bot/go-singleflight-streams" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/meta" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quarantine" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/meta" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quarantine" "github.com/t2bot/matrix-media-repo/util/readers" "github.com/t2bot/matrix-media-repo/util/sfcache" ) diff --git a/pipelines/pipeline_preview/pipeline.go b/pipelines/pipeline_preview/pipeline.go index 040fb3ca..027494de 100644 --- a/pipelines/pipeline_preview/pipeline.go +++ b/pipelines/pipeline_preview/pipeline.go @@ -4,13 +4,13 @@ import ( "errors" "fmt" "net/url" + "time" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/url_preview" + "github.com/t2bot/matrix-media-repo/pipelines/steps/url_preview" "github.com/t2bot/matrix-media-repo/url_previewing/m" - "github.com/t2bot/matrix-media-repo/util" "golang.org/x/sync/singleflight" ) @@ -31,12 +31,13 @@ func Execute(ctx rcontext.RequestContext, onHost string, previewUrl string, user // Step 2: Fix timestamp bucket. If we're within 60 seconds of a bucket, just assume we're okay, so we don't // infinitely recurse into ourselves. - now := util.NowMillis() - atBucket := util.GetHourBucket(opts.Timestamp) // we should only be using this for the remainder of the function - nowBucket := util.GetHourBucket(now) - if (now-opts.Timestamp) > 60000 && atBucket != nowBucket { + requestTS := time.UnixMilli(opts.Timestamp) + atBucket := requestTS.UnixMilli() + nowBucket := time.Now().UnixMilli() + + if time.Now().After(requestTS.Add(60*time.Second)) && atBucket != nowBucket { return Execute(ctx, onHost, previewUrl, userId, PreviewOpts{ - Timestamp: now, + Timestamp: time.Now().Unix() * 1000, LanguageHeader: opts.LanguageHeader, }) } diff --git a/pipelines/pipeline_thumbnail/pipeline.go b/pipelines/pipeline_thumbnail/pipeline.go index e3b19204..7ddc8110 100644 --- a/pipelines/pipeline_thumbnail/pipeline.go +++ b/pipelines/pipeline_thumbnail/pipeline.go @@ -11,10 +11,10 @@ import ( "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quarantine" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/thumbnails" "github.com/t2bot/matrix-media-repo/pipelines/pipeline_download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quarantine" + "github.com/t2bot/matrix-media-repo/pipelines/steps/thumbnails" "github.com/t2bot/matrix-media-repo/util/readers" "github.com/t2bot/matrix-media-repo/util/sfcache" ) diff --git a/pipelines/pipeline_upload/pipeline.go b/pipelines/pipeline_upload/pipeline.go index 369c899e..cbf9e4c5 100644 --- a/pipelines/pipeline_upload/pipeline.go +++ b/pipelines/pipeline_upload/pipeline.go @@ -3,6 +3,7 @@ package pipeline_upload import ( "errors" "io" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common" @@ -11,10 +12,9 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/notifier" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/meta" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/quota" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/upload" - "github.com/t2bot/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/pipelines/steps/meta" + "github.com/t2bot/matrix-media-repo/pipelines/steps/quota" + "github.com/t2bot/matrix-media-repo/pipelines/steps/upload" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -117,7 +117,7 @@ func Execute(ctx rcontext.RequestContext, origin string, mediaId string, r io.Re ContentType: contentType, UserId: userId, SizeBytes: sizeBytes, - CreationTs: util.NowMillis(), + CreationTs: time.Now().UnixMilli(), Quarantined: false, Locatable: &database.Locatable{ Sha256Hash: sha256hash, diff --git a/pipelines/_steps/datastore_op/put_and_return_stream.go b/pipelines/steps/datastore_op/put_and_return_stream.go similarity index 100% rename from pipelines/_steps/datastore_op/put_and_return_stream.go rename to pipelines/steps/datastore_op/put_and_return_stream.go diff --git a/pipelines/_steps/download/open_stream.go b/pipelines/steps/download/open_stream.go similarity index 100% rename from pipelines/_steps/download/open_stream.go rename to pipelines/steps/download/open_stream.go diff --git a/pipelines/_steps/download/try_download.go b/pipelines/steps/download/try_download.go similarity index 97% rename from pipelines/_steps/download/try_download.go rename to pipelines/steps/download/try_download.go index bda7aa62..ddef63f2 100644 --- a/pipelines/_steps/download/try_download.go +++ b/pipelines/steps/download/try_download.go @@ -17,7 +17,7 @@ import ( "github.com/t2bot/matrix-media-repo/errcache" "github.com/t2bot/matrix-media-repo/matrix" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/datastore_op" + "github.com/t2bot/matrix-media-repo/pipelines/steps/datastore_op" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/readers" diff --git a/pipelines/_steps/download/wait.go b/pipelines/steps/download/wait.go similarity index 100% rename from pipelines/_steps/download/wait.go rename to pipelines/steps/download/wait.go diff --git a/pipelines/_steps/meta/flag_access.go b/pipelines/steps/meta/flag_access.go similarity index 69% rename from pipelines/_steps/meta/flag_access.go rename to pipelines/steps/meta/flag_access.go index 18f9ed31..8cc313d3 100644 --- a/pipelines/_steps/meta/flag_access.go +++ b/pipelines/steps/meta/flag_access.go @@ -1,19 +1,21 @@ package meta import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/util" ) func FlagAccess(ctx rcontext.RequestContext, sha256hash string, uploadTime int64) { + uploaded := time.UnixMilli(uploadTime) if uploadTime > 0 { - metrics.MediaAgeAccessed.Observe(float64(util.NowMillis()-uploadTime) / 1000.0) + metrics.MediaAgeAccessed.Observe(time.Since(uploaded).Seconds()) } - if err := database.GetInstance().LastAccess.Prepare(ctx).Upsert(sha256hash, util.NowMillis()); err != nil { - ctx.Log.Warnf("Non-fatal error while updating last access for '%s': %s", sha256hash, err.Error()) + if err := database.GetInstance().LastAccess.Prepare(ctx).Upsert(sha256hash, time.Now().UnixMilli()); err != nil { + ctx.Log.Warnf("Non-fatal error while updating last access for '%s': %v", sha256hash, err) sentry.CaptureException(err) } } diff --git a/pipelines/_steps/quarantine/logic.go b/pipelines/steps/quarantine/logic.go similarity index 100% rename from pipelines/_steps/quarantine/logic.go rename to pipelines/steps/quarantine/logic.go diff --git a/pipelines/_steps/quarantine/thumbnail.go b/pipelines/steps/quarantine/thumbnail.go similarity index 100% rename from pipelines/_steps/quarantine/thumbnail.go rename to pipelines/steps/quarantine/thumbnail.go diff --git a/pipelines/_steps/quota/check.go b/pipelines/steps/quota/check.go similarity index 100% rename from pipelines/_steps/quota/check.go rename to pipelines/steps/quota/check.go diff --git a/pipelines/_steps/thumbnails/generate.go b/pipelines/steps/thumbnails/generate.go similarity index 95% rename from pipelines/_steps/thumbnails/generate.go rename to pipelines/steps/thumbnails/generate.go index a25f9d93..da36977a 100644 --- a/pipelines/_steps/thumbnails/generate.go +++ b/pipelines/steps/thumbnails/generate.go @@ -12,16 +12,16 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/datastores" "github.com/t2bot/matrix-media-repo/metrics" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/datastore_op" - "github.com/t2bot/matrix-media-repo/pipelines/_steps/download" + "github.com/t2bot/matrix-media-repo/pipelines/steps/datastore_op" + "github.com/t2bot/matrix-media-repo/pipelines/steps/download" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/thumbnailing" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" "github.com/t2bot/matrix-media-repo/util" ) type generateResult struct { - i *m.Thumbnail + i *preview.Thumbnail err error } diff --git a/pipelines/_steps/thumbnails/pick_dimensions.go b/pipelines/steps/thumbnails/pick_dimensions.go similarity index 100% rename from pipelines/_steps/thumbnails/pick_dimensions.go rename to pipelines/steps/thumbnails/pick_dimensions.go diff --git a/pipelines/_steps/upload/deduplicate.go b/pipelines/steps/upload/deduplicate.go similarity index 100% rename from pipelines/_steps/upload/deduplicate.go rename to pipelines/steps/upload/deduplicate.go diff --git a/pipelines/_steps/upload/generate_media_id.go b/pipelines/steps/upload/generate_media_id.go similarity index 100% rename from pipelines/_steps/upload/generate_media_id.go rename to pipelines/steps/upload/generate_media_id.go diff --git a/pipelines/_steps/upload/limit.go b/pipelines/steps/upload/limit.go similarity index 100% rename from pipelines/_steps/upload/limit.go rename to pipelines/steps/upload/limit.go diff --git a/pipelines/_steps/upload/lock.go b/pipelines/steps/upload/lock.go similarity index 94% rename from pipelines/_steps/upload/lock.go rename to pipelines/steps/upload/lock.go index 6a499b0d..89721ad0 100644 --- a/pipelines/_steps/upload/lock.go +++ b/pipelines/steps/upload/lock.go @@ -3,6 +3,7 @@ package upload import ( "context" "errors" + "fmt" "time" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -22,7 +23,7 @@ func LockForUpload(ctx rcontext.RequestContext, hash string) (func() error, erro } if err := mutex.LockContext(ctx.Context); err != nil { if time.Now().After(attemptDoneAt) { - return nil, errors.New("failed to acquire upload lock: " + err.Error()) + return nil, fmt.Errorf("failed to acquire upload lock: %w", err) } else { ctx.Log.Warn("failed to acquire upload lock: ", err) } diff --git a/pipelines/_steps/upload/quarantine.go b/pipelines/steps/upload/quarantine.go similarity index 100% rename from pipelines/_steps/upload/quarantine.go rename to pipelines/steps/upload/quarantine.go diff --git a/pipelines/_steps/upload/redis_async.go b/pipelines/steps/upload/redis_async.go similarity index 100% rename from pipelines/_steps/upload/redis_async.go rename to pipelines/steps/upload/redis_async.go diff --git a/pipelines/_steps/upload/spam.go b/pipelines/steps/upload/spam.go similarity index 100% rename from pipelines/_steps/upload/spam.go rename to pipelines/steps/upload/spam.go diff --git a/pipelines/_steps/url_preview/preview.go b/pipelines/steps/url_preview/preview.go similarity index 100% rename from pipelines/_steps/url_preview/preview.go rename to pipelines/steps/url_preview/preview.go diff --git a/pipelines/_steps/url_preview/process.go b/pipelines/steps/url_preview/process.go similarity index 100% rename from pipelines/_steps/url_preview/process.go rename to pipelines/steps/url_preview/process.go diff --git a/pipelines/_steps/url_preview/upload_image.go b/pipelines/steps/url_preview/upload_image.go similarity index 89% rename from pipelines/_steps/url_preview/upload_image.go rename to pipelines/steps/url_preview/upload_image.go index 4feafadf..54182107 100644 --- a/pipelines/_steps/url_preview/upload_image.go +++ b/pipelines/steps/url_preview/upload_image.go @@ -21,8 +21,7 @@ func UploadImage(ctx rcontext.RequestContext, image *m.PreviewImage, onHost stri defer image.Data.Close() pr, pw := io.Pipe() tee := io.TeeReader(image.Data, pw) - mediaChan := make(chan *database.DbMedia) - defer close(mediaChan) + mediaChan := make(chan *database.DbMedia, 1) go func() { media, err := pipeline_upload.Execute(ctx, onHost, "", io.NopCloser(tee), image.ContentType, image.Filename, userId, datastores.LocalMediaKind) if err != nil { @@ -30,12 +29,8 @@ func UploadImage(ctx rcontext.RequestContext, image *m.PreviewImage, onHost stri } else { _ = pw.Close() } - go func() { - defer func() { - recover() // consume write-to-closed-channel errors - }() - mediaChan <- media - }() + mediaChan <- media + close(mediaChan) }() w := 0 diff --git a/plugins/manager.go b/plugins/manager.go index 7ae7a68e..3df22602 100644 --- a/plugins/manager.go +++ b/plugins/manager.go @@ -21,7 +21,7 @@ func ReloadPlugins() { logrus.Info("Loading plugin: ", pl.Executable) mmr, err := newPlugin(pl.Executable, pl.Config) if err != nil { - logrus.Errorf("failed to load plugin %s: %s", pl.Executable, err.Error()) + logrus.Errorf("failed to load plugin %s: %v", pl.Executable, err) continue } @@ -46,7 +46,7 @@ func CheckForSpam(r io.Reader, filename string, contentType string, userId strin for _, pl := range existingPlugins { as, err := pl.Antispam() if err != nil { - logrus.Warnf("error loading antispam plugin: %s", err.Error()) + logrus.Warnf("error loading antispam plugin: %v", err) continue } diff --git a/tasks/exec.go b/tasks/exec.go index 31ac14c2..a842f86e 100644 --- a/tasks/exec.go +++ b/tasks/exec.go @@ -11,7 +11,6 @@ import ( "github.com/t2bot/matrix-media-repo/notifier" "github.com/t2bot/matrix-media-repo/pool" "github.com/t2bot/matrix-media-repo/tasks/task_runner" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/ids" ) @@ -63,7 +62,7 @@ func beginTask(task *database.DbTask) { } runnerCtx := rcontext.Initial().LogWithFields(logrus.Fields{"task_id": task.TaskId}) - oneHourAgo := util.NowMillis() - (60 * 60 * 1000) + oneHourAgo := time.Now().Add(-1 * time.Hour).UnixMilli() if task.StartTs < oneHourAgo { runnerCtx.Log.Warn("Not starting task because it is more than 1 hour old.") return @@ -82,7 +81,7 @@ func beginTask(task *database.DbTask) { sentry.CaptureMessage(m) } }); err != nil { - m := fmt.Sprintf("Error trying to schedule task %s (ID: %d): %s", task.Name, task.TaskId, err.Error()) + m := fmt.Sprintf("Error trying to schedule task %s (ID: %d): %v", task.Name, task.TaskId, err) runnerCtx.Log.Warn(m) sentry.CaptureMessage(m) time.AfterFunc(15*time.Second, func() { diff --git a/tasks/schedule.go b/tasks/schedule.go index 3f4b83b4..b3c0c578 100644 --- a/tasks/schedule.go +++ b/tasks/schedule.go @@ -11,7 +11,6 @@ import ( "github.com/t2bot/matrix-media-repo/database" "github.com/t2bot/matrix-media-repo/notifier" "github.com/t2bot/matrix-media-repo/tasks/task_runner" - "github.com/t2bot/matrix-media-repo/util" "github.com/t2bot/matrix-media-repo/util/ids" ) @@ -34,7 +33,7 @@ const ExecutingMachineId = int64(0) type RecurringTaskFn func(ctx rcontext.RequestContext) -var localRand = rand.New(rand.NewSource(util.NowMillis())) +var localRand = rand.New(rand.NewSource(time.Now().UnixNano())) var recurDoneChs = make(map[RecurringTaskName]chan bool) var recurLock = new(sync.RWMutex) @@ -44,7 +43,7 @@ func scheduleTask(ctx rcontext.RequestContext, name TaskName, params interface{} return nil, err } db := database.GetInstance().Tasks.Prepare(ctx) - r, err := db.Insert(string(name), jsonParams, util.NowMillis()) + r, err := db.Insert(string(name), jsonParams, time.Now().UnixMilli()) if err != nil { return nil, err } diff --git a/tasks/task_runner/00-internal.go b/tasks/task_runner/00-internal.go index 7a39411b..e8c07abb 100644 --- a/tasks/task_runner/00-internal.go +++ b/tasks/task_runner/00-internal.go @@ -1,15 +1,16 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func markDone(ctx rcontext.RequestContext, task *database.DbTask) { taskDb := database.GetInstance().Tasks.Prepare(ctx) - if err := taskDb.SetEndTime(task.TaskId, util.NowMillis()); err != nil { + if err := taskDb.SetEndTime(task.TaskId, time.Now().UnixMilli()); err != nil { ctx.Log.Warn("Error updating task as complete: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_held_media_ids.go b/tasks/task_runner/purge_held_media_ids.go index 719f4f43..4c25cee4 100644 --- a/tasks/task_runner/purge_held_media_ids.go +++ b/tasks/task_runner/purge_held_media_ids.go @@ -1,19 +1,20 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func PurgeHeldMediaIds(ctx rcontext.RequestContext) { // dev note: don't use ctx for config lookup to avoid misreading it - beforeTs := util.NowMillis() - int64(7*24*60*60*1000) // 7 days + before := time.Now().AddDate(0, 0, -7) db := database.GetInstance().HeldMedia.Prepare(ctx) - if err := db.DeleteOlderThan(database.ForCreateHeldReason, beforeTs); err != nil { + if err := db.DeleteOlderThan(database.ForCreateHeldReason, before); err != nil { ctx.Log.Error("Error deleting held media IDs: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_previews.go b/tasks/task_runner/purge_previews.go index c7d3e305..7ecaea89 100644 --- a/tasks/task_runner/purge_previews.go +++ b/tasks/task_runner/purge_previews.go @@ -1,11 +1,12 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" "github.com/t2bot/matrix-media-repo/database" - "github.com/t2bot/matrix-media-repo/util" ) func PurgePreviews(ctx rcontext.RequestContext) { @@ -15,11 +16,11 @@ func PurgePreviews(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().UrlPreviews.ExpireDays*24*60*60*1000) + before := time.Now().AddDate(0, 0, -1*config.Get().UrlPreviews.ExpireDays) db := database.GetInstance().UrlPreviews.Prepare(ctx) // TODO: Fix https://github.com/t2bot/matrix-media-repo/issues/424 ("can't clean up preview media") - if err := db.DeleteOlderThan(beforeTs); err != nil { + if err := db.DeleteOlderThan(before); err != nil { ctx.Log.Error("Error deleting previews: ", err) sentry.CaptureException(err) } diff --git a/tasks/task_runner/purge_remote_media.go b/tasks/task_runner/purge_remote_media.go index dc4a0f3b..1b4a7bf1 100644 --- a/tasks/task_runner/purge_remote_media.go +++ b/tasks/task_runner/purge_remote_media.go @@ -1,6 +1,8 @@ package task_runner import ( + "time" + "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" @@ -15,8 +17,8 @@ func PurgeRemoteMedia(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().Downloads.ExpireDays*24*60*60*1000) - _, err := PurgeRemoteMediaBefore(ctx, beforeTs) + before := time.Now().AddDate(0, 0, -1*config.Get().Downloads.ExpireDays) + _, err := PurgeRemoteMediaBefore(ctx, before) if err != nil { ctx.Log.Error("Error purging media: ", err) sentry.CaptureException(err) @@ -24,7 +26,7 @@ func PurgeRemoteMedia(ctx rcontext.RequestContext) { } // PurgeRemoteMediaBefore returns (count affected, error) -func PurgeRemoteMediaBefore(ctx rcontext.RequestContext, beforeTs int64) (int, error) { +func PurgeRemoteMediaBefore(ctx rcontext.RequestContext, beforeTs time.Time) (int, error) { mediaDb := database.GetInstance().Media.Prepare(ctx) origins := util.GetOurDomains() diff --git a/tasks/task_runner/purge_thumbnails.go b/tasks/task_runner/purge_thumbnails.go index 79ea16ba..bd40b585 100644 --- a/tasks/task_runner/purge_thumbnails.go +++ b/tasks/task_runner/purge_thumbnails.go @@ -2,6 +2,7 @@ package task_runner import ( "fmt" + "time" "github.com/getsentry/sentry-go" "github.com/t2bot/matrix-media-repo/common/config" @@ -18,9 +19,9 @@ func PurgeThumbnails(ctx rcontext.RequestContext) { return } - beforeTs := util.NowMillis() - int64(config.Get().UrlPreviews.ExpireDays*24*60*60*1000) + before := time.Now().AddDate(0, 0, -1*config.Get().UrlPreviews.ExpireDays) thumbsDb := database.GetInstance().Thumbnails.Prepare(ctx) - old, err := thumbsDb.GetOlderThan(beforeTs) + old, err := thumbsDb.GetOlderThan(before) if err != nil { ctx.Log.Error("Error deleting thumbnails: ", err) sentry.CaptureException(err) diff --git a/test/test_internals/deps.go b/test/test_internals/deps.go index 18ba27dd..61b54290 100644 --- a/test/test_internals/deps.go +++ b/test/test_internals/deps.go @@ -167,15 +167,15 @@ func (c *ContainerDeps) Teardown() { hs.Teardown() } if err := c.redisContainer.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down redis container: %s", err.Error()) + log.Fatalf("Error shutting down redis container: %v", err) } if err := c.pgContainer.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down mmr-postgres container: %s", err.Error()) + log.Fatalf("Error shutting down mmr-postgres container: %v", err) } c.minioDep.Teardown() c.depNet.Teardown() if err := os.Remove(c.mmrExtConfigPath); err != nil && !os.IsNotExist(err) { - log.Fatalf("Error cleaning up MMR-External config file '%s': %s", c.mmrExtConfigPath, err.Error()) + log.Fatalf("Error cleaning up MMR-External config file '%s': %v", c.mmrExtConfigPath, err) } } diff --git a/test/test_internals/deps_minio.go b/test/test_internals/deps_minio.go index 07369e70..1601cc52 100644 --- a/test/test_internals/deps_minio.go +++ b/test/test_internals/deps_minio.go @@ -125,6 +125,6 @@ func MakeMinio(depNet *NetworkDep) (*MinioDep, error) { func (c *MinioDep) Teardown() { if err := c.container.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down minio: %s", err.Error()) + log.Fatalf("Error shutting down minio: %v", err) } } diff --git a/test/test_internals/deps_mmr.go b/test/test_internals/deps_mmr.go index a3d3b9af..1ea7354e 100644 --- a/test/test_internals/deps_mmr.go +++ b/test/test_internals/deps_mmr.go @@ -137,10 +137,10 @@ func makeMmrInstances(ctx context.Context, count int, depNet *NetworkDep, tmplAr func (c *mmrContainer) Teardown() { if err := c.container.Terminate(c.ctx); err != nil { - log.Fatalf("Error shutting down MMR machine %d: %s", c.MachineId, err.Error()) + log.Fatalf("Error shutting down MMR machine %d: %v", c.MachineId, err) } if err := os.Remove(c.tmpConfigPath); err != nil && !os.IsNotExist(err) { - log.Fatalf("Error cleaning up MMR config file '%s': %s", c.tmpConfigPath, err.Error()) + log.Fatalf("Error cleaning up MMR config file '%s': %v", c.tmpConfigPath, err) } } diff --git a/test/test_internals/deps_network.go b/test/test_internals/deps_network.go index 3887cfa1..7daba444 100644 --- a/test/test_internals/deps_network.go +++ b/test/test_internals/deps_network.go @@ -57,6 +57,6 @@ func (n *NetworkDep) ApplyToContainer() testcontainers.ContainerCustomizer { func (n *NetworkDep) Teardown() { if err := n.dockerNet.Remove(n.ctx); err != nil { - log.Fatalf("Error cleaning up docker network '%s': %s", n.NetId, err.Error()) + log.Fatalf("Error cleaning up docker network '%s': %v", n.NetId, err) } } diff --git a/thumbnailing/m/thumbnail.go b/thumbnailing/m/thumbnail.go deleted file mode 100644 index 04017f63..00000000 --- a/thumbnailing/m/thumbnail.go +++ /dev/null @@ -1,11 +0,0 @@ -package m - -import ( - "io" -) - -type Thumbnail struct { - Animated bool - ContentType string - Reader io.ReadCloser -} diff --git a/thumbnailing/i/01-factories.go b/thumbnailing/preview/01-factories.go similarity index 91% rename from thumbnailing/i/01-factories.go rename to thumbnailing/preview/01-factories.go index 5c56603e..fbc33278 100644 --- a/thumbnailing/i/01-factories.go +++ b/thumbnailing/preview/01-factories.go @@ -1,10 +1,9 @@ -package i +package preview import ( "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "github.com/t2bot/matrix-media-repo/util/readers" ) @@ -12,13 +11,13 @@ type Generator interface { supportedContentTypes() []string supportsAnimation() bool matches(img io.Reader, contentType string) bool - GenerateThumbnail(img io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) + GenerateThumbnail(img io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) } type AudioGenerator interface { Generator - GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) + GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) } var generators = make([]Generator, 0) diff --git a/thumbnailing/i/apng.go b/thumbnailing/preview/apng.go similarity index 87% rename from thumbnailing/i/apng.go rename to thumbnailing/preview/apng.go index 6bc0f730..ea27095c 100644 --- a/thumbnailing/i/apng.go +++ b/thumbnailing/preview/apng.go @@ -1,7 +1,7 @@ -package i +package preview import ( - "errors" + "fmt" "image" "image/draw" "io" @@ -9,12 +9,10 @@ import ( "github.com/getsentry/sentry-go" "github.com/kettek/apng" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) -type apngGenerator struct { -} +type apngGenerator struct{} func (d apngGenerator) supportedContentTypes() []string { return []string{"image/png", "image/apng"} @@ -36,14 +34,14 @@ func (d apngGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { if !animated { return pngGenerator{}.GenerateThumbnail(b, "image/png", width, height, method, false, ctx) } p, err := apng.DecodeAll(b) if err != nil { - return nil, errors.New("apng: error decoding image: " + err.Error()) + return nil, fmt.Errorf("apng: error decoding image: %w", err) } // prepare a blank frame to use as swap space @@ -70,9 +68,9 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width draw.Draw(frameImg, image.Rect(frame.XOffset, frame.YOffset, frameImg.Rect.Max.X, frameImg.Rect.Max.Y), img, image.Point{X: 0, Y: 0}, draw.Src) // Do the thumbnailing on the copied frame - frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) + frameThumb, err := metadata.MakeThumbnail(frameImg, method, width, height) if err != nil { - return nil, errors.New("apng: error generating thumbnail frame: " + err.Error()) + return nil, fmt.Errorf("apng: error generating thumbnail frame: %w", err) } if frameThumb == nil { tmpImg := image.NewRGBA(frameImg.Bounds()) @@ -94,13 +92,13 @@ func (d apngGenerator) GenerateThumbnail(b io.Reader, contentType string, width go func(pw *io.PipeWriter, p apng.APNG) { err = apng.Encode(pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("apng: error encoding final thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("apng: error encoding final thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, p) - return &m.Thumbnail{ + return &Thumbnail{ ContentType: "image/png", Animated: true, Reader: pr, diff --git a/thumbnailing/i/bmp.go b/thumbnailing/preview/bmp.go similarity index 73% rename from thumbnailing/i/bmp.go rename to thumbnailing/preview/bmp.go index bbf6d50d..67c679b9 100644 --- a/thumbnailing/i/bmp.go +++ b/thumbnailing/preview/bmp.go @@ -1,17 +1,15 @@ -package i +package preview import ( - "errors" + "fmt" "io" + "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" "golang.org/x/image/bmp" ) -type bmpGenerator struct { -} +type bmpGenerator struct{} func (d bmpGenerator) supportedContentTypes() []string { return []string{"image/bmp", "image/x-bmp"} @@ -22,7 +20,7 @@ func (d bmpGenerator) supportsAnimation() bool { } func (d bmpGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d bmpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { @@ -33,10 +31,10 @@ func (d bmpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return true, i.Width, i.Height, nil } -func (d bmpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d bmpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := bmp.Decode(b) if err != nil { - return nil, errors.New("bmp: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("bmp: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/i/flac.go b/thumbnailing/preview/flac.go similarity index 72% rename from thumbnailing/i/flac.go rename to thumbnailing/preview/flac.go index 56a92dc3..a62254c0 100644 --- a/thumbnailing/i/flac.go +++ b/thumbnailing/preview/flac.go @@ -1,18 +1,16 @@ -package i +package preview import ( - "errors" + "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/flac" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type flacGenerator struct { -} +type flacGenerator struct{} func (d flacGenerator) supportedContentTypes() []string { return []string{"audio/flac"} @@ -29,7 +27,7 @@ func (d flacGenerator) matches(img io.Reader, contentType string) bool { func (d flacGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := flac.Decode(b) if err != nil { - return audio, format, errors.New("flac: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("flac: error decoding audio: %w", err) } return audio, format, nil } @@ -38,15 +36,17 @@ func (d flacGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return false, 0, 0, nil } -func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { - tags, rc, err := u.GetID3Tags(r) +func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { + rd, err := newReadSeekerWrapper(r) if err != nil { - return nil, errors.New("flac: error getting tags: " + err.Error()) + return nil, fmt.Errorf("error wrapping reader: %w", err) + } + tags, err := tag.ReadFrom(rd) + if err != nil { + return nil, fmt.Errorf("flac: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func (d flacGenerator) GenerateThumbnail(r io.Reader, contentType string, width return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d flacGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d flacGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/i/gif.go b/thumbnailing/preview/gif.go similarity index 82% rename from thumbnailing/i/gif.go rename to thumbnailing/preview/gif.go index 676d0cba..b9c92fba 100644 --- a/thumbnailing/i/gif.go +++ b/thumbnailing/preview/gif.go @@ -1,7 +1,7 @@ -package i +package preview import ( - "errors" + "fmt" "image" "image/draw" "image/gif" @@ -9,12 +9,10 @@ import ( "math" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) -type gifGenerator struct { -} +type gifGenerator struct{} func (d gifGenerator) supportedContentTypes() []string { return []string{"image/gif"} @@ -32,10 +30,10 @@ func (d gifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return pngGenerator{}.GetOriginDimensions(b, contentType, ctx) } -func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { g, err := gif.DecodeAll(b) if err != nil { - return nil, errors.New("gif: error decoding image: " + err.Error()) + return nil, fmt.Errorf("gif: error decoding image: %w", err) } // Prepare a blank frame to use as swap space @@ -56,9 +54,9 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i draw.Draw(frameImg, frameImg.Bounds(), img, image.Point{X: 0, Y: 0}, draw.Over) // Do the thumbnailing on the copied frame - frameThumb, err := u.MakeThumbnail(frameImg, method, width, height) + frameThumb, err := metadata.MakeThumbnail(frameImg, method, width, height) if err != nil { - return nil, errors.New("gif: error generating thumbnail frame: " + err.Error()) + return nil, fmt.Errorf("gif: error generating thumbnail frame: %w", err) } if frameThumb == nil { tmpImg := image.NewRGBA(frameImg.Bounds()) @@ -78,14 +76,14 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i // The thumbnailer decided that it shouldn't thumbnail, so encode it ourselves pr, pw := io.Pipe() go func(pw *io.PipeWriter, p *image.Paletted) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("gif: error encoding still frame thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("gif: error encoding still frame thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, targetImg) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, @@ -114,13 +112,13 @@ func (d gifGenerator) GenerateThumbnail(b io.Reader, contentType string, width i go func(pw *io.PipeWriter, g *gif.GIF) { err = gif.EncodeAll(pw, g) if err != nil { - _ = pw.CloseWithError(errors.New("gif: error encoding final thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("gif: error encoding final thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, g) - return &m.Thumbnail{ + return &Thumbnail{ ContentType: "image/gif", Animated: true, Reader: pr, diff --git a/thumbnailing/i/heif.go b/thumbnailing/preview/heif.go similarity index 73% rename from thumbnailing/i/heif.go rename to thumbnailing/preview/heif.go index 8d4b005e..7a6f3112 100644 --- a/thumbnailing/i/heif.go +++ b/thumbnailing/preview/heif.go @@ -1,18 +1,16 @@ -package i +package preview import ( - "errors" + "fmt" "image" "io" + "slices" _ "github.com/strukturag/libheif/go/heif" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" ) -type heifGenerator struct { -} +type heifGenerator struct{} func (d heifGenerator) supportedContentTypes() []string { return []string{"image/heif", "image/heic"} @@ -23,7 +21,7 @@ func (d heifGenerator) supportsAnimation() bool { } func (d heifGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d heifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { @@ -34,10 +32,10 @@ func (d heifGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, cfg.Width, cfg.Height, nil } -func (d heifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d heifGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, _, err := image.Decode(b) if err != nil { - return nil, errors.New("heif: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("heif: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/i/jpegxl.go b/thumbnailing/preview/jpegxl.go similarity index 64% rename from thumbnailing/i/jpegxl.go rename to thumbnailing/preview/jpegxl.go index 72f145f4..928b8b77 100644 --- a/thumbnailing/i/jpegxl.go +++ b/thumbnailing/preview/jpegxl.go @@ -1,18 +1,16 @@ -package i +package preview import ( - "errors" + "fmt" "io" "os" "os/exec" "path" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) -type jpegxlGenerator struct { -} +type jpegxlGenerator struct{} func (d jpegxlGenerator) supportedContentTypes() []string { return []string{"image/jxl"} @@ -30,10 +28,10 @@ func (d jpegxlGenerator) GetOriginDimensions(b io.Reader, contentType string, ct return false, 0, 0, nil } -func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-jpegxl") if err != nil { - return nil, errors.New("jpegxl: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.jpegxl") @@ -43,22 +41,22 @@ func (d jpegxlGenerator) GenerateThumbnail(b io.Reader, contentType string, widt defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("jpegxl: error creating temp jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error creating temp jpegxl file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("jpegxl: error writing temp jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error writing temp jpegxl file: %w", err) } err = exec.Command("convert", tempFile1, tempFile2).Run() if err != nil { - return nil, errors.New("jpegxl: error converting jpegxl file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error converting jpegxl file: %w", err) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("jpegxl: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("jpegxl: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/jpg.go b/thumbnailing/preview/jpg.go similarity index 60% rename from thumbnailing/i/jpg.go rename to thumbnailing/preview/jpg.go index f05d8190..5f4bc113 100644 --- a/thumbnailing/i/jpg.go +++ b/thumbnailing/preview/jpg.go @@ -1,21 +1,19 @@ -package i +package preview import ( - "errors" + "fmt" "image" _ "image/jpeg" "io" + "slices" "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" - "github.com/t2bot/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) -type jpgGenerator struct { -} +type jpgGenerator struct{} func (d jpgGenerator) supportedContentTypes() []string { return []string{"image/jpeg", "image/jpg"} @@ -26,41 +24,41 @@ func (d jpgGenerator) supportsAnimation() bool { } func (d jpgGenerator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d jpgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { return pngGenerator{}.GetOriginDimensions(b, contentType, ctx) } -func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d jpgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { br := readers.NewBufferReadsReader(b) - orientation := u.ExtractExifOrientation(br) + orientation := metadata.ExtractExifOrientation(br) b = br.GetRewoundReader() src, err := imaging.Decode(b) if err != nil { - return nil, errors.New("jpg: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("jpg: error decoding thumbnail: %w", err) } - thumb, err := u.MakeThumbnail(src, method, width, height) + thumb, err := metadata.MakeThumbnail(src, method, width, height) if err != nil { - return nil, errors.New("jpg: error making thumbnail: " + err.Error()) + return nil, fmt.Errorf("jpg: error making thumbnail: %w", err) } - thumb = u.ApplyOrientation(thumb, orientation) + thumb = metadata.ApplyOrientation(thumb, orientation) pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p, u.JpegSource) + err = metadata.Encode(ctx, pw, p, metadata.JpegSource) if err != nil { - _ = pw.CloseWithError(errors.New("jpg: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("jpg: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, thumb) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/jpeg", Reader: pr, diff --git a/thumbnailing/u/dimensions.go b/thumbnailing/preview/metadata/dimensions.go similarity index 77% rename from thumbnailing/u/dimensions.go rename to thumbnailing/preview/metadata/dimensions.go index 3b83c4e2..dd8a06a3 100644 --- a/thumbnailing/u/dimensions.go +++ b/thumbnailing/preview/metadata/dimensions.go @@ -1,4 +1,4 @@ -package u +package metadata func AdjustProperties(srcWidth int, srcHeight int, desiredWidth int, desiredHeight int, wantAnimated bool, method string) (bool, int, int, string) { aspectRatio := float32(srcHeight) / float32(srcWidth) @@ -9,11 +9,7 @@ func AdjustProperties(srcWidth int, srcHeight int, desiredWidth int, desiredHeig } if srcWidth <= desiredWidth && srcHeight <= desiredHeight { - if wantAnimated { - return true, srcWidth, srcHeight, method - } else { - return false, desiredWidth, desiredHeight, method - } + return wantAnimated, srcWidth, srcHeight, method } return true, desiredWidth, desiredHeight, method } diff --git a/thumbnailing/u/encode.go b/thumbnailing/preview/metadata/encode.go similarity index 97% rename from thumbnailing/u/encode.go rename to thumbnailing/preview/metadata/encode.go index 9effd8e3..bb339226 100644 --- a/thumbnailing/u/encode.go +++ b/thumbnailing/preview/metadata/encode.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "image" diff --git a/thumbnailing/u/exif.go b/thumbnailing/preview/metadata/exif.go similarity index 86% rename from thumbnailing/u/exif.go rename to thumbnailing/preview/metadata/exif.go index 636831ed..65756264 100644 --- a/thumbnailing/u/exif.go +++ b/thumbnailing/preview/metadata/exif.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "errors" @@ -16,16 +16,16 @@ type ExifOrientation struct { func GetExifOrientation(img io.Reader) (*ExifOrientation, error) { rawExif, err := exif.SearchAndExtractExifWithReader(img) - if err != nil { - if errors.Is(err, exif.ErrNoExif) { - return nil, nil - } - return nil, errors.New("exif: error reading possible exif data: " + err.Error()) + switch { + case errors.Is(err, exif.ErrNoExif): + return nil, nil + case err != nil: + return nil, fmt.Errorf("exif: error reading possible exif data: %w", err) } tags, _, err := exif.GetFlatExifData(rawExif, nil) if err != nil { - return nil, errors.New("exif: error parsing exif data: " + err.Error()) + return nil, fmt.Errorf("exif: error parsing exif data: %w", err) } var tag exif.ExifTag diff --git a/thumbnailing/u/framing.go b/thumbnailing/preview/metadata/framing.go similarity index 78% rename from thumbnailing/u/framing.go rename to thumbnailing/preview/metadata/framing.go index 80f5f9f2..43ba044c 100644 --- a/thumbnailing/u/framing.go +++ b/thumbnailing/preview/metadata/framing.go @@ -1,4 +1,4 @@ -package u +package metadata import ( "errors" @@ -11,22 +11,21 @@ import ( ) func MakeThumbnail(src image.Image, method string, width int, height int) (image.Image, error) { - var result image.Image - if method == "scale" { - result = imaging.Fit(src, width, height, imaging.Linear) - } else if method == "crop" { - result = imaging.Fill(src, width, height, imaging.Center, imaging.Linear) - } else { + switch method { + case "scale": + return imaging.Fit(src, width, height, imaging.Linear), nil + case "crop": + return imaging.Fill(src, width, height, imaging.Center, imaging.Linear), nil + default: return nil, errors.New("unrecognized method: " + method) } - return result, nil } func ExtractExifOrientation(r io.Reader) *ExifOrientation { orientation, err := GetExifOrientation(r) if err != nil { // assume no orientation if there was an error reading the exif header - logrus.Warn("Non-fatal error reading exif headers:", err.Error()) + logrus.Warnf("Non-fatal error reading exif headers: %v", err) sentry.CaptureException(err) orientation = nil } diff --git a/thumbnailing/u/sample.go b/thumbnailing/preview/metadata/sample.go similarity index 89% rename from thumbnailing/u/sample.go rename to thumbnailing/preview/metadata/sample.go index 4e055add..75fcc449 100644 --- a/thumbnailing/u/sample.go +++ b/thumbnailing/preview/metadata/sample.go @@ -1,7 +1,8 @@ -package u +package metadata import ( "errors" + "fmt" "math" "github.com/faiface/beep" @@ -16,7 +17,7 @@ func FastSampleAudio(stream beep.StreamSeekCloser, numSamples int) ([][2]float64 if stream.Position() != pos { err := stream.Seek(pos) if err != nil { - return nil, errors.New("fast-sample: could not seek: " + err.Error()) + return nil, fmt.Errorf("fast-sample: could not seek: %w", err) } } diff --git a/thumbnailing/i/mp3.go b/thumbnailing/preview/mp3.go similarity index 80% rename from thumbnailing/i/mp3.go rename to thumbnailing/preview/mp3.go index 19641313..b1ed1dd6 100644 --- a/thumbnailing/i/mp3.go +++ b/thumbnailing/preview/mp3.go @@ -1,8 +1,8 @@ -package i +package preview import ( "bytes" - "errors" + "fmt" "image" "image/color" "image/draw" @@ -17,13 +17,11 @@ import ( "github.com/sirupsen/logrus" "github.com/t2bot/matrix-media-repo/common/config" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) -type mp3Generator struct { -} +type mp3Generator struct{} func (d mp3Generator) supportedContentTypes() []string { return []string{"audio/mpeg"} @@ -40,7 +38,7 @@ func (d mp3Generator) matches(img io.Reader, contentType string) bool { func (d mp3Generator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := mp3.Decode(readers.MakeCloser(b)) if err != nil { - return audio, format, errors.New("mp3: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("mp3: error decoding audio: %w", err) } return audio, format, nil } @@ -49,15 +47,14 @@ func (d mp3Generator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) +func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { + rd, err := newReadSeekerWrapper(b) + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { - return nil, errors.New("mp3: error getting tags: " + err.Error()) + return nil, fmt.Errorf("mp3: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } @@ -67,7 +64,7 @@ func (d mp3Generator) GenerateThumbnail(b io.Reader, contentType string, width i return d.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err @@ -78,14 +75,14 @@ func (d mp3Generator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestC return d.GetDataFromStream(audio, format, nKeys) } -func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*m.AudioInfo, error) { +func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep.Format, nKeys int) (*AudioInfo, error) { totalSamples := audio.Len() - downsampled, err := u.FastSampleAudio(audio, nKeys) + downsampled, err := metadata.FastSampleAudio(audio, nKeys) if err != nil { return nil, err } - return &m.AudioInfo{ + return &AudioInfo{ Duration: format.SampleRate.D(totalSamples), Channels: format.NumChannels, TotalSamples: totalSamples, @@ -93,7 +90,7 @@ func (d mp3Generator) GetDataFromStream(audio beep.StreamSeekCloser, format beep }, nil } -func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format beep.Format, meta tag.Metadata, width int, height int, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format beep.Format, meta tag.Metadata, width int, height int, ctx rcontext.RequestContext) (*Thumbnail, error) { bgColor := color.RGBA{A: 255, R: 41, G: 57, B: 92} fgColor := color.RGBA{A: 255, R: 240, G: 240, B: 240} @@ -105,7 +102,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee if meta != nil && meta.Picture() != nil { artwork, _, _ := image.Decode(bytes.NewBuffer(meta.Picture().Data)) if artwork != nil { - artworkImg, _ = u.MakeThumbnail(artwork, "crop", sq, sq) + artworkImg, _ = metadata.MakeThumbnail(artwork, "crop", sq, sq) } } @@ -124,12 +121,12 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee r := image.Rect(dx, dy, ddx, ddy) if artworkImg == nil { - f, _ := os.OpenFile(path.Join(config.Runtime.AssetsPath, "default-artwork.png"), os.O_RDONLY, 0640) + f, _ := os.OpenFile(path.Join(config.Runtime.AssetsPath, "default-artwork.png"), os.O_RDONLY, 0o640) if f != nil { defer f.Close() tmp, _, _ := image.Decode(f) if tmp != nil { - artworkImg, _ = u.MakeThumbnail(tmp, "crop", ax, ay) + artworkImg, _ = metadata.MakeThumbnail(tmp, "crop", ax, ay) } } if artworkImg == nil { @@ -149,7 +146,7 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee waveformX := padding + r.Max.X info, err := d.GetDataFromStream(audio, format, (int)(math.Max((float64)(width-waveformX-padding), 1))) if err != nil { - return nil, errors.New("beep-visual: error sampling audio: " + err.Error()) + return nil, fmt.Errorf("beep-visual: error sampling audio: %w", err) } // Average out all the samples @@ -198,15 +195,15 @@ func (d mp3Generator) GenerateFromStream(audio beep.StreamSeekCloser, format bee // Encode to a png pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("beep-visual: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("beep-visual: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, img) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, diff --git a/thumbnailing/i/mp4.go b/thumbnailing/preview/mp4.go similarity index 61% rename from thumbnailing/i/mp4.go rename to thumbnailing/preview/mp4.go index 99d69ad4..6d13a32f 100644 --- a/thumbnailing/i/mp4.go +++ b/thumbnailing/preview/mp4.go @@ -1,19 +1,17 @@ -package i +package preview import ( - "errors" + "fmt" "io" "os" "os/exec" "path" + "slices" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/util" ) -type mp4Generator struct { -} +type mp4Generator struct{} func (d mp4Generator) supportedContentTypes() []string { return []string{"video/mp4"} @@ -24,17 +22,17 @@ func (d mp4Generator) supportsAnimation() bool { } func (d mp4Generator) matches(img io.Reader, contentType string) bool { - return util.ArrayContains(d.supportedContentTypes(), contentType) + return slices.Contains(d.supportedContentTypes(), contentType) } func (d mp4Generator) GetOriginDimensions(b io.Reader, contentType string, ctx rcontext.RequestContext) (bool, int, int, error) { return false, 0, 0, nil } -func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-mp4") if err != nil { - return nil, errors.New("mp4: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("mp4: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.mp4") @@ -44,22 +42,22 @@ func (d mp4Generator) GenerateThumbnail(b io.Reader, contentType string, width i defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("mp4: error creating temp video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error creating temp video file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("mp4: error writing temp video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error writing temp video file: %w", err) } err = exec.Command("ffmpeg", "-i", tempFile1, "-vf", "select=eq(n\\,0)", tempFile2).Run() if err != nil { - return nil, errors.New("mp4: error converting video file: " + err.Error()) + return nil, fmt.Errorf("mp4: error converting video file: %w", err) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("mp4: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("mp4: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/ogg.go b/thumbnailing/preview/ogg.go similarity index 73% rename from thumbnailing/i/ogg.go rename to thumbnailing/preview/ogg.go index 0eca7fab..b8b22b73 100644 --- a/thumbnailing/i/ogg.go +++ b/thumbnailing/preview/ogg.go @@ -1,19 +1,17 @@ -package i +package preview import ( - "errors" + "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/vorbis" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" "github.com/t2bot/matrix-media-repo/util/readers" ) -type oggGenerator struct { -} +type oggGenerator struct{} func (d oggGenerator) supportedContentTypes() []string { return []string{"audio/ogg"} @@ -30,7 +28,7 @@ func (d oggGenerator) matches(img io.Reader, contentType string) bool { func (d oggGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := vorbis.Decode(readers.MakeCloser(b)) if err != nil { - return audio, format, errors.New("ogg: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("ogg: error decoding audio: %w", err) } return audio, format, nil } @@ -39,15 +37,14 @@ func (d oggGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) +func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { + rd, err := newReadSeekerWrapper(b) + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { - return nil, errors.New("ogg: error getting tags: " + err.Error()) + return nil, fmt.Errorf("ogg: error getting tags: %v", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } @@ -57,7 +54,7 @@ func (d oggGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d oggGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d oggGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/i/png.go b/thumbnailing/preview/png.go similarity index 73% rename from thumbnailing/i/png.go rename to thumbnailing/preview/png.go index f546ecf6..9835afc7 100644 --- a/thumbnailing/i/png.go +++ b/thumbnailing/preview/png.go @@ -1,19 +1,17 @@ -package i +package preview import ( - "errors" + "fmt" "image" _ "image/png" "io" "github.com/disintegration/imaging" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" ) -type pngGenerator struct { -} +type pngGenerator struct{} func (d pngGenerator) supportedContentTypes() []string { return []string{"image/png"} @@ -35,32 +33,32 @@ func (d pngGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return true, i.Width, i.Height, nil } -func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d pngGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := imaging.Decode(b) if err != nil { - return nil, errors.New("png: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("png: error decoding thumbnail: %w", err) } return d.GenerateThumbnailOf(src, width, height, method, ctx) } -func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int, method string, ctx rcontext.RequestContext) (*m.Thumbnail, error) { - thumb, err := u.MakeThumbnail(src, method, width, height) +func (d pngGenerator) GenerateThumbnailOf(src image.Image, width int, height int, method string, ctx rcontext.RequestContext) (*Thumbnail, error) { + thumb, err := metadata.MakeThumbnail(src, method, width, height) if err != nil || thumb == nil { return nil, err } pr, pw := io.Pipe() go func(pw *io.PipeWriter, p image.Image) { - err = u.Encode(ctx, pw, p) + err = metadata.Encode(ctx, pw, p) if err != nil { - _ = pw.CloseWithError(errors.New("png: error encoding thumbnail: " + err.Error())) + _ = pw.CloseWithError(fmt.Errorf("png: error encoding thumbnail: %w", err)) } else { _ = pw.Close() } }(pw, thumb) - return &m.Thumbnail{ + return &Thumbnail{ Animated: false, ContentType: "image/png", Reader: pr, diff --git a/thumbnailing/preview/readSeeker.go b/thumbnailing/preview/readSeeker.go new file mode 100644 index 00000000..d3244170 --- /dev/null +++ b/thumbnailing/preview/readSeeker.go @@ -0,0 +1,42 @@ +package preview + +import "io" + +type readSeekerWrapper struct { + data []byte + offset int64 +} + +func (r *readSeekerWrapper) Read(p []byte) (n int, err error) { + if r.offset >= int64(len(r.data)) { + return 0, io.EOF + } + n = copy(p, r.data[r.offset:]) + r.offset += int64(n) + return +} + +func (r *readSeekerWrapper) Seek(offset int64, whence int) (int64, error) { + var absOffset int64 + switch whence { + case io.SeekStart: + absOffset = offset + case io.SeekCurrent: + absOffset = r.offset + offset + case io.SeekEnd: + absOffset = int64(len(r.data)) + offset + } + if absOffset < 0 || absOffset > int64(len(r.data)) { + return 0, io.EOF + } + r.offset = absOffset + return absOffset, nil +} + +func newReadSeekerWrapper(reader io.Reader) (*readSeekerWrapper, error) { + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return &readSeekerWrapper{data: data}, nil +} diff --git a/thumbnailing/i/svg.go b/thumbnailing/preview/svg.go similarity index 65% rename from thumbnailing/i/svg.go rename to thumbnailing/preview/svg.go index fe232a36..1d3aff16 100644 --- a/thumbnailing/i/svg.go +++ b/thumbnailing/preview/svg.go @@ -1,18 +1,16 @@ -package i +package preview import ( - "errors" + "fmt" "io" "os" "os/exec" "path" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" ) -type svgGenerator struct { -} +type svgGenerator struct{} func (d svgGenerator) supportedContentTypes() []string { return []string{"image/svg+xml"} @@ -30,10 +28,10 @@ func (d svgGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { dir, err := os.MkdirTemp(os.TempDir(), "mmr-svg") if err != nil { - return nil, errors.New("svg: error creating temporary directory: " + err.Error()) + return nil, fmt.Errorf("svg: error creating temporary directory: %w", err) } tempFile1 := path.Join(dir, "i.svg") @@ -43,22 +41,22 @@ func (d svgGenerator) GenerateThumbnail(b io.Reader, contentType string, width i defer os.Remove(tempFile2) defer os.Remove(dir) - f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0640) + f, err := os.OpenFile(tempFile1, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { - return nil, errors.New("svg: error creating temp svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error creating temp svg file: %w", err) } if _, err = io.Copy(f, b); err != nil { - return nil, errors.New("svg: error writing temp svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error writing temp svg file: %w", err) } err = exec.Command("convert", tempFile1, tempFile2).Run() if err != nil { - return nil, errors.New("svg: error converting svg file: " + err.Error()) + return nil, fmt.Errorf("svg: error converting svg file: %w", err) } - f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0640) + f, err = os.OpenFile(tempFile2, os.O_RDONLY, 0o640) if err != nil { - return nil, errors.New("svg: error reading temp png file: " + err.Error()) + return nil, fmt.Errorf("svg: error reading temp png file: %w", err) } defer f.Close() diff --git a/thumbnailing/i/tiff.go b/thumbnailing/preview/tiff.go similarity index 80% rename from thumbnailing/i/tiff.go rename to thumbnailing/preview/tiff.go index ddd511f7..d77e02fa 100644 --- a/thumbnailing/i/tiff.go +++ b/thumbnailing/preview/tiff.go @@ -1,16 +1,14 @@ -package i +package preview import ( - "errors" + "fmt" "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "golang.org/x/image/tiff" ) -type tiffGenerator struct { -} +type tiffGenerator struct{} func (d tiffGenerator) supportedContentTypes() []string { return []string{"image/tiff"} @@ -32,10 +30,10 @@ func (d tiffGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d tiffGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d tiffGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := tiff.Decode(b) if err != nil { - return nil, errors.New("tiff: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("tiff: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/m/audio_info.go b/thumbnailing/preview/types.go similarity index 54% rename from thumbnailing/m/audio_info.go rename to thumbnailing/preview/types.go index a194cd4d..8f11bda8 100644 --- a/thumbnailing/m/audio_info.go +++ b/thumbnailing/preview/types.go @@ -1,6 +1,7 @@ -package m +package preview import ( + "io" "time" ) @@ -10,3 +11,9 @@ type AudioInfo struct { TotalSamples int Channels int } + +type Thumbnail struct { + Animated bool + ContentType string + Reader io.ReadCloser +} diff --git a/thumbnailing/i/wav.go b/thumbnailing/preview/wav.go similarity index 72% rename from thumbnailing/i/wav.go rename to thumbnailing/preview/wav.go index a1e497d7..5f1cdff8 100644 --- a/thumbnailing/i/wav.go +++ b/thumbnailing/preview/wav.go @@ -1,18 +1,16 @@ -package i +package preview import ( - "errors" + "fmt" "io" + "github.com/dhowden/tag" "github.com/faiface/beep" "github.com/faiface/beep/wav" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" ) -type wavGenerator struct { -} +type wavGenerator struct{} func (d wavGenerator) supportedContentTypes() []string { return []string{"audio/wav"} @@ -29,7 +27,7 @@ func (d wavGenerator) matches(img io.Reader, contentType string) bool { func (d wavGenerator) decode(b io.Reader) (beep.StreamSeekCloser, beep.Format, error) { audio, format, err := wav.Decode(b) if err != nil { - return audio, format, errors.New("wav: error decoding audio: " + err.Error()) + return audio, format, fmt.Errorf("wav: error decoding audio: %w", err) } return audio, format, nil } @@ -38,15 +36,14 @@ func (d wavGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx r return false, 0, 0, nil } -func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { - tags, rc, err := u.GetID3Tags(b) +func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { + rd, err := newReadSeekerWrapper(b) + tags, err := tag.ReadFrom(rd) // we don't care about errors in this process if err != nil { - return nil, errors.New("wav: error getting tags: " + err.Error()) + return nil, fmt.Errorf("wav: error getting tags: %w", err) } - //goland:noinspection GoUnhandledErrorResult - defer rc.Close() - audio, format, err := d.decode(rc) + audio, format, err := d.decode(rd) if err != nil { return nil, err } @@ -56,7 +53,7 @@ func (d wavGenerator) GenerateThumbnail(b io.Reader, contentType string, width i return mp3Generator{}.GenerateFromStream(audio, format, tags, width, height, ctx) } -func (d wavGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*m.AudioInfo, error) { +func (d wavGenerator) GetAudioData(b io.Reader, nKeys int, ctx rcontext.RequestContext) (*AudioInfo, error) { audio, format, err := d.decode(b) if err != nil { return nil, err diff --git a/thumbnailing/i/webp.go b/thumbnailing/preview/webp.go similarity index 80% rename from thumbnailing/i/webp.go rename to thumbnailing/preview/webp.go index d36b451f..14f9d22d 100644 --- a/thumbnailing/i/webp.go +++ b/thumbnailing/preview/webp.go @@ -1,16 +1,14 @@ -package i +package preview import ( - "errors" + "fmt" "io" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" "golang.org/x/image/webp" ) -type webpGenerator struct { -} +type webpGenerator struct{} func (d webpGenerator) supportedContentTypes() []string { return []string{"image/webp"} @@ -32,10 +30,10 @@ func (d webpGenerator) GetOriginDimensions(b io.Reader, contentType string, ctx return true, i.Width, i.Height, nil } -func (d webpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func (d webpGenerator) GenerateThumbnail(b io.Reader, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*Thumbnail, error) { src, err := webp.Decode(b) if err != nil { - return nil, errors.New("webp: error decoding thumbnail: " + err.Error()) + return nil, fmt.Errorf("webp: error decoding thumbnail: %w", err) } return pngGenerator{}.GenerateThumbnailOf(src, width, height, method, ctx) diff --git a/thumbnailing/thumbnail.go b/thumbnailing/thumbnail.go index 2323aa06..15639236 100644 --- a/thumbnailing/thumbnail.go +++ b/thumbnailing/thumbnail.go @@ -2,36 +2,36 @@ package thumbnailing import ( "errors" + "fmt" "io" "reflect" + "slices" "github.com/t2bot/matrix-media-repo/common" "github.com/t2bot/matrix-media-repo/common/rcontext" - "github.com/t2bot/matrix-media-repo/thumbnailing/i" - "github.com/t2bot/matrix-media-repo/thumbnailing/m" - "github.com/t2bot/matrix-media-repo/thumbnailing/u" - "github.com/t2bot/matrix-media-repo/util" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview" + "github.com/t2bot/matrix-media-repo/thumbnailing/preview/metadata" "github.com/t2bot/matrix-media-repo/util/readers" ) var ErrUnsupported = errors.New("unsupported thumbnail type") func IsSupported(contentType string) bool { - return util.ArrayContains(i.GetSupportedContentTypes(), contentType) + return slices.Contains(preview.GetSupportedContentTypes(), contentType) } -func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*m.Thumbnail, error) { +func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, height int, method string, animated bool, ctx rcontext.RequestContext) (*preview.Thumbnail, error) { defer imgStream.Close() if !IsSupported(contentType) { ctx.Log.Debugf("Unsupported content type '%s'", contentType) return nil, ErrUnsupported } - if !util.ArrayContains(ctx.Config.Thumbnails.Types, contentType) { + if !slices.Contains(ctx.Config.Thumbnails.Types, contentType) { ctx.Log.Debugf("Disabled content type '%s'", contentType) return nil, ErrUnsupported } - generator, reconstructed := i.GetGenerator(imgStream, contentType, animated) + generator, reconstructed := preview.GetGenerator(imgStream, contentType, animated) if generator == nil { ctx.Log.Debugf("Unsupported thumbnail type at generator for '%s'", contentType) return nil, ErrUnsupported @@ -43,7 +43,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h buffered := readers.NewBufferReadsReader(reconstructed) dimensional, w, h, err := generator.GetOriginDimensions(buffered, contentType, ctx) if err != nil { - return nil, errors.New("error getting dimensions: " + err.Error()) + return nil, fmt.Errorf("error getting dimensions: %w", err) } if dimensional { if (w * h) >= ctx.Config.Thumbnails.MaxPixels { @@ -53,7 +53,7 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h // While we're here, check to ensure we're not about to produce a thumbnail which is larger than the source material shouldThumbnail := true - shouldThumbnail, width, height, method = u.AdjustProperties(w, h, width, height, animated, method) + shouldThumbnail, width, height, method = metadata.AdjustProperties(w, h, width, height, animated, method) if !shouldThumbnail { return nil, common.ErrMediaDimensionsTooSmall } @@ -62,8 +62,8 @@ func GenerateThumbnail(imgStream io.ReadCloser, contentType string, width int, h return generator.GenerateThumbnail(buffered.GetRewoundReader(), contentType, width, height, method, animated, ctx) } -func GetGenerator(imgStream io.Reader, contentType string, animated bool) (i.Generator, io.Reader, error) { - generator, reconstructed := i.GetGenerator(imgStream, contentType, animated) +func GetGenerator(imgStream io.Reader, contentType string, animated bool) (preview.Generator, io.Reader, error) { + generator, reconstructed := preview.GetGenerator(imgStream, contentType, animated) if generator == nil { return nil, reconstructed, ErrUnsupported } diff --git a/thumbnailing/u/metadata.go b/thumbnailing/u/metadata.go deleted file mode 100644 index 5607ded4..00000000 --- a/thumbnailing/u/metadata.go +++ /dev/null @@ -1,52 +0,0 @@ -package u - -import ( - "errors" - "io" - "os" - - "github.com/dhowden/tag" - "github.com/getsentry/sentry-go" - "github.com/sirupsen/logrus" - "github.com/t2bot/matrix-media-repo/util/readers" -) - -func GetID3Tags(b io.Reader) (tag.Metadata, io.ReadSeekCloser, error) { - var f *os.File - var err error - - tryCleanup := func() { - if f != nil { - if err = os.Remove(f.Name()); err != nil && !os.IsNotExist(err) { - logrus.Warnf("Error deleting temp file '%s': %s", f.Name(), err.Error()) - sentry.CaptureException(errors.New("id3: error deleting temp file: " + err.Error())) - } - } - } - - f, err = os.CreateTemp(os.TempDir(), "mmr-id3") - if err != nil { - tryCleanup() - return nil, nil, err - } - if _, err = io.Copy(f, b); err != nil { - tryCleanup() - return nil, nil, err - } - if err = f.Close(); err != nil { - tryCleanup() - return nil, nil, err - } - if f, err = os.OpenFile(f.Name(), os.O_WRONLY, 0644); err != nil { - tryCleanup() - return nil, nil, err - } - - meta, _ := tag.ReadFrom(f) // we don't care about errors in this process - if _, err = f.Seek(0, io.SeekStart); err != nil { - tryCleanup() - return nil, nil, err - } - - return meta, readers.NewTempFileCloser("", f.Name(), f), nil -} diff --git a/util/arrays.go b/util/arrays.go deleted file mode 100644 index 4c9255bb..00000000 --- a/util/arrays.go +++ /dev/null @@ -1,11 +0,0 @@ -package util - -func ArrayContains(a []string, v string) bool { - for _, e := range a { - if e == v { - return true - } - } - - return false -} diff --git a/util/mime.go b/util/mime.go index 8d34d03f..1f6c4743 100644 --- a/util/mime.go +++ b/util/mime.go @@ -2,6 +2,7 @@ package util import ( "mime" + "slices" "strings" ) @@ -19,7 +20,7 @@ func ExtensionForContentType(ct string) string { func CanInline(ct string) bool { ct = FixContentType(ct) - return ArrayContains(InlineContentTypes, ct) + return slices.Contains(InlineContentTypes, ct) } var InlineContentTypes = []string{ diff --git a/util/strings.go b/util/strings.go index 9ecc3ead..f7e5d46f 100644 --- a/util/strings.go +++ b/util/strings.go @@ -5,8 +5,8 @@ import ( ) func HasAnyPrefix(val string, prefixes []string) bool { - for _, p := range prefixes { - if strings.HasPrefix(val, p) { + for _, prefix := range prefixes { + if strings.HasPrefix(val, prefix) { return true } } diff --git a/util/time.go b/util/time.go deleted file mode 100644 index 2b8d5a30..00000000 --- a/util/time.go +++ /dev/null @@ -1,36 +0,0 @@ -package util - -import ( - "strconv" - "time" -) - -func NowMillis() int64 { - return time.Now().UnixNano() / 1000000 -} - -func FromMillis(m int64) time.Time { - return time.Unix(0, m*int64(time.Millisecond)) -} - -func CalcBlockForDuration(timeoutMs string) (time.Duration, error) { - blockFor := 20 * time.Second - if timeoutMs != "" { - parsed, err := strconv.Atoi(timeoutMs) - if err != nil { - return 0, err - } - if parsed > 0 { - // Limit to 60 seconds - if parsed > 60000 { - parsed = 60000 - } - blockFor = time.Duration(parsed) * time.Millisecond - } - } - return blockFor, nil -} - -func GetHourBucket(ts int64) int64 { - return (ts / 3600000) * 3600000 -}