From 16cc83371711c13fb47048a43b65c1e3a20aa644 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Sun, 22 Dec 2024 20:19:26 -0700 Subject: [PATCH 1/8] Add udp_port to example config --- gbans_example.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gbans_example.yml b/gbans_example.yml index 8c89e0d6a..3a2b91a68 100644 --- a/gbans_example.yml +++ b/gbans_example.yml @@ -27,6 +27,9 @@ external_url: "https://example.com" # # client_timeout: 10 +# UDP Port to receive srcds log messages (logaddress_add) +udp_port: 27715 + # Encryption key for JWT. https://numbergenerator.org/random-64-digit-hex-codes-generator is a good option. # # Minimum length of 10 characters From 89a79a94d2545a64b6dfdde5c8ba35aae4745c8f Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Mon, 23 Dec 2024 00:28:11 -0700 Subject: [PATCH 2/8] Remove existing demo_parser pkg. Minor refactors. --- frontend/src/routeTree.gen.ts | 72 +++++++++-- internal/cmd/serve.go | 52 ++++---- internal/demo/demo_usecase.go | 79 ++++++++++-- internal/demo/fetcher.go | 47 ++++--- internal/domain/config.go | 1 + internal/domain/demo.go | 33 +++++ internal/domain/errors.go | 1 + internal/match/match_repository.go | 2 +- internal/network/scp.go | 22 ++-- internal/state/state_usecase.go | 3 +- internal/test/demos_test.go | 6 + pkg/demoparser/demo_parser.go | 191 ----------------------------- pkg/demoparser/demo_parser_test.go | 30 ----- 13 files changed, 227 insertions(+), 312 deletions(-) delete mode 100644 pkg/demoparser/demo_parser.go delete mode 100644 pkg/demoparser/demo_parser_test.go diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 136558289..198956c59 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -1,12 +1,12 @@ -/* prettier-ignore-start */ - /* eslint-disable */ // @ts-nocheck // noinspection JSUnusedGlobalSymbols -// This file is auto-generated by TanStack Router +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. // Import Routes @@ -91,264 +91,316 @@ const AdminRoute = AdminImport.update({ } as any) const GuestIndexRoute = GuestIndexImport.update({ + id: '/', path: '/', getParentRoute: () => GuestRoute, } as any) const GuestWikiRoute = GuestWikiImport.update({ + id: '/wiki', path: '/wiki', getParentRoute: () => GuestRoute, } as any) const GuestStvRoute = GuestStvImport.update({ + id: '/stv', path: '/stv', getParentRoute: () => GuestRoute, } as any) const GuestServersRoute = GuestServersImport.update({ + id: '/servers', path: '/servers', getParentRoute: () => GuestRoute, } as any) const GuestPrivacyPolicyRoute = GuestPrivacyPolicyImport.update({ + id: '/privacy-policy', path: '/privacy-policy', getParentRoute: () => GuestRoute, } as any) const GuestPatreonRoute = GuestPatreonImport.update({ + id: '/patreon', path: '/patreon', getParentRoute: () => GuestRoute, } as any) const GuestContestsRoute = GuestContestsImport.update({ + id: '/contests', path: '/contests', getParentRoute: () => GuestRoute, } as any) const GuestChangelogRoute = GuestChangelogImport.update({ + id: '/changelog', path: '/changelog', getParentRoute: () => GuestRoute, } as any) const AuthStatsRoute = AuthStatsImport.update({ + id: '/stats', path: '/stats', getParentRoute: () => AuthRoute, } as any) const AuthSettingsRoute = AuthSettingsImport.update({ + id: '/settings', path: '/settings', getParentRoute: () => AuthRoute, } as any) const AuthReportRoute = AuthReportImport.update({ + id: '/report', path: '/report', getParentRoute: () => AuthRoute, } as any) const AuthPermissionRoute = AuthPermissionImport.update({ + id: '/permission', path: '/permission', getParentRoute: () => AuthRoute, } as any) const AuthPageNotFoundRoute = AuthPageNotFoundImport.update({ + id: '/page-not-found', path: '/page-not-found', getParentRoute: () => AuthRoute, } as any) const AuthNotificationsRoute = AuthNotificationsImport.update({ + id: '/notifications', path: '/notifications', getParentRoute: () => AuthRoute, } as any) const AuthLogoutRoute = AuthLogoutImport.update({ + id: '/logout', path: '/logout', getParentRoute: () => AuthRoute, } as any) const AuthForumsRoute = AuthForumsImport.update({ + id: '/forums', path: '/forums', getParentRoute: () => AuthRoute, } as any) const AuthChatlogsRoute = AuthChatlogsImport.update({ + id: '/chatlogs', path: '/chatlogs', getParentRoute: () => AuthRoute, } as any) const GuestWikiIndexRoute = GuestWikiIndexImport.update({ + id: '/', path: '/', getParentRoute: () => GuestWikiRoute, } as any) const GuestLoginIndexRoute = GuestLoginIndexImport.update({ + id: '/login/', path: '/login/', getParentRoute: () => GuestRoute, } as any) const AuthStatsIndexRoute = AuthStatsIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthStatsRoute, } as any) const AuthReportIndexRoute = AuthReportIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthReportRoute, } as any) const AuthForumsIndexRoute = AuthForumsIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthForumsRoute, } as any) const ModAdminVotesRoute = ModAdminVotesImport.update({ + id: '/admin/votes', path: '/admin/votes', getParentRoute: () => ModRoute, } as any) const ModAdminReportsRoute = ModAdminReportsImport.update({ + id: '/admin/reports', path: '/admin/reports', getParentRoute: () => ModRoute, } as any) const ModAdminPeopleRoute = ModAdminPeopleImport.update({ + id: '/admin/people', path: '/admin/people', getParentRoute: () => ModRoute, } as any) const ModAdminNewsRoute = ModAdminNewsImport.update({ + id: '/admin/news', path: '/admin/news', getParentRoute: () => ModRoute, } as any) const ModAdminFiltersRoute = ModAdminFiltersImport.update({ + id: '/admin/filters', path: '/admin/filters', getParentRoute: () => ModRoute, } as any) const ModAdminContestsRoute = ModAdminContestsImport.update({ + id: '/admin/contests', path: '/admin/contests', getParentRoute: () => ModRoute, } as any) const ModAdminAppealsRoute = ModAdminAppealsImport.update({ + id: '/admin/appeals', path: '/admin/appeals', getParentRoute: () => ModRoute, } as any) const GuestWikiSlugRoute = GuestWikiSlugImport.update({ + id: '/$slug', path: '/$slug', getParentRoute: () => GuestWikiRoute, } as any) const GuestProfileSteamIdRoute = GuestProfileSteamIdImport.update({ + id: '/profile/$steamId', path: '/profile/$steamId', getParentRoute: () => GuestRoute, } as any) const GuestLoginSuccessRoute = GuestLoginSuccessImport.update({ + id: '/login/success', path: '/login/success', getParentRoute: () => GuestRoute, } as any) const AuthReportReportIdRoute = AuthReportReportIdImport.update({ + id: '/$reportId', path: '/$reportId', getParentRoute: () => AuthReportRoute, } as any) const AuthMatchMatchIdRoute = AuthMatchMatchIdImport.update({ + id: '/match/$matchId', path: '/match/$matchId', getParentRoute: () => AuthRoute, } as any) const AuthForumsForumidRoute = AuthForumsForumidImport.update({ + id: '/$forum_id', path: '/$forum_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthContestsContestidRoute = AuthContestsContestidImport.update({ + id: '/contests/$contest_id', path: '/contests/$contest_id', getParentRoute: () => AuthRoute, } as any) const AuthBanBanidRoute = AuthBanBanidImport.update({ + id: '/ban/$ban_id', path: '/ban/$ban_id', getParentRoute: () => AuthRoute, } as any) const AdminAdminSettingsRoute = AdminAdminSettingsImport.update({ + id: '/admin/settings', path: '/admin/settings', getParentRoute: () => AdminRoute, } as any) const AdminAdminServersRoute = AdminAdminServersImport.update({ + id: '/admin/servers', path: '/admin/servers', getParentRoute: () => AdminRoute, } as any) const AdminAdminGameAdminsRoute = AdminAdminGameAdminsImport.update({ + id: '/admin/game-admins', path: '/admin/game-admins', getParentRoute: () => AdminRoute, } as any) const ModAdminNetworkIndexRoute = ModAdminNetworkIndexImport.update({ + id: '/admin/network/', path: '/admin/network/', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkPlayersbyipRoute = ModAdminNetworkPlayersbyipImport.update( { + id: '/admin/network/playersbyip', path: '/admin/network/playersbyip', getParentRoute: () => ModRoute, } as any, ) const ModAdminNetworkIphistRoute = ModAdminNetworkIphistImport.update({ + id: '/admin/network/iphist', path: '/admin/network/iphist', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkIpInfoRoute = ModAdminNetworkIpInfoImport.update({ + id: '/admin/network/ipInfo', path: '/admin/network/ipInfo', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkCidrblocksRoute = ModAdminNetworkCidrblocksImport.update({ + id: '/admin/network/cidrblocks', path: '/admin/network/cidrblocks', getParentRoute: () => ModRoute, } as any) const ModAdminBanSteamRoute = ModAdminBanSteamImport.update({ + id: '/admin/ban/steam', path: '/admin/ban/steam', getParentRoute: () => ModRoute, } as any) const ModAdminBanGroupRoute = ModAdminBanGroupImport.update({ + id: '/admin/ban/group', path: '/admin/ban/group', getParentRoute: () => ModRoute, } as any) const ModAdminBanCidrRoute = ModAdminBanCidrImport.update({ + id: '/admin/ban/cidr', path: '/admin/ban/cidr', getParentRoute: () => ModRoute, } as any) const ModAdminBanAsnRoute = ModAdminBanAsnImport.update({ + id: '/admin/ban/asn', path: '/admin/ban/asn', getParentRoute: () => ModRoute, } as any) const AuthStatsWeaponWeaponidRoute = AuthStatsWeaponWeaponidImport.update({ + id: '/weapon/$weapon_id', path: '/weapon/$weapon_id', getParentRoute: () => AuthStatsRoute, } as any) const AuthForumsThreadForumthreadidRoute = AuthForumsThreadForumthreadidImport.update({ + id: '/thread/$forum_thread_id', path: '/thread/$forum_thread_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthLogsSteamIdRoute = AuthLogsSteamIdImport.update({ + id: '/logs/$steamId/', path: '/logs/$steamId/', getParentRoute: () => AuthRoute, } as any) @@ -930,7 +982,7 @@ const ModRouteChildren: ModRouteChildren = { const ModRouteWithChildren = ModRoute._addFileChildren(ModRouteChildren) -interface FileRoutesByFullPath { +export interface FileRoutesByFullPath { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/forums': typeof AuthForumsRouteWithChildren @@ -986,7 +1038,7 @@ interface FileRoutesByFullPath { '/admin/network': typeof ModAdminNetworkIndexRoute } -interface FileRoutesByTo { +export interface FileRoutesByTo { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/logout': typeof AuthLogoutRoute @@ -1038,7 +1090,8 @@ interface FileRoutesByTo { '/admin/network': typeof ModAdminNetworkIndexRoute } -interface FileRoutesById { +export interface FileRoutesById { + __root__: typeof rootRoute '/_admin': typeof AdminRouteWithChildren '/_auth': typeof AuthRouteWithChildren '/_guest': typeof GuestRouteWithChildren @@ -1097,7 +1150,7 @@ interface FileRoutesById { '/_mod/admin/network/': typeof ModAdminNetworkIndexRoute } -interface FileRouteTypes { +export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '' @@ -1205,6 +1258,7 @@ interface FileRouteTypes { | '/admin/network/playersbyip' | '/admin/network' id: + | '__root__' | '/_admin' | '/_auth' | '/_guest' @@ -1264,7 +1318,7 @@ interface FileRouteTypes { fileRoutesById: FileRoutesById } -interface RootRouteChildren { +export interface RootRouteChildren { AdminRoute: typeof AdminRouteWithChildren AuthRoute: typeof AuthRouteWithChildren GuestRoute: typeof GuestRouteWithChildren @@ -1282,8 +1336,6 @@ export const routeTree = rootRoute ._addFileChildren(rootRouteChildren) ._addFileTypes() -/* prettier-ignore-end */ - /* ROUTE_MANIFEST_START { "routes": { diff --git a/internal/cmd/serve.go b/internal/cmd/serve.go index 11c946c44..4da1a95b1 100644 --- a/internal/cmd/serve.go +++ b/internal/cmd/serve.go @@ -100,7 +100,7 @@ func firstTimeSetup(ctx context.Context, persons domain.PersonUsecase, news doma func createQueueWorkers(people domain.PersonUsecase, notifications domain.NotificationUsecase, discordUC domain.DiscordUsecase, authRepo domain.AuthRepository, memberships *steamgroup.Memberships, patreonUC domain.PatreonUsecase, bansSteam domain.BanSteamUsecase, bansNet domain.BanNetUsecase, bansASN domain.BanASNUsecase, - configUC domain.ConfigUsecase, fetcher *demo.Fetcher, demos domain.DemoUsecase, reports domain.ReportUsecase, + configUC domain.ConfigUsecase, demos domain.DemoUsecase, reports domain.ReportUsecase, blocklists domain.BlocklistUsecase, discordOAuth domain.DiscordOAuthUsecase, ) *river.Workers { workers := river.NewWorkers() @@ -110,7 +110,6 @@ func createQueueWorkers(people domain.PersonUsecase, notifications domain.Notifi river.AddWorker[steamgroup.MembershipArgs](workers, steamgroup.NewMembershipWorker(memberships)) river.AddWorker[patreon.AuthUpdateArgs](workers, patreon.NewSyncWorker(patreonUC)) river.AddWorker[ban.ExpirationArgs](workers, ban.NewExpirationWorker(bansSteam, bansNet, bansASN, people, notifications, configUC)) - river.AddWorker[demo.FetcherArgs](workers, demo.NewFetcherWorker(fetcher, configUC)) river.AddWorker[demo.CleanupArgs](workers, demo.NewCleanupWorker(demos)) river.AddWorker[report.MetaInfoArgs](workers, report.NewMetaInfoWorker(reports)) river.AddWorker[blocklist.ListUpdaterArgs](workers, blocklist.NewListUpdaterWorker(blocklists)) @@ -150,13 +149,6 @@ func createPeriodicJobs() []*river.PeriodicJob { }, &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( - river.PeriodicInterval(time.Minute*10), - func() (river.JobArgs, *river.InsertOpts) { - return demo.FetcherArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( river.PeriodicInterval(time.Hour*24), func() (river.JobArgs, *river.InsertOpts) { @@ -305,14 +297,14 @@ func serveCmd() *cobra.Command { //nolint:maintidx return err } - assetUsecase := asset.NewAssetUsecase(assetRepository) - serversUsecase := servers.NewServersUsecase(servers.NewServersRepository(dbConn)) - demoUsecase := demo.NewDemoUsecase(domain.BucketDemo, demo.NewDemoRepository(dbConn), assetUsecase, configUsecase, serversUsecase) + assets := asset.NewAssetUsecase(assetRepository) + serversUC := servers.NewServersUsecase(servers.NewServersRepository(dbConn)) + demos := demo.NewDemoUsecase(domain.BucketDemo, demo.NewDemoRepository(dbConn), assets, configUsecase, serversUC) - reportUsecase := report.NewReportUsecase(report.NewReportRepository(dbConn), notificationUsecase, configUsecase, personUsecase, demoUsecase) + reportUsecase := report.NewReportUsecase(report.NewReportRepository(dbConn), notificationUsecase, configUsecase, personUsecase, demos) stateUsecase := state.NewStateUsecase(eventBroadcaster, - state.NewStateRepository(state.NewCollector(serversUsecase)), configUsecase, serversUsecase) + state.NewStateRepository(state.NewCollector(serversUC)), configUsecase, serversUC) banUsecase := ban.NewBanSteamUsecase(ban.NewBanSteamRepository(dbConn, personUsecase, networkUsecase), personUsecase, configUsecase, notificationUsecase, reportUsecase, stateUsecase) @@ -334,10 +326,10 @@ func serveCmd() *cobra.Command { //nolint:maintidx appeals := appeal.NewAppealUsecase(appeal.NewAppealRepository(dbConn), banUsecase, personUsecase, notificationUsecase, configUsecase) - matchRepo := match.NewMatchRepository(eventBroadcaster, dbConn, personUsecase, serversUsecase, notificationUsecase, stateUsecase, weaponsMap) + matchRepo := match.NewMatchRepository(eventBroadcaster, dbConn, personUsecase, serversUC, notificationUsecase, stateUsecase, weaponsMap) go matchRepo.Start(ctx) - matchUsecase := match.NewMatchUsecase(matchRepo, stateUsecase, serversUsecase, notificationUsecase) + matchUsecase := match.NewMatchUsecase(matchRepo, stateUsecase, serversUC, notificationUsecase) if errWeapons := matchUsecase.LoadWeapons(ctx, weaponsMap); errWeapons != nil { slog.Error("Failed to import weapons", log.ErrAttr(errWeapons)) @@ -360,12 +352,12 @@ func serveCmd() *cobra.Command { //nolint:maintidx patreonUsecase := patreon.NewPatreonUsecase(patreon.NewPatreonRepository(dbConn), configUsecase) - srcdsUsecase := srcds.NewSrcdsUsecase(srcds.NewRepository(dbConn), configUsecase, serversUsecase, personUsecase, reportUsecase, notificationUsecase, banUsecase) + srcdsUsecase := srcds.NewSrcdsUsecase(srcds.NewRepository(dbConn), configUsecase, serversUC, personUsecase, reportUsecase, notificationUsecase, banUsecase) wikiUsecase := wiki.NewWikiUsecase(wiki.NewWikiRepository(dbConn)) authRepo := auth.NewAuthRepository(dbConn) - authUsecase := auth.NewAuthUsecase(authRepo, configUsecase, personUsecase, banUsecase, serversUsecase) + authUsecase := auth.NewAuthUsecase(authRepo, configUsecase, personUsecase, banUsecase, serversUC) voteUsecase := votes.NewVoteUsecase(votes.NewVoteRepository(dbConn), personUsecase, matchUsecase, notificationUsecase, configUsecase, eventBroadcaster) go voteUsecase.Start(ctx) @@ -393,7 +385,7 @@ func serveCmd() *cobra.Command { //nolint:maintidx } discordHandler := discord.NewDiscordHandler(discordUsecase, personUsecase, banUsecase, - stateUsecase, serversUsecase, configUsecase, networkUsecase, wordFilterUsecase, matchUsecase, banNetUsecase, banASNUsecase) + stateUsecase, serversUC, configUsecase, networkUsecase, wordFilterUsecase, matchUsecase, banNetUsecase, banASNUsecase) discordHandler.Start(ctx) appeal.NewAppealHandler(router, appeals, authUsecase) @@ -406,11 +398,11 @@ func serveCmd() *cobra.Command { //nolint:maintidx steamgroup.NewSteamgroupHandler(router, banGroupUsecase, authUsecase) blocklist.NewBlocklistHandler(router, blocklistUsecase, networkUsecase, authUsecase) chat.NewChatHandler(router, chatUsecase, authUsecase) - contest.NewContestHandler(router, contestUsecase, configUsecase, assetUsecase, authUsecase) - demo.NewDemoHandler(router, demoUsecase) + contest.NewContestHandler(router, contestUsecase, configUsecase, assets, authUsecase) + demo.NewDemoHandler(router, demos) forum.NewForumHandler(router, forumUsecase, authUsecase) - match.NewMatchHandler(ctx, router, matchUsecase, serversUsecase, authUsecase, configUsecase) - asset.NewAssetHandler(router, configUsecase, assetUsecase, authUsecase) + match.NewMatchHandler(ctx, router, matchUsecase, serversUC, authUsecase, configUsecase) + asset.NewAssetHandler(router, configUsecase, assets, authUsecase) metrics.NewMetricsHandler(router) network.NewNetworkHandler(router, networkUsecase, authUsecase) news.NewNewsHandler(router, newsUsecase, notificationUsecase, authUsecase) @@ -418,9 +410,9 @@ func serveCmd() *cobra.Command { //nolint:maintidx patreon.NewPatreonHandler(router, patreonUsecase, authUsecase, configUsecase) person.NewPersonHandler(router, configUsecase, personUsecase, authUsecase) report.NewReportHandler(router, reportUsecase, authUsecase, notificationUsecase) - servers.NewServerHandler(router, serversUsecase, stateUsecase, authUsecase, personUsecase) - srcds.NewSRCDSHandler(router, srcdsUsecase, serversUsecase, personUsecase, assetUsecase, - reportUsecase, banUsecase, networkUsecase, banGroupUsecase, demoUsecase, authUsecase, banASNUsecase, banNetUsecase, + servers.NewServerHandler(router, serversUC, stateUsecase, authUsecase, personUsecase) + srcds.NewSRCDSHandler(router, srcdsUsecase, serversUC, personUsecase, assets, + reportUsecase, banUsecase, networkUsecase, banGroupUsecase, demos, authUsecase, banASNUsecase, banNetUsecase, configUsecase, notificationUsecase, stateUsecase, blocklistUsecase) votes.NewVoteHandler(router, voteUsecase, authUsecase) wiki.NewWIkiHandler(router, wikiUsecase, authUsecase) @@ -430,8 +422,6 @@ func serveCmd() *cobra.Command { //nolint:maintidx go stateUsecase.LogAddressAdd(ctx, conf.Debug.AddRCONLogAddress) } - demoFetcher := demo.NewFetcher(dbConn, configUsecase, serversUsecase, assetUsecase, demoUsecase) - // River Queue workers := createQueueWorkers( personUsecase, @@ -444,8 +434,7 @@ func serveCmd() *cobra.Command { //nolint:maintidx banNetUsecase, banASNUsecase, configUsecase, - demoFetcher, - demoUsecase, + demos, reportUsecase, blocklistUsecase, discordOAuthUsecase) @@ -468,6 +457,9 @@ func serveCmd() *cobra.Command { //nolint:maintidx httpServer := httphelper.NewHTTPServer(conf.Addr(), router) + demoDownloader := demo.NewDownloader(configUsecase, dbConn, serversUC, assets, demos) + go demoDownloader.Start(ctx) + go func() { <-ctx.Done() diff --git a/internal/demo/demo_usecase.go b/internal/demo/demo_usecase.go index b7b55a571..82daee8b7 100644 --- a/internal/demo/demo_usecase.go +++ b/internal/demo/demo_usecase.go @@ -1,10 +1,16 @@ package demo import ( + "bytes" "context" + "encoding/json" "errors" "fmt" + "io" "log/slog" + "mime/multipart" + "net/http" + "os" "strings" "time" @@ -12,7 +18,6 @@ import ( "github.com/gin-gonic/gin" "github.com/leighmacdonald/gbans/internal/domain" "github.com/leighmacdonald/gbans/internal/queue" - "github.com/leighmacdonald/gbans/pkg/demoparser" "github.com/leighmacdonald/gbans/pkg/fs" "github.com/leighmacdonald/gbans/pkg/log" "github.com/ricochet2200/go-disk-usage/du" @@ -202,6 +207,60 @@ func (d demoUsecase) GetDemos(ctx context.Context) ([]domain.DemoFile, error) { return d.repository.GetDemos(ctx) } +func (d demoUsecase) SendAndParseDemo(ctx context.Context, path string) (*domain.DemoDetails, error) { + df, errDF := os.Open(path) + if errDF != nil { + return nil, errors.Join(errDF, domain.ErrDemoLoad) + } + + content, errContent := io.ReadAll(df) + if errContent != nil { + return nil, errors.Join(errDF, domain.ErrDemoLoad) + } + + info, errInfo := df.Stat() + if errInfo != nil { + return nil, errors.Join(errInfo, domain.ErrDemoLoad) + } + + log.Closer(df) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + + part, errCreate := writer.CreateFormFile("file", info.Name()) + if errCreate != nil { + return nil, errors.Join(errCreate, domain.ErrDemoLoad) + } + + if _, err := part.Write(content); err != nil { + return nil, errors.Join(errCreate, domain.ErrDemoLoad) + } + + if errClose := writer.Close(); errClose != nil { + return nil, errors.Join(errClose, domain.ErrDemoLoad) + } + + req, errReq := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8811/", body) + if errReq != nil { + return nil, errors.Join(errReq, domain.ErrDemoLoad) + } + + client := &http.Client{} + resp, errSend := client.Do(req) + if errSend != nil { + return nil, errors.Join(errSend, domain.ErrDemoLoad) + } + + var demo domain.DemoDetails + + if errDecode := json.NewDecoder(resp.Body).Decode(&demo); errDecode != nil { + return nil, errors.Join(errDecode, domain.ErrDemoLoad) + } + + return &demo, nil +} + func (d demoUsecase) CreateFromAsset(ctx context.Context, asset domain.Asset, serverID int) (*domain.DemoFile, error) { _, errGetServer := d.servers.Server(ctx, serverID) if errGetServer != nil { @@ -221,20 +280,16 @@ func (d demoUsecase) CreateFromAsset(ctx context.Context, asset domain.Asset, se mapName = nameParts[0] } + // TODO change this data shape as we have not needed this in a long time. Only keys the are used. intStats := map[string]gin.H{} - // temp thing until proper demo parsing is implemented - if d.config.Config().General.Mode != domain.TestMode { - var demoInfo demoparser.DemoInfo - if errParse := demoparser.Parse(ctx, asset.LocalPath, &demoInfo); errParse != nil { - return nil, errParse - } + demoDetail, errDetail := d.SendAndParseDemo(ctx, asset.LocalPath) + if errDetail != nil { + return nil, errDetail + } - for _, steamID := range demoInfo.SteamIDs() { - intStats[steamID.String()] = gin.H{} - } - } else { - intStats[d.config.Config().Owner] = gin.H{} + for key := range demoDetail.State.Users { + intStats[key] = gin.H{} } timeStr := fmt.Sprintf("%s-%s", namePartsAll[0], namePartsAll[1]) diff --git a/internal/demo/fetcher.go b/internal/demo/fetcher.go index 79e0d9645..dbbc17e5b 100644 --- a/internal/demo/fetcher.go +++ b/internal/demo/fetcher.go @@ -15,10 +15,8 @@ import ( "github.com/leighmacdonald/gbans/internal/database" "github.com/leighmacdonald/gbans/internal/domain" "github.com/leighmacdonald/gbans/internal/network" - "github.com/leighmacdonald/gbans/internal/queue" "github.com/leighmacdonald/gbans/pkg/log" "github.com/leighmacdonald/steamid/v4/steamid" - "github.com/riverqueue/river" "github.com/viant/afs/option" "github.com/viant/afs/storage" ) @@ -145,39 +143,36 @@ func (d Fetcher) OnClientConnect(ctx context.Context, client storage.Storager, s return nil } -type FetcherArgs struct{} +func NewDownloader(config domain.ConfigUsecase, dbConn database.Database, servers domain.ServersUsecase, assets domain.AssetUsecase, demos domain.DemoUsecase) Downloader { + fetcher := NewFetcher(dbConn, config, servers, assets, demos) -func (args FetcherArgs) Kind() string { - return "demo_fetch" -} - -func (args FetcherArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Demo), UniqueOpts: river.UniqueOpts{ByPeriod: time.Minute * 10}} -} - -func NewFetcherWorker(fetcher *Fetcher, config domain.ConfigUsecase) *FetcherWorker { - return &FetcherWorker{ - scpExec: network.NewSCPExecer(fetcher.database, fetcher.configUsecase, fetcher.serversUsecase, fetcher.OnClientConnect), + return Downloader{ + fetcher: fetcher, + scpExec: network.NewSCPExecer(dbConn, config, servers, fetcher.OnClientConnect), config: config, } } -type FetcherWorker struct { - river.WorkerDefaults[FetcherArgs] +type Downloader struct { + fetcher *Fetcher scpExec network.SCPExecer config domain.ConfigUsecase } -func (worker *FetcherWorker) Work(ctx context.Context, _ *river.Job[FetcherArgs]) error { - if !worker.config.Config().SSH.Enabled { - return nil - } - - if err := worker.scpExec.Update(ctx); err != nil { - slog.Error("Failed to execute demo fetcher", log.ErrAttr(err)) +func (d Downloader) Start(ctx context.Context) { + ticker := time.NewTicker(time.Second * 5) + for { + select { + case <-ticker.C: + if !d.config.Config().SSH.Enabled { + continue + } - return err + if err := d.scpExec.Update(ctx); err != nil { + slog.Error("Error trying to download demos", log.ErrAttr(err)) + } + case <-ctx.Done(): + return + } } - - return nil } diff --git a/internal/domain/config.go b/internal/domain/config.go index 73a93ab3d..aa8ae54c7 100644 --- a/internal/domain/config.go +++ b/internal/domain/config.go @@ -102,6 +102,7 @@ type ConfigSSH struct { UpdateInterval int `json:"update_interval,string"` Timeout int `json:"timeout,string"` DemoPathFmt string `json:"demo_path_fmt"` + // TODO configurable handling of host keys } type ConfigExports struct { diff --git a/internal/domain/demo.go b/internal/domain/demo.go index 086cf4097..0ad9158ef 100644 --- a/internal/domain/demo.go +++ b/internal/domain/demo.go @@ -16,6 +16,7 @@ type DemoUsecase interface { GetDemos(ctx context.Context) ([]DemoFile, error) CreateFromAsset(ctx context.Context, asset Asset, serverID int) (*DemoFile, error) Cleanup(ctx context.Context) + SendAndParseDemo(ctx context.Context, path string) (*DemoDetails, error) } type DemoRepository interface { @@ -58,3 +59,35 @@ type DemoInfo struct { Title string AssetID uuid.UUID } + +type DemoPlayer struct { + Classes struct { + } `json:"classes"` + Name string `json:"name"` + UserID int `json:"userId"` + SteamID string `json:"steamId"` + Team string `json:"team"` +} + +type DemoHeader struct { + DemoType string `json:"demo_type"` + Version int `json:"version"` + Protocol int `json:"protocol"` + Server string `json:"server"` + Nick string `json:"nick"` + Map string `json:"map"` + Game string `json:"game"` + Duration float64 `json:"duration"` + Ticks int `json:"ticks"` + Frames int `json:"frames"` + Signon int `json:"signon"` +} + +type DemoDetails struct { + State struct { + PlayerSummaries struct { + } `json:"player_summaries"` + Users map[string]DemoPlayer `json:"users"` + } `json:"state"` + Header DemoHeader `json:"header"` +} diff --git a/internal/domain/errors.go b/internal/domain/errors.go index ccb201d00..1f207b8a2 100644 --- a/internal/domain/errors.go +++ b/internal/domain/errors.go @@ -166,4 +166,5 @@ var ( ErrOpenFile = errors.New("could not open output file") ErrFrontendRoutes = errors.New("failed to initialize frontend asset routes") ErrPathInvalid = errors.New("invalid path specified") + ErrDemoLoad = errors.New("could not load demo file") ) diff --git a/internal/match/match_repository.go b/internal/match/match_repository.go index fb5f65309..a90486d45 100644 --- a/internal/match/match_repository.go +++ b/internal/match/match_repository.go @@ -1333,7 +1333,7 @@ func (r *matchRepository) HealersOverallByHealing(ctx context.Context, count int coalesce(c.assists, 0) as assists, coalesce(c.kills, 0) + coalesce(c.assists, 0) as ka, coalesce(c.deaths, 0) as deaths, - case c.playtime WHEN 0 THEN 0 ELSE h.healing::float / (c.playtime::float / 60) END as hpm, + case c.playtime WHEN 0 THEN 0 ELSE coalesce(h.healing::float / (c.playtime::float / 60), 0) END as hpm, case c.deaths WHEN 0 THEN -1 ELSE ((c.assists::float + c.kills::float) / c.deaths::float) END kad, coalesce(c.playtime, 0) as playtime, coalesce(c.dominations, 0) as dominations, diff --git a/internal/network/scp.go b/internal/network/scp.go index 09cf9f7e5..5de5d9e86 100644 --- a/internal/network/scp.go +++ b/internal/network/scp.go @@ -38,23 +38,23 @@ type OnClientConnect func(ctx context.Context, client storage.Storager, server [ // to implement this function and handle any required functionality within it. Caller does not need to close the // connection. type SCPExecer struct { - serversUsecase domain.ServersUsecase - database database.Database - configUsecase domain.ConfigUsecase - onConnect OnClientConnect + servers domain.ServersUsecase + database database.Database + config domain.ConfigUsecase + onConnect OnClientConnect } -func NewSCPExecer(database database.Database, configUsecase domain.ConfigUsecase, serversUsecase domain.ServersUsecase, onConnect OnClientConnect) SCPExecer { +func NewSCPExecer(database database.Database, config domain.ConfigUsecase, servers domain.ServersUsecase, onConnect OnClientConnect) SCPExecer { return SCPExecer{ - database: database, - configUsecase: configUsecase, - serversUsecase: serversUsecase, - onConnect: onConnect, + database: database, + config: config, + servers: servers, + onConnect: onConnect, } } func (f SCPExecer) Update(ctx context.Context) error { - servers, _, errServers := f.serversUsecase.Servers(ctx, domain.ServerQueryFilter{}) + servers, _, errServers := f.servers.Servers(ctx, domain.ServerQueryFilter{}) if errServers != nil { return errServers } @@ -72,7 +72,7 @@ func (f SCPExecer) Update(ctx context.Context) error { mappedServers[server.Address] = append(mappedServers[server.Address], server) } - sshConfig := f.configUsecase.Config().SSH + sshConfig := f.config.Config().SSH waitGroup := &sync.WaitGroup{} diff --git a/internal/state/state_usecase.go b/internal/state/state_usecase.go index 0b206f1fc..8c0f77732 100644 --- a/internal/state/state_usecase.go +++ b/internal/state/state_usecase.go @@ -309,7 +309,8 @@ func (s *stateUsecase) Broadcast(ctx context.Context, serverIDs []int, cmd strin resp, errExec := s.state.ExecRaw(egCtx, serverConf.Addr(), serverConf.RconPassword, cmd) if errExec != nil { - slog.Error("Failed to exec server command", slog.Int("server_id", sid), log.ErrAttr(errExec)) + slog.Error("Failed to exec server command", slog.String("name", serverConf.DefaultHostname), + slog.Int("server_id", sid), log.ErrAttr(errExec)) // Don't error out since we don't want a single servers potentially temporary issue to prevent the rest // from executing. diff --git a/internal/test/demos_test.go b/internal/test/demos_test.go index deffdc163..55c2fdcda 100644 --- a/internal/test/demos_test.go +++ b/internal/test/demos_test.go @@ -52,3 +52,9 @@ func TestDemosCleanup(t *testing.T) { require.NoError(t, err) require.Len(t, allDemos, 5) } + +func TestDemoUpload(t *testing.T) { + detail, err := demoUC.SendAndParseDemo(context.Background(), "test_data/test.dem") + require.NoError(t, err) + require.True(t, len(detail.State.Users) == 10) +} diff --git a/pkg/demoparser/demo_parser.go b/pkg/demoparser/demo_parser.go deleted file mode 100644 index a652d13d2..000000000 --- a/pkg/demoparser/demo_parser.go +++ /dev/null @@ -1,191 +0,0 @@ -// Package demoparser provides a basic wrapper around https://github.com/demostf/parser -// If the binary does not exist, it will be downloaded to the current directory -package demoparser - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "io" - "io/fs" - "log/slog" - "net/http" - "os" - "os/exec" - "path/filepath" - "time" - - "github.com/leighmacdonald/gbans/pkg/log" - "github.com/leighmacdonald/steamid/v4/steamid" -) - -const ( - binPath = "parse_demo" - downloadURL = "https://github.com/demostf/parser/releases/download/v0.4.0/parse_demo" -) - -var ( - ErrDecode = errors.New("failed to decode into parse_demo output json") - ErrCreateRequest = errors.New("failed to create download request") - ErrDownload = errors.New("failed to download parse_demo binary") - ErrOpenFile = errors.New("failed to create new fd") - ErrWrite = errors.New("failed to write binary") - ErrCloseBin = errors.New("failed to close binary file") - ErrCall = errors.New("failed to call parser binary") -) - -//nolint:tagliatelle -type Player struct { - Classes map[string]int `json:"classes"` // class -> count? - Name string `json:"name"` - UserID int `json:"userId"` - SteamID string `json:"steamId"` - Team string `json:"team"` -} - -type Message struct { - Kind string `json:"kind"` - From string `json:"from"` - Text string `json:"text"` - Tick int `json:"tick"` -} - -type Death struct { - Weapon string `json:"weapon"` - Victim int `json:"victim"` - Assister *int `json:"assister"` - Killer int `json:"killer"` - Tick int `json:"tick"` -} - -type Round struct { - Winner string `json:"winner"` - Length float64 `json:"length"` - EndTick int `json:"end_tick"` -} - -//nolint:tagliatelle -type DemoInfo struct { - Chat []Message `json:"chat"` - Users map[string]Player `json:"users"` // userid -> player - Deaths []Death `json:"deaths"` - Rounds []Round `json:"rounds"` - StartTick int `json:"startTick"` - IntervalPerTick float64 `json:"intervalPerTick"` -} - -func (d DemoInfo) SteamIDs() steamid.Collection { - var ids steamid.Collection - - for _, user := range d.Users { - sid64 := steamid.New(user.SteamID) - if !sid64.Valid() { - continue - } - - ids = append(ids, sid64) - } - - return ids -} - -func Parse(ctx context.Context, demoPath string, info *DemoInfo) error { - if errEnsure := ensureBinary(ctx); errEnsure != nil { - return errEnsure - } - - output, errExec := callBin(demoPath) - if errExec != nil { - return errExec - } - - if errDecode := json.NewDecoder(bytes.NewReader(output)).Decode(info); errDecode != nil { - return errors.Join(errDecode, ErrDecode) - } - - return nil -} - -func Exists(path string) bool { - _, err := os.Stat(path) - - return err == nil -} - -func ensureBinary(ctx context.Context) error { - fullPath := fullBinPath() - - if Exists(fullPath) { - return nil - } - - client := http.Client{ - Timeout: time.Second * 60, - } - - req, errReq := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if errReq != nil { - return errors.Join(errReq, ErrCreateRequest) - } - - resp, errResp := client.Do(req) - if errResp != nil { - return errors.Join(errResp, ErrDownload) - } - - defer func() { - if errClose := resp.Body.Close(); errClose != nil { - slog.Error("failed to close response body", log.ErrAttr(errClose)) - } - }() - - openFile, err := os.OpenFile(fullPath, os.O_CREATE|os.O_RDWR|os.O_EXCL, 0x755) - if err != nil { - return errors.Join(err, ErrOpenFile) - } - - defer func() { - if errClose := openFile.Close(); errClose != nil { - slog.Error("failed to close output file", log.ErrAttr(errClose)) - } - }() - - if _, errWrite := io.Copy(openFile, resp.Body); errWrite != nil { - return errors.Join(errWrite, ErrWrite) - } - - return nil -} - -func appDir() string { - dir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - - fullDir := filepath.Join(dir, ".config", "parse_demo") - if errMkdir := os.MkdirAll(fullDir, fs.ModePerm); errMkdir != nil { - panic(errMkdir) - } - - return fullDir -} - -func fullBinPath() string { - return filepath.Join(appDir(), binPath) -} - -func callBin(arg string) ([]byte, error) { - cmd, errOutput := exec.Command(fullBinPath(), arg).Output() //nolint:gosec - if errOutput != nil { - var ee *exec.ExitError - if errors.As(errOutput, &ee) { - return nil, errors.Join(ee, ErrCall) - } - - return nil, errors.Join(errOutput, ErrCall) - } - - return cmd, nil -} diff --git a/pkg/demoparser/demo_parser_test.go b/pkg/demoparser/demo_parser_test.go deleted file mode 100644 index 2ef7ac3a6..000000000 --- a/pkg/demoparser/demo_parser_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package demoparser_test - -import ( - "context" - "path/filepath" - "testing" - - "github.com/leighmacdonald/gbans/pkg/demoparser" - "github.com/stretchr/testify/require" -) - -func TestParse(t *testing.T) { - path, _ := filepath.Abs("testdata/test.dem") - if !demoparser.Exists(path) { - path, _ = filepath.Abs("../../testdata/test.dem") - if !demoparser.Exists(path) { - return - } - } - - var info demoparser.DemoInfo - - require.NoError(t, demoparser.Parse(context.Background(), path, &info)) - require.Len(t, info.Chat, 20) - require.Len(t, info.Deaths, 243) - require.Len(t, info.Rounds, 2) - require.Len(t, info.Users, 45) - require.Equal(t, 509, info.StartTick) - require.InEpsilon(t, 0.015, info.IntervalPerTick, 0.001) -} From 370eb6d09fcb90dab4ab0e360845b641e663e1f0 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Mon, 23 Dec 2024 03:43:09 -0700 Subject: [PATCH 3/8] Basic demo parser service upload functional --- frontend/src/routeTree.gen.ts | 72 +++++------------------------------ internal/demo/demo_usecase.go | 21 +++++++--- internal/domain/demo.go | 16 ++++---- internal/test/demos_test.go | 7 +++- 4 files changed, 37 insertions(+), 79 deletions(-) diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 198956c59..136558289 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -1,12 +1,12 @@ +/* prettier-ignore-start */ + /* eslint-disable */ // @ts-nocheck // noinspection JSUnusedGlobalSymbols -// This file was automatically generated by TanStack Router. -// You should NOT make any changes in this file as it will be overwritten. -// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. +// This file is auto-generated by TanStack Router // Import Routes @@ -91,316 +91,264 @@ const AdminRoute = AdminImport.update({ } as any) const GuestIndexRoute = GuestIndexImport.update({ - id: '/', path: '/', getParentRoute: () => GuestRoute, } as any) const GuestWikiRoute = GuestWikiImport.update({ - id: '/wiki', path: '/wiki', getParentRoute: () => GuestRoute, } as any) const GuestStvRoute = GuestStvImport.update({ - id: '/stv', path: '/stv', getParentRoute: () => GuestRoute, } as any) const GuestServersRoute = GuestServersImport.update({ - id: '/servers', path: '/servers', getParentRoute: () => GuestRoute, } as any) const GuestPrivacyPolicyRoute = GuestPrivacyPolicyImport.update({ - id: '/privacy-policy', path: '/privacy-policy', getParentRoute: () => GuestRoute, } as any) const GuestPatreonRoute = GuestPatreonImport.update({ - id: '/patreon', path: '/patreon', getParentRoute: () => GuestRoute, } as any) const GuestContestsRoute = GuestContestsImport.update({ - id: '/contests', path: '/contests', getParentRoute: () => GuestRoute, } as any) const GuestChangelogRoute = GuestChangelogImport.update({ - id: '/changelog', path: '/changelog', getParentRoute: () => GuestRoute, } as any) const AuthStatsRoute = AuthStatsImport.update({ - id: '/stats', path: '/stats', getParentRoute: () => AuthRoute, } as any) const AuthSettingsRoute = AuthSettingsImport.update({ - id: '/settings', path: '/settings', getParentRoute: () => AuthRoute, } as any) const AuthReportRoute = AuthReportImport.update({ - id: '/report', path: '/report', getParentRoute: () => AuthRoute, } as any) const AuthPermissionRoute = AuthPermissionImport.update({ - id: '/permission', path: '/permission', getParentRoute: () => AuthRoute, } as any) const AuthPageNotFoundRoute = AuthPageNotFoundImport.update({ - id: '/page-not-found', path: '/page-not-found', getParentRoute: () => AuthRoute, } as any) const AuthNotificationsRoute = AuthNotificationsImport.update({ - id: '/notifications', path: '/notifications', getParentRoute: () => AuthRoute, } as any) const AuthLogoutRoute = AuthLogoutImport.update({ - id: '/logout', path: '/logout', getParentRoute: () => AuthRoute, } as any) const AuthForumsRoute = AuthForumsImport.update({ - id: '/forums', path: '/forums', getParentRoute: () => AuthRoute, } as any) const AuthChatlogsRoute = AuthChatlogsImport.update({ - id: '/chatlogs', path: '/chatlogs', getParentRoute: () => AuthRoute, } as any) const GuestWikiIndexRoute = GuestWikiIndexImport.update({ - id: '/', path: '/', getParentRoute: () => GuestWikiRoute, } as any) const GuestLoginIndexRoute = GuestLoginIndexImport.update({ - id: '/login/', path: '/login/', getParentRoute: () => GuestRoute, } as any) const AuthStatsIndexRoute = AuthStatsIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthStatsRoute, } as any) const AuthReportIndexRoute = AuthReportIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthReportRoute, } as any) const AuthForumsIndexRoute = AuthForumsIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthForumsRoute, } as any) const ModAdminVotesRoute = ModAdminVotesImport.update({ - id: '/admin/votes', path: '/admin/votes', getParentRoute: () => ModRoute, } as any) const ModAdminReportsRoute = ModAdminReportsImport.update({ - id: '/admin/reports', path: '/admin/reports', getParentRoute: () => ModRoute, } as any) const ModAdminPeopleRoute = ModAdminPeopleImport.update({ - id: '/admin/people', path: '/admin/people', getParentRoute: () => ModRoute, } as any) const ModAdminNewsRoute = ModAdminNewsImport.update({ - id: '/admin/news', path: '/admin/news', getParentRoute: () => ModRoute, } as any) const ModAdminFiltersRoute = ModAdminFiltersImport.update({ - id: '/admin/filters', path: '/admin/filters', getParentRoute: () => ModRoute, } as any) const ModAdminContestsRoute = ModAdminContestsImport.update({ - id: '/admin/contests', path: '/admin/contests', getParentRoute: () => ModRoute, } as any) const ModAdminAppealsRoute = ModAdminAppealsImport.update({ - id: '/admin/appeals', path: '/admin/appeals', getParentRoute: () => ModRoute, } as any) const GuestWikiSlugRoute = GuestWikiSlugImport.update({ - id: '/$slug', path: '/$slug', getParentRoute: () => GuestWikiRoute, } as any) const GuestProfileSteamIdRoute = GuestProfileSteamIdImport.update({ - id: '/profile/$steamId', path: '/profile/$steamId', getParentRoute: () => GuestRoute, } as any) const GuestLoginSuccessRoute = GuestLoginSuccessImport.update({ - id: '/login/success', path: '/login/success', getParentRoute: () => GuestRoute, } as any) const AuthReportReportIdRoute = AuthReportReportIdImport.update({ - id: '/$reportId', path: '/$reportId', getParentRoute: () => AuthReportRoute, } as any) const AuthMatchMatchIdRoute = AuthMatchMatchIdImport.update({ - id: '/match/$matchId', path: '/match/$matchId', getParentRoute: () => AuthRoute, } as any) const AuthForumsForumidRoute = AuthForumsForumidImport.update({ - id: '/$forum_id', path: '/$forum_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthContestsContestidRoute = AuthContestsContestidImport.update({ - id: '/contests/$contest_id', path: '/contests/$contest_id', getParentRoute: () => AuthRoute, } as any) const AuthBanBanidRoute = AuthBanBanidImport.update({ - id: '/ban/$ban_id', path: '/ban/$ban_id', getParentRoute: () => AuthRoute, } as any) const AdminAdminSettingsRoute = AdminAdminSettingsImport.update({ - id: '/admin/settings', path: '/admin/settings', getParentRoute: () => AdminRoute, } as any) const AdminAdminServersRoute = AdminAdminServersImport.update({ - id: '/admin/servers', path: '/admin/servers', getParentRoute: () => AdminRoute, } as any) const AdminAdminGameAdminsRoute = AdminAdminGameAdminsImport.update({ - id: '/admin/game-admins', path: '/admin/game-admins', getParentRoute: () => AdminRoute, } as any) const ModAdminNetworkIndexRoute = ModAdminNetworkIndexImport.update({ - id: '/admin/network/', path: '/admin/network/', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkPlayersbyipRoute = ModAdminNetworkPlayersbyipImport.update( { - id: '/admin/network/playersbyip', path: '/admin/network/playersbyip', getParentRoute: () => ModRoute, } as any, ) const ModAdminNetworkIphistRoute = ModAdminNetworkIphistImport.update({ - id: '/admin/network/iphist', path: '/admin/network/iphist', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkIpInfoRoute = ModAdminNetworkIpInfoImport.update({ - id: '/admin/network/ipInfo', path: '/admin/network/ipInfo', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkCidrblocksRoute = ModAdminNetworkCidrblocksImport.update({ - id: '/admin/network/cidrblocks', path: '/admin/network/cidrblocks', getParentRoute: () => ModRoute, } as any) const ModAdminBanSteamRoute = ModAdminBanSteamImport.update({ - id: '/admin/ban/steam', path: '/admin/ban/steam', getParentRoute: () => ModRoute, } as any) const ModAdminBanGroupRoute = ModAdminBanGroupImport.update({ - id: '/admin/ban/group', path: '/admin/ban/group', getParentRoute: () => ModRoute, } as any) const ModAdminBanCidrRoute = ModAdminBanCidrImport.update({ - id: '/admin/ban/cidr', path: '/admin/ban/cidr', getParentRoute: () => ModRoute, } as any) const ModAdminBanAsnRoute = ModAdminBanAsnImport.update({ - id: '/admin/ban/asn', path: '/admin/ban/asn', getParentRoute: () => ModRoute, } as any) const AuthStatsWeaponWeaponidRoute = AuthStatsWeaponWeaponidImport.update({ - id: '/weapon/$weapon_id', path: '/weapon/$weapon_id', getParentRoute: () => AuthStatsRoute, } as any) const AuthForumsThreadForumthreadidRoute = AuthForumsThreadForumthreadidImport.update({ - id: '/thread/$forum_thread_id', path: '/thread/$forum_thread_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthLogsSteamIdRoute = AuthLogsSteamIdImport.update({ - id: '/logs/$steamId/', path: '/logs/$steamId/', getParentRoute: () => AuthRoute, } as any) @@ -982,7 +930,7 @@ const ModRouteChildren: ModRouteChildren = { const ModRouteWithChildren = ModRoute._addFileChildren(ModRouteChildren) -export interface FileRoutesByFullPath { +interface FileRoutesByFullPath { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/forums': typeof AuthForumsRouteWithChildren @@ -1038,7 +986,7 @@ export interface FileRoutesByFullPath { '/admin/network': typeof ModAdminNetworkIndexRoute } -export interface FileRoutesByTo { +interface FileRoutesByTo { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/logout': typeof AuthLogoutRoute @@ -1090,8 +1038,7 @@ export interface FileRoutesByTo { '/admin/network': typeof ModAdminNetworkIndexRoute } -export interface FileRoutesById { - __root__: typeof rootRoute +interface FileRoutesById { '/_admin': typeof AdminRouteWithChildren '/_auth': typeof AuthRouteWithChildren '/_guest': typeof GuestRouteWithChildren @@ -1150,7 +1097,7 @@ export interface FileRoutesById { '/_mod/admin/network/': typeof ModAdminNetworkIndexRoute } -export interface FileRouteTypes { +interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '' @@ -1258,7 +1205,6 @@ export interface FileRouteTypes { | '/admin/network/playersbyip' | '/admin/network' id: - | '__root__' | '/_admin' | '/_auth' | '/_guest' @@ -1318,7 +1264,7 @@ export interface FileRouteTypes { fileRoutesById: FileRoutesById } -export interface RootRouteChildren { +interface RootRouteChildren { AdminRoute: typeof AdminRouteWithChildren AuthRoute: typeof AuthRouteWithChildren GuestRoute: typeof GuestRouteWithChildren @@ -1336,6 +1282,8 @@ export const routeTree = rootRoute ._addFileChildren(rootRouteChildren) ._addFileTypes() +/* prettier-ignore-end */ + /* ROUTE_MANIFEST_START { "routes": { diff --git a/internal/demo/demo_usecase.go b/internal/demo/demo_usecase.go index 82daee8b7..4e57d7d9a 100644 --- a/internal/demo/demo_usecase.go +++ b/internal/demo/demo_usecase.go @@ -208,22 +208,22 @@ func (d demoUsecase) GetDemos(ctx context.Context) ([]domain.DemoFile, error) { } func (d demoUsecase) SendAndParseDemo(ctx context.Context, path string) (*domain.DemoDetails, error) { - df, errDF := os.Open(path) + fileHandle, errDF := os.Open(path) if errDF != nil { return nil, errors.Join(errDF, domain.ErrDemoLoad) } - content, errContent := io.ReadAll(df) + content, errContent := io.ReadAll(fileHandle) if errContent != nil { return nil, errors.Join(errDF, domain.ErrDemoLoad) } - info, errInfo := df.Stat() + info, errInfo := fileHandle.Stat() if errInfo != nil { return nil, errors.Join(errInfo, domain.ErrDemoLoad) } - log.Closer(df) + log.Closer(fileHandle) body := new(bytes.Buffer) writer := multipart.NewWriter(body) @@ -241,10 +241,11 @@ func (d demoUsecase) SendAndParseDemo(ctx context.Context, path string) (*domain return nil, errors.Join(errClose, domain.ErrDemoLoad) } - req, errReq := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8811/", body) + req, errReq := http.NewRequestWithContext(ctx, http.MethodPost, "http://localhost:8811/", body) if errReq != nil { return nil, errors.Join(errReq, domain.ErrDemoLoad) } + req.Header.Set("Content-Type", writer.FormDataContentType()) client := &http.Client{} resp, errSend := client.Do(req) @@ -252,9 +253,17 @@ func (d demoUsecase) SendAndParseDemo(ctx context.Context, path string) (*domain return nil, errors.Join(errSend, domain.ErrDemoLoad) } + defer resp.Body.Close() + var demo domain.DemoDetails - if errDecode := json.NewDecoder(resp.Body).Decode(&demo); errDecode != nil { + // TODO remove this extra copy once this feature doesnt have much need for debugging/inspection. + rawBody, errRead := io.ReadAll(resp.Body) + if errRead != nil { + return nil, errors.Join(errRead, domain.ErrDemoLoad) + } + + if errDecode := json.NewDecoder(bytes.NewReader(rawBody)).Decode(&demo); errDecode != nil { return nil, errors.Join(errDecode, domain.ErrDemoLoad) } diff --git a/internal/domain/demo.go b/internal/domain/demo.go index 0ad9158ef..91c3b5589 100644 --- a/internal/domain/demo.go +++ b/internal/domain/demo.go @@ -61,12 +61,11 @@ type DemoInfo struct { } type DemoPlayer struct { - Classes struct { - } `json:"classes"` - Name string `json:"name"` - UserID int `json:"userId"` - SteamID string `json:"steamId"` - Team string `json:"team"` + Classes struct{} `json:"classes"` + Name string `json:"name"` + UserID int `json:"userId"` //nolint:tagliatelle + SteamID string `json:"steamId"` //nolint:tagliatelle + Team string `json:"team"` } type DemoHeader struct { @@ -85,9 +84,8 @@ type DemoHeader struct { type DemoDetails struct { State struct { - PlayerSummaries struct { - } `json:"player_summaries"` - Users map[string]DemoPlayer `json:"users"` + PlayerSummaries struct{} `json:"player_summaries"` + Users map[string]DemoPlayer `json:"users"` } `json:"state"` Header DemoHeader `json:"header"` } diff --git a/internal/test/demos_test.go b/internal/test/demos_test.go index 55c2fdcda..e3f1ee2da 100644 --- a/internal/test/demos_test.go +++ b/internal/test/demos_test.go @@ -5,10 +5,12 @@ import ( "crypto/rand" "fmt" "os" + "path" "testing" "github.com/leighmacdonald/gbans/internal/demo" "github.com/leighmacdonald/gbans/internal/domain" + "github.com/leighmacdonald/gbans/pkg/fs" "github.com/stretchr/testify/require" ) @@ -54,7 +56,8 @@ func TestDemosCleanup(t *testing.T) { } func TestDemoUpload(t *testing.T) { - detail, err := demoUC.SendAndParseDemo(context.Background(), "test_data/test.dem") + demoPath := fs.FindFile(path.Join("testdata", "test.dem"), "gbans") + detail, err := demoUC.SendAndParseDemo(context.Background(), demoPath) require.NoError(t, err) - require.True(t, len(detail.State.Users) == 10) + require.Len(t, len(detail.State.Users), 46) } From c10a30ec6e65fe0e8b3c386517cfa4acc6f2357c Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Mon, 23 Dec 2024 16:01:29 -0700 Subject: [PATCH 4/8] Use require.Len --- internal/test/demos_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test/demos_test.go b/internal/test/demos_test.go index e3f1ee2da..1a5137a24 100644 --- a/internal/test/demos_test.go +++ b/internal/test/demos_test.go @@ -59,5 +59,5 @@ func TestDemoUpload(t *testing.T) { demoPath := fs.FindFile(path.Join("testdata", "test.dem"), "gbans") detail, err := demoUC.SendAndParseDemo(context.Background(), demoPath) require.NoError(t, err) - require.Len(t, len(detail.State.Users), 46) + require.Len(t, detail.State.Users, 46) } From a9502cc2d234f6070e97ad0b3c16ec43a0ad02e8 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Thu, 26 Dec 2024 18:30:16 -0700 Subject: [PATCH 5/8] Adds support for demo_parser_url config --- frontend/src/api/admin.ts | 1 + frontend/src/routeTree.gen.ts | 72 ++++++++++++++++--- frontend/src/routes/_admin.admin.settings.tsx | 19 ++++- internal/config/config_repository.go | 5 +- .../migrations/000102_add_demo_url.down.sql | 6 ++ .../migrations/000102_add_demo_url.up.sql | 5 ++ internal/demo/demo_usecase.go | 2 +- internal/domain/config.go | 1 + 8 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 internal/database/migrations/000102_add_demo_url.down.sql create mode 100644 internal/database/migrations/000102_add_demo_url.up.sql diff --git a/frontend/src/api/admin.ts b/frontend/src/api/admin.ts index c01fb227d..7121ac54d 100644 --- a/frontend/src/api/admin.ts +++ b/frontend/src/api/admin.ts @@ -58,6 +58,7 @@ type Demos = { demo_cleanup_min_pct: string; demo_cleanup_mount: string; demo_count_limit: string; + demo_parser_url: string; }; type Patreon = { diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 136558289..198956c59 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -1,12 +1,12 @@ -/* prettier-ignore-start */ - /* eslint-disable */ // @ts-nocheck // noinspection JSUnusedGlobalSymbols -// This file is auto-generated by TanStack Router +// This file was automatically generated by TanStack Router. +// You should NOT make any changes in this file as it will be overwritten. +// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. // Import Routes @@ -91,264 +91,316 @@ const AdminRoute = AdminImport.update({ } as any) const GuestIndexRoute = GuestIndexImport.update({ + id: '/', path: '/', getParentRoute: () => GuestRoute, } as any) const GuestWikiRoute = GuestWikiImport.update({ + id: '/wiki', path: '/wiki', getParentRoute: () => GuestRoute, } as any) const GuestStvRoute = GuestStvImport.update({ + id: '/stv', path: '/stv', getParentRoute: () => GuestRoute, } as any) const GuestServersRoute = GuestServersImport.update({ + id: '/servers', path: '/servers', getParentRoute: () => GuestRoute, } as any) const GuestPrivacyPolicyRoute = GuestPrivacyPolicyImport.update({ + id: '/privacy-policy', path: '/privacy-policy', getParentRoute: () => GuestRoute, } as any) const GuestPatreonRoute = GuestPatreonImport.update({ + id: '/patreon', path: '/patreon', getParentRoute: () => GuestRoute, } as any) const GuestContestsRoute = GuestContestsImport.update({ + id: '/contests', path: '/contests', getParentRoute: () => GuestRoute, } as any) const GuestChangelogRoute = GuestChangelogImport.update({ + id: '/changelog', path: '/changelog', getParentRoute: () => GuestRoute, } as any) const AuthStatsRoute = AuthStatsImport.update({ + id: '/stats', path: '/stats', getParentRoute: () => AuthRoute, } as any) const AuthSettingsRoute = AuthSettingsImport.update({ + id: '/settings', path: '/settings', getParentRoute: () => AuthRoute, } as any) const AuthReportRoute = AuthReportImport.update({ + id: '/report', path: '/report', getParentRoute: () => AuthRoute, } as any) const AuthPermissionRoute = AuthPermissionImport.update({ + id: '/permission', path: '/permission', getParentRoute: () => AuthRoute, } as any) const AuthPageNotFoundRoute = AuthPageNotFoundImport.update({ + id: '/page-not-found', path: '/page-not-found', getParentRoute: () => AuthRoute, } as any) const AuthNotificationsRoute = AuthNotificationsImport.update({ + id: '/notifications', path: '/notifications', getParentRoute: () => AuthRoute, } as any) const AuthLogoutRoute = AuthLogoutImport.update({ + id: '/logout', path: '/logout', getParentRoute: () => AuthRoute, } as any) const AuthForumsRoute = AuthForumsImport.update({ + id: '/forums', path: '/forums', getParentRoute: () => AuthRoute, } as any) const AuthChatlogsRoute = AuthChatlogsImport.update({ + id: '/chatlogs', path: '/chatlogs', getParentRoute: () => AuthRoute, } as any) const GuestWikiIndexRoute = GuestWikiIndexImport.update({ + id: '/', path: '/', getParentRoute: () => GuestWikiRoute, } as any) const GuestLoginIndexRoute = GuestLoginIndexImport.update({ + id: '/login/', path: '/login/', getParentRoute: () => GuestRoute, } as any) const AuthStatsIndexRoute = AuthStatsIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthStatsRoute, } as any) const AuthReportIndexRoute = AuthReportIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthReportRoute, } as any) const AuthForumsIndexRoute = AuthForumsIndexImport.update({ + id: '/', path: '/', getParentRoute: () => AuthForumsRoute, } as any) const ModAdminVotesRoute = ModAdminVotesImport.update({ + id: '/admin/votes', path: '/admin/votes', getParentRoute: () => ModRoute, } as any) const ModAdminReportsRoute = ModAdminReportsImport.update({ + id: '/admin/reports', path: '/admin/reports', getParentRoute: () => ModRoute, } as any) const ModAdminPeopleRoute = ModAdminPeopleImport.update({ + id: '/admin/people', path: '/admin/people', getParentRoute: () => ModRoute, } as any) const ModAdminNewsRoute = ModAdminNewsImport.update({ + id: '/admin/news', path: '/admin/news', getParentRoute: () => ModRoute, } as any) const ModAdminFiltersRoute = ModAdminFiltersImport.update({ + id: '/admin/filters', path: '/admin/filters', getParentRoute: () => ModRoute, } as any) const ModAdminContestsRoute = ModAdminContestsImport.update({ + id: '/admin/contests', path: '/admin/contests', getParentRoute: () => ModRoute, } as any) const ModAdminAppealsRoute = ModAdminAppealsImport.update({ + id: '/admin/appeals', path: '/admin/appeals', getParentRoute: () => ModRoute, } as any) const GuestWikiSlugRoute = GuestWikiSlugImport.update({ + id: '/$slug', path: '/$slug', getParentRoute: () => GuestWikiRoute, } as any) const GuestProfileSteamIdRoute = GuestProfileSteamIdImport.update({ + id: '/profile/$steamId', path: '/profile/$steamId', getParentRoute: () => GuestRoute, } as any) const GuestLoginSuccessRoute = GuestLoginSuccessImport.update({ + id: '/login/success', path: '/login/success', getParentRoute: () => GuestRoute, } as any) const AuthReportReportIdRoute = AuthReportReportIdImport.update({ + id: '/$reportId', path: '/$reportId', getParentRoute: () => AuthReportRoute, } as any) const AuthMatchMatchIdRoute = AuthMatchMatchIdImport.update({ + id: '/match/$matchId', path: '/match/$matchId', getParentRoute: () => AuthRoute, } as any) const AuthForumsForumidRoute = AuthForumsForumidImport.update({ + id: '/$forum_id', path: '/$forum_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthContestsContestidRoute = AuthContestsContestidImport.update({ + id: '/contests/$contest_id', path: '/contests/$contest_id', getParentRoute: () => AuthRoute, } as any) const AuthBanBanidRoute = AuthBanBanidImport.update({ + id: '/ban/$ban_id', path: '/ban/$ban_id', getParentRoute: () => AuthRoute, } as any) const AdminAdminSettingsRoute = AdminAdminSettingsImport.update({ + id: '/admin/settings', path: '/admin/settings', getParentRoute: () => AdminRoute, } as any) const AdminAdminServersRoute = AdminAdminServersImport.update({ + id: '/admin/servers', path: '/admin/servers', getParentRoute: () => AdminRoute, } as any) const AdminAdminGameAdminsRoute = AdminAdminGameAdminsImport.update({ + id: '/admin/game-admins', path: '/admin/game-admins', getParentRoute: () => AdminRoute, } as any) const ModAdminNetworkIndexRoute = ModAdminNetworkIndexImport.update({ + id: '/admin/network/', path: '/admin/network/', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkPlayersbyipRoute = ModAdminNetworkPlayersbyipImport.update( { + id: '/admin/network/playersbyip', path: '/admin/network/playersbyip', getParentRoute: () => ModRoute, } as any, ) const ModAdminNetworkIphistRoute = ModAdminNetworkIphistImport.update({ + id: '/admin/network/iphist', path: '/admin/network/iphist', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkIpInfoRoute = ModAdminNetworkIpInfoImport.update({ + id: '/admin/network/ipInfo', path: '/admin/network/ipInfo', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkCidrblocksRoute = ModAdminNetworkCidrblocksImport.update({ + id: '/admin/network/cidrblocks', path: '/admin/network/cidrblocks', getParentRoute: () => ModRoute, } as any) const ModAdminBanSteamRoute = ModAdminBanSteamImport.update({ + id: '/admin/ban/steam', path: '/admin/ban/steam', getParentRoute: () => ModRoute, } as any) const ModAdminBanGroupRoute = ModAdminBanGroupImport.update({ + id: '/admin/ban/group', path: '/admin/ban/group', getParentRoute: () => ModRoute, } as any) const ModAdminBanCidrRoute = ModAdminBanCidrImport.update({ + id: '/admin/ban/cidr', path: '/admin/ban/cidr', getParentRoute: () => ModRoute, } as any) const ModAdminBanAsnRoute = ModAdminBanAsnImport.update({ + id: '/admin/ban/asn', path: '/admin/ban/asn', getParentRoute: () => ModRoute, } as any) const AuthStatsWeaponWeaponidRoute = AuthStatsWeaponWeaponidImport.update({ + id: '/weapon/$weapon_id', path: '/weapon/$weapon_id', getParentRoute: () => AuthStatsRoute, } as any) const AuthForumsThreadForumthreadidRoute = AuthForumsThreadForumthreadidImport.update({ + id: '/thread/$forum_thread_id', path: '/thread/$forum_thread_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthLogsSteamIdRoute = AuthLogsSteamIdImport.update({ + id: '/logs/$steamId/', path: '/logs/$steamId/', getParentRoute: () => AuthRoute, } as any) @@ -930,7 +982,7 @@ const ModRouteChildren: ModRouteChildren = { const ModRouteWithChildren = ModRoute._addFileChildren(ModRouteChildren) -interface FileRoutesByFullPath { +export interface FileRoutesByFullPath { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/forums': typeof AuthForumsRouteWithChildren @@ -986,7 +1038,7 @@ interface FileRoutesByFullPath { '/admin/network': typeof ModAdminNetworkIndexRoute } -interface FileRoutesByTo { +export interface FileRoutesByTo { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/logout': typeof AuthLogoutRoute @@ -1038,7 +1090,8 @@ interface FileRoutesByTo { '/admin/network': typeof ModAdminNetworkIndexRoute } -interface FileRoutesById { +export interface FileRoutesById { + __root__: typeof rootRoute '/_admin': typeof AdminRouteWithChildren '/_auth': typeof AuthRouteWithChildren '/_guest': typeof GuestRouteWithChildren @@ -1097,7 +1150,7 @@ interface FileRoutesById { '/_mod/admin/network/': typeof ModAdminNetworkIndexRoute } -interface FileRouteTypes { +export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '' @@ -1205,6 +1258,7 @@ interface FileRouteTypes { | '/admin/network/playersbyip' | '/admin/network' id: + | '__root__' | '/_admin' | '/_auth' | '/_guest' @@ -1264,7 +1318,7 @@ interface FileRouteTypes { fileRoutesById: FileRoutesById } -interface RootRouteChildren { +export interface RootRouteChildren { AdminRoute: typeof AdminRouteWithChildren AuthRoute: typeof AuthRouteWithChildren GuestRoute: typeof GuestRouteWithChildren @@ -1282,8 +1336,6 @@ export const routeTree = rootRoute ._addFileChildren(rootRouteChildren) ._addFileTypes() -/* prettier-ignore-end */ - /* ROUTE_MANIFEST_START { "routes": { diff --git a/frontend/src/routes/_admin.admin.settings.tsx b/frontend/src/routes/_admin.admin.settings.tsx index e8cd42360..e292cf3df 100644 --- a/frontend/src/routes/_admin.admin.settings.tsx +++ b/frontend/src/routes/_admin.admin.settings.tsx @@ -676,7 +676,8 @@ const DemosSection = ({ tab, settings, mutate }: { tab: tabs; settings: Config; demo_cleanup_strategy: settings.demo.demo_cleanup_strategy, demo_cleanup_min_pct: settings.demo.demo_cleanup_min_pct, demo_cleanup_mount: settings.demo.demo_cleanup_mount, - demo_count_limit: settings.demo.demo_count_limit + demo_count_limit: settings.demo.demo_count_limit, + demo_parser_url: settings.demo.demo_parser_url } }); @@ -797,6 +798,22 @@ const DemosSection = ({ tab, settings, mutate }: { tab: tabs; settings: Config; + + { + return ; + }} + /> + + This url should point to an instance of https://github.com/leighmacdonald/tf2_demostats. + This is used to pull stats & player steamids out of demos that are fetched. + + + [state.canSubmit, state.isSubmitting]} diff --git a/internal/config/config_repository.go b/internal/config/config_repository.go index e0ed26bf3..b3faedc44 100644 --- a/internal/config/config_repository.go +++ b/internal/config/config_repository.go @@ -35,7 +35,7 @@ func (c *configRepository) Read(ctx context.Context) (domain.Config, error) { filters_enabled, filters_dry, filters_ping_discord, filters_max_weight, filters_warning_timeout, filters_check_timeout, filters_match_timeout, - demo_cleanup_enabled, demo_cleanup_strategy, demo_cleanup_min_pct, demo_cleanup_mount, demo_count_limit, + demo_cleanup_enabled, demo_cleanup_strategy, demo_cleanup_min_pct, demo_cleanup_mount, demo_count_limit, demo_parser_url, patreon_enabled, patreon_client_id, patreon_client_secret, patreon_creator_access_token, patreon_creator_refresh_token, patreon_integrations_enabled, @@ -69,7 +69,7 @@ func (c *configRepository) Read(ctx context.Context) (domain.Config, error) { &cfg.General.DefaultRoute, &cfg.General.NewsEnabled, &cfg.General.ForumsEnabled, &cfg.General.ContestsEnabled, &cfg.General.WikiEnabled, &cfg.General.StatsEnabled, &cfg.General.ServersEnabled, &cfg.General.ReportsEnabled, &cfg.General.ChatlogsEnabled, &cfg.General.DemosEnabled, &cfg.Filters.Enabled, &cfg.Filters.Dry, &cfg.Filters.PingDiscord, &cfg.Filters.MaxWeight, &cfg.Filters.WarningTimeout, &cfg.Filters.CheckTimeout, &cfg.Filters.MatchTimeout, - &cfg.Demo.DemoCleanupEnabled, &cfg.Demo.DemoCleanupStrategy, &cfg.Demo.DemoCleanupMinPct, &cfg.Demo.DemoCleanupMount, &cfg.Demo.DemoCountLimit, + &cfg.Demo.DemoCleanupEnabled, &cfg.Demo.DemoCleanupStrategy, &cfg.Demo.DemoCleanupMinPct, &cfg.Demo.DemoCleanupMount, &cfg.Demo.DemoCountLimit, &cfg.Demo.DemoParserURL, &cfg.Patreon.Enabled, &cfg.Patreon.ClientID, &cfg.Patreon.ClientSecret, &cfg.Patreon.CreatorAccessToken, &cfg.Patreon.CreatorRefreshToken, &cfg.Patreon.IntegrationsEnabled, &cfg.Discord.Enabled, &cfg.Discord.AppID, &cfg.Discord.AppSecret, &cfg.Discord.LinkID, &cfg.Discord.Token, &cfg.Discord.GuildID, &cfg.Discord.LogChannelID, &cfg.Discord.PublicLogChannelEnable, &cfg.Discord.PublicLogChannelID, &cfg.Discord.PublicMatchLogChannelID, &cfg.Discord.ModPingRoleID, @@ -144,6 +144,7 @@ func (c *configRepository) Write(ctx context.Context, config domain.Config) erro "demo_cleanup_min_pct": config.Demo.DemoCleanupMinPct, "demo_cleanup_mount": config.Demo.DemoCleanupMount, "demo_count_limit": config.Demo.DemoCountLimit, + "demo_parser_url": config.Demo.DemoParserURL, "patreon_enabled": config.Patreon.Enabled, "patreon_integrations_enabled": config.Patreon.IntegrationsEnabled, "patreon_client_id": config.Patreon.ClientID, diff --git a/internal/database/migrations/000102_add_demo_url.down.sql b/internal/database/migrations/000102_add_demo_url.down.sql new file mode 100644 index 000000000..44ab15301 --- /dev/null +++ b/internal/database/migrations/000102_add_demo_url.down.sql @@ -0,0 +1,6 @@ +BEGIN; + +ALTER TABLE config + DROP COLUMN IF EXISTS demo_parser_url; + +COMMIT; diff --git a/internal/database/migrations/000102_add_demo_url.up.sql b/internal/database/migrations/000102_add_demo_url.up.sql new file mode 100644 index 000000000..a4230fe33 --- /dev/null +++ b/internal/database/migrations/000102_add_demo_url.up.sql @@ -0,0 +1,5 @@ +BEGIN; + +ALTER TABLE config ADD COLUMN IF NOT EXISTS demo_parser_url text default 'http://localhost:8811/'; + +COMMIT; diff --git a/internal/demo/demo_usecase.go b/internal/demo/demo_usecase.go index 4e57d7d9a..b5bf17619 100644 --- a/internal/demo/demo_usecase.go +++ b/internal/demo/demo_usecase.go @@ -241,7 +241,7 @@ func (d demoUsecase) SendAndParseDemo(ctx context.Context, path string) (*domain return nil, errors.Join(errClose, domain.ErrDemoLoad) } - req, errReq := http.NewRequestWithContext(ctx, http.MethodPost, "http://localhost:8811/", body) + req, errReq := http.NewRequestWithContext(ctx, http.MethodPost, d.config.Config().Demo.DemoParserURL, body) if errReq != nil { return nil, errors.Join(errReq, domain.ErrDemoLoad) } diff --git a/internal/domain/config.go b/internal/domain/config.go index aa8ae54c7..57a6e9612 100644 --- a/internal/domain/config.go +++ b/internal/domain/config.go @@ -201,6 +201,7 @@ type ConfigDemo struct { DemoCleanupMinPct float32 `json:"demo_cleanup_min_pct,string"` DemoCleanupMount string `json:"demo_cleanup_mount"` DemoCountLimit uint64 `json:"demo_count_limit,string"` + DemoParserURL string `json:"demo_parser_url"` } type ConfigDiscord struct { From f992eaa90a7e55c21c534660f7af73228498d854 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Fri, 27 Dec 2024 16:18:16 -0700 Subject: [PATCH 6/8] Bump tooling versions --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8932ebd0b..40a15d621 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ test-go-cover: install_deps: go install github.com/daixiang0/gci@v0.13.4 go install mvdan.cc/gofumpt@v0.7.0 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2 go install honnef.co/go/tools/cmd/staticcheck@v0.5.1 check: lint_golangci static lint_ts typecheck_ts From bdfecde8b095f498bbac5ef2f13b8c4258f12a65 Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Fri, 27 Dec 2024 17:13:58 -0700 Subject: [PATCH 7/8] Skip test when no parser --- frontend/src/routeTree.gen.ts | 72 +++++------------------------------ internal/test/demos_test.go | 20 ++++++---- 2 files changed, 23 insertions(+), 69 deletions(-) diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index 198956c59..136558289 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -1,12 +1,12 @@ +/* prettier-ignore-start */ + /* eslint-disable */ // @ts-nocheck // noinspection JSUnusedGlobalSymbols -// This file was automatically generated by TanStack Router. -// You should NOT make any changes in this file as it will be overwritten. -// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. +// This file is auto-generated by TanStack Router // Import Routes @@ -91,316 +91,264 @@ const AdminRoute = AdminImport.update({ } as any) const GuestIndexRoute = GuestIndexImport.update({ - id: '/', path: '/', getParentRoute: () => GuestRoute, } as any) const GuestWikiRoute = GuestWikiImport.update({ - id: '/wiki', path: '/wiki', getParentRoute: () => GuestRoute, } as any) const GuestStvRoute = GuestStvImport.update({ - id: '/stv', path: '/stv', getParentRoute: () => GuestRoute, } as any) const GuestServersRoute = GuestServersImport.update({ - id: '/servers', path: '/servers', getParentRoute: () => GuestRoute, } as any) const GuestPrivacyPolicyRoute = GuestPrivacyPolicyImport.update({ - id: '/privacy-policy', path: '/privacy-policy', getParentRoute: () => GuestRoute, } as any) const GuestPatreonRoute = GuestPatreonImport.update({ - id: '/patreon', path: '/patreon', getParentRoute: () => GuestRoute, } as any) const GuestContestsRoute = GuestContestsImport.update({ - id: '/contests', path: '/contests', getParentRoute: () => GuestRoute, } as any) const GuestChangelogRoute = GuestChangelogImport.update({ - id: '/changelog', path: '/changelog', getParentRoute: () => GuestRoute, } as any) const AuthStatsRoute = AuthStatsImport.update({ - id: '/stats', path: '/stats', getParentRoute: () => AuthRoute, } as any) const AuthSettingsRoute = AuthSettingsImport.update({ - id: '/settings', path: '/settings', getParentRoute: () => AuthRoute, } as any) const AuthReportRoute = AuthReportImport.update({ - id: '/report', path: '/report', getParentRoute: () => AuthRoute, } as any) const AuthPermissionRoute = AuthPermissionImport.update({ - id: '/permission', path: '/permission', getParentRoute: () => AuthRoute, } as any) const AuthPageNotFoundRoute = AuthPageNotFoundImport.update({ - id: '/page-not-found', path: '/page-not-found', getParentRoute: () => AuthRoute, } as any) const AuthNotificationsRoute = AuthNotificationsImport.update({ - id: '/notifications', path: '/notifications', getParentRoute: () => AuthRoute, } as any) const AuthLogoutRoute = AuthLogoutImport.update({ - id: '/logout', path: '/logout', getParentRoute: () => AuthRoute, } as any) const AuthForumsRoute = AuthForumsImport.update({ - id: '/forums', path: '/forums', getParentRoute: () => AuthRoute, } as any) const AuthChatlogsRoute = AuthChatlogsImport.update({ - id: '/chatlogs', path: '/chatlogs', getParentRoute: () => AuthRoute, } as any) const GuestWikiIndexRoute = GuestWikiIndexImport.update({ - id: '/', path: '/', getParentRoute: () => GuestWikiRoute, } as any) const GuestLoginIndexRoute = GuestLoginIndexImport.update({ - id: '/login/', path: '/login/', getParentRoute: () => GuestRoute, } as any) const AuthStatsIndexRoute = AuthStatsIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthStatsRoute, } as any) const AuthReportIndexRoute = AuthReportIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthReportRoute, } as any) const AuthForumsIndexRoute = AuthForumsIndexImport.update({ - id: '/', path: '/', getParentRoute: () => AuthForumsRoute, } as any) const ModAdminVotesRoute = ModAdminVotesImport.update({ - id: '/admin/votes', path: '/admin/votes', getParentRoute: () => ModRoute, } as any) const ModAdminReportsRoute = ModAdminReportsImport.update({ - id: '/admin/reports', path: '/admin/reports', getParentRoute: () => ModRoute, } as any) const ModAdminPeopleRoute = ModAdminPeopleImport.update({ - id: '/admin/people', path: '/admin/people', getParentRoute: () => ModRoute, } as any) const ModAdminNewsRoute = ModAdminNewsImport.update({ - id: '/admin/news', path: '/admin/news', getParentRoute: () => ModRoute, } as any) const ModAdminFiltersRoute = ModAdminFiltersImport.update({ - id: '/admin/filters', path: '/admin/filters', getParentRoute: () => ModRoute, } as any) const ModAdminContestsRoute = ModAdminContestsImport.update({ - id: '/admin/contests', path: '/admin/contests', getParentRoute: () => ModRoute, } as any) const ModAdminAppealsRoute = ModAdminAppealsImport.update({ - id: '/admin/appeals', path: '/admin/appeals', getParentRoute: () => ModRoute, } as any) const GuestWikiSlugRoute = GuestWikiSlugImport.update({ - id: '/$slug', path: '/$slug', getParentRoute: () => GuestWikiRoute, } as any) const GuestProfileSteamIdRoute = GuestProfileSteamIdImport.update({ - id: '/profile/$steamId', path: '/profile/$steamId', getParentRoute: () => GuestRoute, } as any) const GuestLoginSuccessRoute = GuestLoginSuccessImport.update({ - id: '/login/success', path: '/login/success', getParentRoute: () => GuestRoute, } as any) const AuthReportReportIdRoute = AuthReportReportIdImport.update({ - id: '/$reportId', path: '/$reportId', getParentRoute: () => AuthReportRoute, } as any) const AuthMatchMatchIdRoute = AuthMatchMatchIdImport.update({ - id: '/match/$matchId', path: '/match/$matchId', getParentRoute: () => AuthRoute, } as any) const AuthForumsForumidRoute = AuthForumsForumidImport.update({ - id: '/$forum_id', path: '/$forum_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthContestsContestidRoute = AuthContestsContestidImport.update({ - id: '/contests/$contest_id', path: '/contests/$contest_id', getParentRoute: () => AuthRoute, } as any) const AuthBanBanidRoute = AuthBanBanidImport.update({ - id: '/ban/$ban_id', path: '/ban/$ban_id', getParentRoute: () => AuthRoute, } as any) const AdminAdminSettingsRoute = AdminAdminSettingsImport.update({ - id: '/admin/settings', path: '/admin/settings', getParentRoute: () => AdminRoute, } as any) const AdminAdminServersRoute = AdminAdminServersImport.update({ - id: '/admin/servers', path: '/admin/servers', getParentRoute: () => AdminRoute, } as any) const AdminAdminGameAdminsRoute = AdminAdminGameAdminsImport.update({ - id: '/admin/game-admins', path: '/admin/game-admins', getParentRoute: () => AdminRoute, } as any) const ModAdminNetworkIndexRoute = ModAdminNetworkIndexImport.update({ - id: '/admin/network/', path: '/admin/network/', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkPlayersbyipRoute = ModAdminNetworkPlayersbyipImport.update( { - id: '/admin/network/playersbyip', path: '/admin/network/playersbyip', getParentRoute: () => ModRoute, } as any, ) const ModAdminNetworkIphistRoute = ModAdminNetworkIphistImport.update({ - id: '/admin/network/iphist', path: '/admin/network/iphist', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkIpInfoRoute = ModAdminNetworkIpInfoImport.update({ - id: '/admin/network/ipInfo', path: '/admin/network/ipInfo', getParentRoute: () => ModRoute, } as any) const ModAdminNetworkCidrblocksRoute = ModAdminNetworkCidrblocksImport.update({ - id: '/admin/network/cidrblocks', path: '/admin/network/cidrblocks', getParentRoute: () => ModRoute, } as any) const ModAdminBanSteamRoute = ModAdminBanSteamImport.update({ - id: '/admin/ban/steam', path: '/admin/ban/steam', getParentRoute: () => ModRoute, } as any) const ModAdminBanGroupRoute = ModAdminBanGroupImport.update({ - id: '/admin/ban/group', path: '/admin/ban/group', getParentRoute: () => ModRoute, } as any) const ModAdminBanCidrRoute = ModAdminBanCidrImport.update({ - id: '/admin/ban/cidr', path: '/admin/ban/cidr', getParentRoute: () => ModRoute, } as any) const ModAdminBanAsnRoute = ModAdminBanAsnImport.update({ - id: '/admin/ban/asn', path: '/admin/ban/asn', getParentRoute: () => ModRoute, } as any) const AuthStatsWeaponWeaponidRoute = AuthStatsWeaponWeaponidImport.update({ - id: '/weapon/$weapon_id', path: '/weapon/$weapon_id', getParentRoute: () => AuthStatsRoute, } as any) const AuthForumsThreadForumthreadidRoute = AuthForumsThreadForumthreadidImport.update({ - id: '/thread/$forum_thread_id', path: '/thread/$forum_thread_id', getParentRoute: () => AuthForumsRoute, } as any) const AuthLogsSteamIdRoute = AuthLogsSteamIdImport.update({ - id: '/logs/$steamId/', path: '/logs/$steamId/', getParentRoute: () => AuthRoute, } as any) @@ -982,7 +930,7 @@ const ModRouteChildren: ModRouteChildren = { const ModRouteWithChildren = ModRoute._addFileChildren(ModRouteChildren) -export interface FileRoutesByFullPath { +interface FileRoutesByFullPath { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/forums': typeof AuthForumsRouteWithChildren @@ -1038,7 +986,7 @@ export interface FileRoutesByFullPath { '/admin/network': typeof ModAdminNetworkIndexRoute } -export interface FileRoutesByTo { +interface FileRoutesByTo { '': typeof ModRouteWithChildren '/chatlogs': typeof AuthChatlogsRoute '/logout': typeof AuthLogoutRoute @@ -1090,8 +1038,7 @@ export interface FileRoutesByTo { '/admin/network': typeof ModAdminNetworkIndexRoute } -export interface FileRoutesById { - __root__: typeof rootRoute +interface FileRoutesById { '/_admin': typeof AdminRouteWithChildren '/_auth': typeof AuthRouteWithChildren '/_guest': typeof GuestRouteWithChildren @@ -1150,7 +1097,7 @@ export interface FileRoutesById { '/_mod/admin/network/': typeof ModAdminNetworkIndexRoute } -export interface FileRouteTypes { +interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '' @@ -1258,7 +1205,6 @@ export interface FileRouteTypes { | '/admin/network/playersbyip' | '/admin/network' id: - | '__root__' | '/_admin' | '/_auth' | '/_guest' @@ -1318,7 +1264,7 @@ export interface FileRouteTypes { fileRoutesById: FileRoutesById } -export interface RootRouteChildren { +interface RootRouteChildren { AdminRoute: typeof AdminRouteWithChildren AuthRoute: typeof AuthRouteWithChildren GuestRoute: typeof GuestRouteWithChildren @@ -1336,6 +1282,8 @@ export const routeTree = rootRoute ._addFileChildren(rootRouteChildren) ._addFileTypes() +/* prettier-ignore-end */ + /* ROUTE_MANIFEST_START { "routes": { diff --git a/internal/test/demos_test.go b/internal/test/demos_test.go index 1a5137a24..bfa616b06 100644 --- a/internal/test/demos_test.go +++ b/internal/test/demos_test.go @@ -34,12 +34,13 @@ func TestDemosCleanup(t *testing.T) { content := make([]byte, 100000) _, err := rand.Read(content) require.NoError(t, err) - - require.NoError(t, fetcher.OnDemoReceived(ctx, demo.UploadedDemo{ - Name: fmt.Sprintf("2023111%d-063943-koth_harvest_final.dem", demoNum), - Server: testServer, - Content: content, - })) + if configUC.Config().Demo.DemoParserURL != "" { + require.NoError(t, fetcher.OnDemoReceived(ctx, demo.UploadedDemo{ + Name: fmt.Sprintf("2023111%d-063943-koth_harvest_final.dem", demoNum), + Server: testServer, + Content: content, + })) + } } expired, errExpired := demoRepository.ExpiredDemos(ctx, 5) @@ -52,10 +53,15 @@ func TestDemosCleanup(t *testing.T) { allDemos, err := demoUC.GetDemos(ctx) require.NoError(t, err) - require.Len(t, allDemos, 5) + if configUC.Config().Demo.DemoParserURL != "" { + require.Len(t, allDemos, 5) + } } func TestDemoUpload(t *testing.T) { + if configUC.Config().Demo.DemoParserURL == "" { + t.Skip("Parser url undefined") + } demoPath := fs.FindFile(path.Join("testdata", "test.dem"), "gbans") detail, err := demoUC.SendAndParseDemo(context.Background(), demoPath) require.NoError(t, err) From fdc0521ddea289fc6db928aa916ea0359a71c60d Mon Sep 17 00:00:00 2001 From: Leigh MacDonald Date: Sat, 28 Dec 2024 00:52:50 -0700 Subject: [PATCH 8/8] Remove more workers --- internal/ban/background.go | 67 +++++++---------- internal/blocklist/blocklist_usecase.go | 37 ++-------- internal/cmd/serve.go | 95 ++++++++++--------------- internal/demo/demo_usecase.go | 27 ------- internal/person/person_usecase.go | 45 ------------ internal/steamgroup/steam_group.go | 29 +------- 6 files changed, 69 insertions(+), 231 deletions(-) diff --git a/internal/ban/background.go b/internal/ban/background.go index 88e24d753..a90fafa1b 100644 --- a/internal/ban/background.go +++ b/internal/ban/background.go @@ -3,59 +3,44 @@ package ban import ( "context" "errors" - "log/slog" - "sync" - "time" - "github.com/leighmacdonald/gbans/internal/discord" "github.com/leighmacdonald/gbans/internal/domain" - "github.com/leighmacdonald/gbans/internal/queue" "github.com/leighmacdonald/gbans/pkg/log" "github.com/leighmacdonald/steamid/v4/steamid" - "github.com/riverqueue/river" + "log/slog" + "sync" ) -type ExpirationArgs struct{} - -func (args ExpirationArgs) Kind() string { - return "bans_expired" -} - -func (args ExpirationArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Default), UniqueOpts: river.UniqueOpts{ByPeriod: time.Minute}} -} - -func NewExpirationWorker(bansSteam domain.BanSteamUsecase, bansNet domain.BanNetUsecase, bansASN domain.BanASNUsecase, - bansPerson domain.PersonUsecase, notifications domain.NotificationUsecase, config domain.ConfigUsecase, -) *ExpirationWorker { - return &ExpirationWorker{ - bansSteam: bansSteam, - bansNet: bansNet, - bansASN: bansASN, - bansPerson: bansPerson, +func NewExpirationMonitor(steam domain.BanSteamUsecase, net domain.BanNetUsecase, asn domain.BanASNUsecase, + person domain.PersonUsecase, notifications domain.NotificationUsecase, config domain.ConfigUsecase, +) *ExpirationMonitor { + return &ExpirationMonitor{ + steam: steam, + net: net, + asn: asn, + person: person, notifications: notifications, config: config, } } -type ExpirationWorker struct { - river.WorkerDefaults[ExpirationArgs] - bansSteam domain.BanSteamUsecase - bansNet domain.BanNetUsecase - bansASN domain.BanASNUsecase - bansPerson domain.PersonUsecase +type ExpirationMonitor struct { + steam domain.BanSteamUsecase + net domain.BanNetUsecase + asn domain.BanASNUsecase + person domain.PersonUsecase notifications domain.NotificationUsecase config domain.ConfigUsecase } -func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[ExpirationArgs]) error { +func (monitor *ExpirationMonitor) Update(ctx context.Context) { waitGroup := &sync.WaitGroup{} waitGroup.Add(3) go func() { defer waitGroup.Done() - expiredBans, errExpiredBans := worker.bansSteam.Expired(ctx) + expiredBans, errExpiredBans := monitor.steam.Expired(ctx) if errExpiredBans != nil && !errors.Is(errExpiredBans, domain.ErrNoResult) { slog.Error("Failed to get expired expiredBans", log.ErrAttr(errExpiredBans)) @@ -64,13 +49,13 @@ func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[Expiratio for _, expiredBan := range expiredBans { ban := expiredBan - if errDrop := worker.bansSteam.Delete(ctx, &ban, false); errDrop != nil { + if errDrop := monitor.steam.Delete(ctx, &ban, false); errDrop != nil { slog.Error("Failed to drop expired expiredBan", log.ErrAttr(errDrop)) continue } - person, errPerson := worker.bansPerson.GetPersonBySteamID(ctx, ban.TargetID) + person, errPerson := monitor.person.GetPersonBySteamID(ctx, ban.TargetID) if errPerson != nil { slog.Error("Failed to get expired Person", log.ErrAttr(errPerson)) @@ -82,9 +67,9 @@ func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[Expiratio name = person.SteamID.String() } - worker.notifications.Enqueue(ctx, domain.NewDiscordNotification(domain.ChannelBanLog, discord.BanExpiresMessage(ban, person, worker.config.ExtURL(ban)))) + monitor.notifications.Enqueue(ctx, domain.NewDiscordNotification(domain.ChannelBanLog, discord.BanExpiresMessage(ban, person, monitor.config.ExtURL(ban)))) - worker.notifications.Enqueue(ctx, domain.NewSiteUserNotification( + monitor.notifications.Enqueue(ctx, domain.NewSiteUserNotification( []steamid.SteamID{person.SteamID}, domain.SeverityInfo, "Your mute/ban period has expired", @@ -99,13 +84,13 @@ func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[Expiratio go func() { defer waitGroup.Done() - expiredNetBans, errExpiredNetBans := worker.bansNet.Expired(ctx) + expiredNetBans, errExpiredNetBans := monitor.net.Expired(ctx) if errExpiredNetBans != nil && !errors.Is(errExpiredNetBans, domain.ErrNoResult) { slog.Warn("Failed to get expired network bans", log.ErrAttr(errExpiredNetBans)) } else { for _, expiredNetBan := range expiredNetBans { expiredBan := expiredNetBan - if errDropBanNet := worker.bansNet.Delete(ctx, expiredNetBan.NetID, domain.RequestUnban{UnbanReasonText: "Expired"}, false); errDropBanNet != nil { + if errDropBanNet := monitor.net.Delete(ctx, expiredNetBan.NetID, domain.RequestUnban{UnbanReasonText: "Expired"}, false); errDropBanNet != nil { if !errors.Is(errDropBanNet, domain.ErrNoResult) { slog.Error("Failed to drop expired network expiredNetBan", log.ErrAttr(errDropBanNet)) } @@ -119,12 +104,12 @@ func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[Expiratio go func() { defer waitGroup.Done() - expiredASNBans, errExpiredASNBans := worker.bansASN.Expired(ctx) + expiredASNBans, errExpiredASNBans := monitor.asn.Expired(ctx) if errExpiredASNBans != nil && !errors.Is(errExpiredASNBans, domain.ErrNoResult) { slog.Error("Failed to get expired asn bans", log.ErrAttr(errExpiredASNBans)) } else { for _, expired := range expiredASNBans { - if errDropASN := worker.bansASN.Delete(ctx, expired.BanASNId, domain.RequestUnban{UnbanReasonText: "Expired"}); errDropASN != nil { + if errDropASN := monitor.asn.Delete(ctx, expired.BanASNId, domain.RequestUnban{UnbanReasonText: "Expired"}); errDropASN != nil { slog.Error("Failed to drop expired asn ban", log.ErrAttr(errDropASN)) } else { slog.Info("ASN ban expired", slog.Int64("ban_id", expired.BanASNId)) @@ -135,5 +120,5 @@ func (worker *ExpirationWorker) Work(ctx context.Context, _ *river.Job[Expiratio waitGroup.Wait() - return nil + return } diff --git a/internal/blocklist/blocklist_usecase.go b/internal/blocklist/blocklist_usecase.go index c98b1ee1e..3552a8115 100644 --- a/internal/blocklist/blocklist_usecase.go +++ b/internal/blocklist/blocklist_usecase.go @@ -4,6 +4,10 @@ import ( "context" "errors" "fmt" + "github.com/leighmacdonald/gbans/internal/domain" + "github.com/leighmacdonald/gbans/internal/httphelper" + "github.com/leighmacdonald/gbans/pkg/log" + "github.com/leighmacdonald/steamid/v4/steamid" "io" "log/slog" "net" @@ -13,14 +17,6 @@ import ( "regexp" "strings" "sync" - "time" - - "github.com/leighmacdonald/gbans/internal/domain" - "github.com/leighmacdonald/gbans/internal/httphelper" - "github.com/leighmacdonald/gbans/internal/queue" - "github.com/leighmacdonald/gbans/pkg/log" - "github.com/leighmacdonald/steamid/v4/steamid" - "github.com/riverqueue/river" ) type blocklistUsecase struct { @@ -313,28 +309,3 @@ func (b blocklistUsecase) DeleteCIDRBlockWhitelist(ctx context.Context, whitelis return nil } - -type ListUpdaterArgs struct{} - -func (args ListUpdaterArgs) Kind() string { - return "blocklist_update" -} - -func (args ListUpdaterArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Default), UniqueOpts: river.UniqueOpts{ByPeriod: time.Hour * 24}} -} - -func NewListUpdaterWorker(lists domain.BlocklistUsecase) *ListUpdaterWorker { - return &ListUpdaterWorker{lists: lists} -} - -type ListUpdaterWorker struct { - river.WorkerDefaults[ListUpdaterArgs] - lists domain.BlocklistUsecase -} - -func (worker *ListUpdaterWorker) Work(ctx context.Context, _ *river.Job[ListUpdaterArgs]) error { - worker.lists.Sync(ctx) - - return nil -} diff --git a/internal/cmd/serve.go b/internal/cmd/serve.go index 4da1a95b1..03f580cf4 100644 --- a/internal/cmd/serve.go +++ b/internal/cmd/serve.go @@ -98,22 +98,15 @@ func firstTimeSetup(ctx context.Context, persons domain.PersonUsecase, news doma } func createQueueWorkers(people domain.PersonUsecase, notifications domain.NotificationUsecase, - discordUC domain.DiscordUsecase, authRepo domain.AuthRepository, memberships *steamgroup.Memberships, - patreonUC domain.PatreonUsecase, bansSteam domain.BanSteamUsecase, bansNet domain.BanNetUsecase, bansASN domain.BanASNUsecase, - configUC domain.ConfigUsecase, demos domain.DemoUsecase, reports domain.ReportUsecase, - blocklists domain.BlocklistUsecase, discordOAuth domain.DiscordOAuthUsecase, + discordUC domain.DiscordUsecase, authRepo domain.AuthRepository, + patreonUC domain.PatreonUsecase, reports domain.ReportUsecase, discordOAuth domain.DiscordOAuthUsecase, ) *river.Workers { workers := river.NewWorkers() river.AddWorker[notification.SenderArgs](workers, notification.NewSenderWorker(people, notifications, discordUC)) river.AddWorker[auth.CleanupArgs](workers, auth.NewCleanupWorker(authRepo)) - river.AddWorker[steamgroup.MembershipArgs](workers, steamgroup.NewMembershipWorker(memberships)) river.AddWorker[patreon.AuthUpdateArgs](workers, patreon.NewSyncWorker(patreonUC)) - river.AddWorker[ban.ExpirationArgs](workers, ban.NewExpirationWorker(bansSteam, bansNet, bansASN, people, notifications, configUC)) - river.AddWorker[demo.CleanupArgs](workers, demo.NewCleanupWorker(demos)) river.AddWorker[report.MetaInfoArgs](workers, report.NewMetaInfoWorker(reports)) - river.AddWorker[blocklist.ListUpdaterArgs](workers, blocklist.NewListUpdaterWorker(blocklists)) - river.AddWorker[person.ExpiredArgs](workers, person.NewExpiredWorker(people)) river.AddWorker[discord.TokenRefreshArgs](workers, discord.NewTokenRefreshWorker(discordOAuth)) return workers @@ -128,13 +121,6 @@ func createPeriodicJobs() []*river.PeriodicJob { }, &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( - river.PeriodicInterval(6*time.Hour), - func() (river.JobArgs, *river.InsertOpts) { - return steamgroup.MembershipArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( river.PeriodicInterval(time.Hour), func() (river.JobArgs, *river.InsertOpts) { @@ -142,20 +128,6 @@ func createPeriodicJobs() []*river.PeriodicJob { }, &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( - river.PeriodicInterval(time.Minute), - func() (river.JobArgs, *river.InsertOpts) { - return ban.ExpirationArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - - river.NewPeriodicJob( - river.PeriodicInterval(time.Hour*24), - func() (river.JobArgs, *river.InsertOpts) { - return demo.CleanupArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( river.PeriodicInterval(24*time.Hour), func() (river.JobArgs, *river.InsertOpts) { @@ -163,20 +135,6 @@ func createPeriodicJobs() []*river.PeriodicJob { }, &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( - river.PeriodicInterval(24*time.Hour), - func() (river.JobArgs, *river.InsertOpts) { - return blocklist.ListUpdaterArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - - river.NewPeriodicJob( - river.PeriodicInterval(time.Minute*5), - func() (river.JobArgs, *river.InsertOpts) { - return person.ExpiredArgs{}, nil - }, - &river.PeriodicJobOpts{RunOnStart: true}), - river.NewPeriodicJob( river.PeriodicInterval(time.Hour*12), func() (river.JobArgs, *river.InsertOpts) { @@ -342,20 +300,15 @@ func serveCmd() *cobra.Command { //nolint:maintidx go chatUsecase.Start(ctx) forumUsecase := forum.NewForumUsecase(forum.NewForumRepository(dbConn), notificationUsecase) + go forumUsecase.Start(ctx) metricsUsecase := metrics.NewMetricsUsecase(eventBroadcaster) go metricsUsecase.Start(ctx) - go forumUsecase.Start(ctx) - newsUsecase := news.NewNewsUsecase(news.NewNewsRepository(dbConn)) - patreonUsecase := patreon.NewPatreonUsecase(patreon.NewPatreonRepository(dbConn), configUsecase) - srcdsUsecase := srcds.NewSrcdsUsecase(srcds.NewRepository(dbConn), configUsecase, serversUC, personUsecase, reportUsecase, notificationUsecase, banUsecase) - wikiUsecase := wiki.NewWikiUsecase(wiki.NewWikiRepository(dbConn)) - authRepo := auth.NewAuthRepository(dbConn) authUsecase := auth.NewAuthUsecase(authRepo, configUsecase, personUsecase, banUsecase, serversUC) @@ -428,17 +381,45 @@ func serveCmd() *cobra.Command { //nolint:maintidx notificationUsecase, discordUsecase, authRepo, - steamgroup.NewMemberships(banGroupRepo), patreonUsecase, - banUsecase, - banNetUsecase, - banASNUsecase, - configUsecase, - demos, reportUsecase, - blocklistUsecase, discordOAuthUsecase) + memberships := steamgroup.NewMemberships(banGroupRepo) + banExpirations := ban.NewExpirationMonitor(banUsecase, banNetUsecase, banASNUsecase, personUsecase, notificationUsecase, configUsecase) + + go func() { + go memberships.Update(ctx) + go banExpirations.Update(ctx) + go blocklistUsecase.Sync(ctx) + go demos.Cleanup(ctx) + + membershipsTicker := time.NewTicker(12 * time.Hour) + expirationsTicker := time.NewTicker(60 * time.Second) + reportIntoTicker := time.NewTicker(24 * time.Hour) + blocklistTicker := time.NewTicker(6 * time.Hour) + demoTicker := time.NewTicker(5 * time.Minute) + + select { + case <-ctx.Done(): + return + case <-membershipsTicker.C: + go memberships.Update(ctx) + case <-expirationsTicker.C: + go banExpirations.Update(ctx) + case <-reportIntoTicker.C: + go func() { + if errMeta := reportUsecase.GenerateMetaStats(ctx); errMeta != nil { + slog.Error("Failed to generate meta stats", log.ErrAttr(errMeta)) + } + }() + case <-blocklistTicker.C: + go blocklistUsecase.Sync(ctx) + case <-demoTicker.C: + go demos.Cleanup(ctx) + } + }() + periodicJons := createPeriodicJobs() queueClient, errClient := queue.Client(dbConn.Pool(), workers, periodicJons) if errClient != nil { diff --git a/internal/demo/demo_usecase.go b/internal/demo/demo_usecase.go index b5bf17619..e0d52dfdf 100644 --- a/internal/demo/demo_usecase.go +++ b/internal/demo/demo_usecase.go @@ -17,11 +17,9 @@ import ( "github.com/dustin/go-humanize" "github.com/gin-gonic/gin" "github.com/leighmacdonald/gbans/internal/domain" - "github.com/leighmacdonald/gbans/internal/queue" "github.com/leighmacdonald/gbans/pkg/fs" "github.com/leighmacdonald/gbans/pkg/log" "github.com/ricochet2200/go-disk-usage/du" - "github.com/riverqueue/river" ) type demoUsecase struct { @@ -369,28 +367,3 @@ func (d demoUsecase) RemoveOrphans(ctx context.Context) error { return nil } - -type CleanupArgs struct{} - -func (args CleanupArgs) Kind() string { - return "demo_cleanup" -} - -func (args CleanupArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Default), UniqueOpts: river.UniqueOpts{ByPeriod: time.Hour * 24}} -} - -func NewCleanupWorker(demos domain.DemoUsecase) *CleanupWorker { - return &CleanupWorker{demos: demos} -} - -type CleanupWorker struct { - river.WorkerDefaults[CleanupArgs] - demos domain.DemoUsecase -} - -func (worker *CleanupWorker) Work(ctx context.Context, _ *river.Job[CleanupArgs]) error { - worker.demos.Cleanup(ctx) - - return nil -} diff --git a/internal/person/person_usecase.go b/internal/person/person_usecase.go index 404b25cb9..f732b60ac 100644 --- a/internal/person/person_usecase.go +++ b/internal/person/person_usecase.go @@ -9,13 +9,11 @@ import ( "github.com/leighmacdonald/gbans/internal/domain" "github.com/leighmacdonald/gbans/internal/httphelper" - "github.com/leighmacdonald/gbans/internal/queue" "github.com/leighmacdonald/gbans/internal/thirdparty" "github.com/leighmacdonald/gbans/pkg/log" "github.com/leighmacdonald/gbans/pkg/stringutil" "github.com/leighmacdonald/steamid/v4/steamid" "github.com/leighmacdonald/steamweb/v2" - "github.com/riverqueue/river" "golang.org/x/sync/errgroup" ) @@ -267,46 +265,3 @@ func (u personUsecase) SetPermissionLevel(ctx context.Context, steamID steamid.S return u.persons.SavePerson(ctx, &person) } - -type ExpiredArgs struct{} - -func (args ExpiredArgs) Kind() string { - return "update_expired_profiles" -} - -func (args ExpiredArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Default), UniqueOpts: river.UniqueOpts{ByPeriod: time.Minute * 5}} -} - -func NewExpiredWorker(persons domain.PersonUsecase) *ExpiredWorker { - return &ExpiredWorker{persons: persons} -} - -type ExpiredWorker struct { - river.WorkerDefaults[ExpiredArgs] - persons domain.PersonUsecase -} - -func (worker *ExpiredWorker) Work(ctx context.Context, _ *river.Job[ExpiredArgs]) error { - people, errGetExpired := worker.persons.GetExpiredProfiles(ctx, 100) - if errGetExpired != nil { - if !errors.Is(errGetExpired, domain.ErrNoResult) { - return nil - } - - return errGetExpired - } - - if len(people) == 0 { - return nil - } - - _, errUpdate := worker.persons.UpdateProfiles(ctx, people) - if errUpdate != nil { - slog.Error("Failed to update profiles", log.ErrAttr(errUpdate)) - - return errUpdate - } - - return nil -} diff --git a/internal/steamgroup/steam_group.go b/internal/steamgroup/steam_group.go index caf400189..23523756d 100644 --- a/internal/steamgroup/steam_group.go +++ b/internal/steamgroup/steam_group.go @@ -9,10 +9,8 @@ import ( "github.com/leighmacdonald/gbans/internal/domain" "github.com/leighmacdonald/gbans/internal/httphelper" - "github.com/leighmacdonald/gbans/internal/queue" "github.com/leighmacdonald/steamid/v4/steamid" "github.com/leighmacdonald/steamweb/v2" - "github.com/riverqueue/river" ) var ( @@ -54,7 +52,7 @@ func (g *Memberships) IsMember(steamID steamid.SteamID) (steamid.SteamID, bool) return steamid.SteamID{}, false } -func (g *Memberships) update(ctx context.Context) { +func (g *Memberships) Update(ctx context.Context) { newMap := map[steamid.SteamID]steamid.Collection{} var total int @@ -124,28 +122,3 @@ func (g *Memberships) updateGroupBanMembers(ctx context.Context) (map[steamid.St return newMap, nil } - -type MembershipArgs struct{} - -func (args MembershipArgs) Kind() string { - return "group_members_update" -} - -func (args MembershipArgs) InsertOpts() river.InsertOpts { - return river.InsertOpts{Queue: string(queue.Default), UniqueOpts: river.UniqueOpts{ByPeriod: time.Hour * 6}} -} - -func NewMembershipWorker(memberships *Memberships) *MembershipWorker { - return &MembershipWorker{memberships: memberships} -} - -type MembershipWorker struct { - river.WorkerDefaults[MembershipArgs] - memberships *Memberships -} - -func (worker *MembershipWorker) Work(ctx context.Context, _ *river.Job[MembershipArgs]) error { - worker.memberships.update(ctx) - - return nil -}