From 2e27c633248dd9f0ac3237ba39a2816f1405e76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wysoki=C5=84ski?= Date: Tue, 27 Dec 2022 08:57:44 +0000 Subject: [PATCH] feat: new job - clean up (#147) Reviewed-on: https://gitea.dwysokinski.me/twhelp/core/pulls/147 --- cmd/twhelp/internal/job/job.go | 59 ++++-- internal/bundb/bundb_test.go | 24 +++ internal/bundb/ennoblement_test.go | 6 +- ...221112070547_tribes_players_add_columns.go | 24 +-- internal/bundb/player_snapshot.go | 55 ++++++ internal/bundb/player_snapshot_test.go | 178 +++++++++++++++++- internal/bundb/player_test.go | 2 +- internal/bundb/testdata/fixture.yml | 92 +++++++++ internal/bundb/tribe_change_test.go | 4 +- internal/bundb/tribe_snapshot_test.go | 4 +- internal/bundb/tribe_test.go | 2 +- internal/bundb/village_test.go | 2 +- internal/domain/snapshot.go | 5 + internal/service/job.go | 50 ++++- internal/service/player_snapshot.go | 14 ++ internal/service/player_snapshot_test.go | 26 +++ k8s/base/api.yml | 2 - k8s/base/ennoblement-consumer.yml | 2 - k8s/base/jobs.yml | 50 ++++- k8s/base/player-consumer.yml | 2 - k8s/base/server-consumer.yml | 2 - k8s/base/tribe-consumer.yml | 2 - k8s/base/village-consumer.yml | 2 - k8s/overlays/prod/api.yml | 2 - k8s/overlays/prod/ennoblement-consumer.yml | 2 - k8s/overlays/prod/jobs.yml | 37 +++- k8s/overlays/prod/player-consumer.yml | 2 - k8s/overlays/prod/server-consumer.yml | 2 - k8s/overlays/prod/tribe-consumer.yml | 2 - k8s/overlays/prod/village-consumer.yml | 2 - 30 files changed, 578 insertions(+), 80 deletions(-) diff --git a/cmd/twhelp/internal/job/job.go b/cmd/twhelp/internal/job/job.go index eee9bf1..e95ad4b 100644 --- a/cmd/twhelp/internal/job/job.go +++ b/cmd/twhelp/internal/job/job.go @@ -16,7 +16,7 @@ import ( ) const ( - jobTimeout = 30 * time.Second + jobDefaultTimeout = 30 * time.Second ) func New() *cli.Command { @@ -26,6 +26,7 @@ func New() *cli.Command { Subcommands: []*cli.Command{ newCreateCommand(), newUpdateCommand(), + newCleanUpCommand(), }, } } @@ -39,6 +40,25 @@ func newCreateCommand() *cli.Command { } } +func newCreateSnapshotsCommand() *cli.Command { + return &cli.Command{ + Name: "snapshots", + Usage: "Launches snapshot update", + Action: func(c *cli.Context) error { + return runJob( + c, + jobDefaultTimeout, + func(ctx context.Context, job *service.Job) error { + if err := job.CreateSnapshots(ctx); err != nil { + return fmt.Errorf("Job.CreateSnapshots: %w", err) + } + return nil + }, + ) + }, + } +} + func newUpdateCommand() *cli.Command { return &cli.Command{ Name: "update", @@ -56,6 +76,7 @@ func newUpdateDataCommand() *cli.Command { Action: func(c *cli.Context) error { return runJob( c, + jobDefaultTimeout, func(ctx context.Context, job *service.Job) error { if err := job.UpdateData(ctx); err != nil { return fmt.Errorf("Job.UpdateData: %w", err) @@ -74,6 +95,7 @@ func newUpdateEnnoblementsCommand() *cli.Command { Action: func(c *cli.Context) error { return runJob( c, + jobDefaultTimeout, func(ctx context.Context, job *service.Job) error { if err := job.UpdateEnnoblements(ctx); err != nil { return fmt.Errorf("Job.UpdateEnnoblements: %w", err) @@ -85,16 +107,17 @@ func newUpdateEnnoblementsCommand() *cli.Command { } } -func newCreateSnapshotsCommand() *cli.Command { +func newCleanUpCommand() *cli.Command { return &cli.Command{ - Name: "snapshots", - Usage: "Launches snapshot update", + Name: "cleanup", + Usage: "Launches database cleanup", Action: func(c *cli.Context) error { return runJob( c, + time.Hour, func(ctx context.Context, job *service.Job) error { - if err := job.CreateSnapshots(ctx); err != nil { - return fmt.Errorf("Job.CreateSnapshots: %w", err) + if err := job.CleanUp(ctx); err != nil { + return fmt.Errorf("Job.CleanUp: %w", err) } return nil }, @@ -105,7 +128,7 @@ func newCreateSnapshotsCommand() *cli.Command { type runJobFunc func(ctx context.Context, job *service.Job) error -func runJob(c *cli.Context, fn runJobFunc) error { +func runJob(c *cli.Context, timeout time.Duration, fn runJobFunc) error { db, err := internal.NewBunDB() if err != nil { return fmt.Errorf("internal.NewBunDB: %w", err) @@ -120,19 +143,23 @@ func runJob(c *cli.Context, fn runJobFunc) error { } marshaler := internal.NewCommandEventMarshaler() - ctx, cancel := context.WithTimeout(c.Context, jobTimeout) + ctx, cancel := context.WithTimeout(c.Context, timeout) defer cancel() - err = fn(ctx, service.NewJob( - service.NewVersion(bundb.NewVersion(db)), - service.NewServer(bundb.NewServer(db), internal.NewTWClient(c.App.Version)), + client := internal.NewTWClient(c.App.Version) + + versionSvc := service.NewVersion(bundb.NewVersion(db)) + serverSvc := service.NewServer(bundb.NewServer(db), client) + tribeChangeSvc := service.NewTribeChange(bundb.NewTribeChange(db)) + playerSvc := service.NewPlayer(bundb.NewPlayer(db), tribeChangeSvc, client) + playerSnapshotSvc := service.NewPlayerSnapshot(bundb.NewPlayerSnapshot(db), playerSvc) + + return fn(ctx, service.NewJob( + versionSvc, + serverSvc, msg.NewServerPublisher(publisher, marshaler), msg.NewEnnoblementPublisher(publisher, marshaler), msg.NewSnapshotPublisher(publisher, marshaler), + []service.CleanUper{playerSnapshotSvc}, )) - if err != nil { - return err - } - - return nil } diff --git a/internal/bundb/bundb_test.go b/internal/bundb/bundb_test.go index 7a2b3a7..653dbf1 100644 --- a/internal/bundb/bundb_test.go +++ b/internal/bundb/bundb_test.go @@ -251,6 +251,30 @@ func (f *bunfixture) ennoblement(tb testing.TB, id string) domain.Ennoblement { return e.ToDomain() } +func (f *bunfixture) playerSnapshots(tb testing.TB) []domain.PlayerSnapshot { + tb.Helper() + + //nolint:lll + ids := []string{"pl169-chcesz-remont-2021-09-02", "pl169-chcesz-remont-2021-09-03", "pl169-maddov-2021-09-08", "de188-1scoo-2021-06-25", "de188-1scoo-2021-06-26"} + + snapshots := make([]domain.PlayerSnapshot, 0, len(ids)) + for _, id := range ids { + snapshots = append(snapshots, f.playerSnapshot(tb, id)) + } + + return snapshots +} + +func (f *bunfixture) playerSnapshot(tb testing.TB, id string) domain.PlayerSnapshot { + tb.Helper() + + row, err := f.Row("PlayerSnapshot." + id) + require.NoError(tb, err) + p, ok := row.(*model.PlayerSnapshot) + require.True(tb, ok) + return p.ToDomain() +} + func generateSchema() string { return strings.TrimFunc(strings.ReplaceAll(uuid.NewString(), "-", "_"), unicode.IsNumber) } diff --git a/internal/bundb/ennoblement_test.go b/internal/bundb/ennoblement_test.go index c057c51..c1e22a3 100644 --- a/internal/bundb/ennoblement_test.go +++ b/internal/bundb/ennoblement_test.go @@ -83,7 +83,7 @@ func TestEnnoblement_Create(t *testing.T) { for _, p := range params { var found bool for _, e := range ennoblements { - p2 := domain.CreateEnnoblementParams{ + if cmp.Equal(p, domain.CreateEnnoblementParams{ VillageID: e.VillageID, NewOwnerID: e.NewOwnerID, NewTribeID: e.NewTribeID, @@ -92,9 +92,7 @@ func TestEnnoblement_Create(t *testing.T) { Points: e.Points, CreatedAt: e.CreatedAt, ServerKey: e.ServerKey, - } - - if cmp.Equal(p, p2, cmpopts.EquateApproxTime(1*time.Second)) { + }, cmpopts.EquateApproxTime(1*time.Second)) { found = true break } diff --git a/internal/bundb/migrations/20221112070547_tribes_players_add_columns.go b/internal/bundb/migrations/20221112070547_tribes_players_add_columns.go index 9d1625b..c405c44 100644 --- a/internal/bundb/migrations/20221112070547_tribes_players_add_columns.go +++ b/internal/bundb/migrations/20221112070547_tribes_players_add_columns.go @@ -68,34 +68,34 @@ func init() { return fmt.Errorf("couldn't select servers from the db: %w", err) } - for _, s := range servers { + for _, srv := range servers { if _, err = db.NewUpdate(). Model(&model.Player{}). Set("best_rank = rank"). - Set("best_rank_at = ?", s.PlayerDataUpdatedAt). + Set("best_rank_at = ?", srv.PlayerDataUpdatedAt). Set("most_points = points"). - Set("most_points_at = ?", s.PlayerDataUpdatedAt). + Set("most_points_at = ?", srv.PlayerDataUpdatedAt). Set("most_villages = num_villages"). - Set("most_villages_at = ?", s.PlayerDataUpdatedAt). - Set("last_activity_at = ?", s.PlayerDataUpdatedAt). + Set("most_villages_at = ?", srv.PlayerDataUpdatedAt). + Set("last_activity_at = ?", srv.PlayerDataUpdatedAt). Where("deleted_at IS NULL"). - Where("server_key = ?", s.Key). + Where("server_key = ?", srv.Key). Exec(ctx); err != nil { - return fmt.Errorf("couldn't update players (server=%s): %w", s.Key, err) + return fmt.Errorf("couldn't update players (server=%s): %w", srv.Key, err) } if _, err = db.NewUpdate(). Model(&model.Tribe{}). Set("best_rank = rank"). - Set("best_rank_at = ?", s.TribeDataUpdatedAt). + Set("best_rank_at = ?", srv.TribeDataUpdatedAt). Set("most_points = points"). - Set("most_points_at = ?", s.TribeDataUpdatedAt). + Set("most_points_at = ?", srv.TribeDataUpdatedAt). Set("most_villages = num_villages"). - Set("most_villages_at = ?", s.TribeDataUpdatedAt). + Set("most_villages_at = ?", srv.TribeDataUpdatedAt). Where("deleted_at IS NULL"). - Where("server_key = ?", s.Key). + Where("server_key = ?", srv.Key). Exec(ctx); err != nil { - return fmt.Errorf("couldn't update tribes (server=%s): %w", s.Key, err) + return fmt.Errorf("couldn't update tribes (server=%s): %w", srv.Key, err) } } diff --git a/internal/bundb/player_snapshot.go b/internal/bundb/player_snapshot.go index 37095c3..878c4b4 100644 --- a/internal/bundb/player_snapshot.go +++ b/internal/bundb/player_snapshot.go @@ -2,7 +2,10 @@ package bundb import ( "context" + "database/sql" + "errors" "fmt" + "time" "gitea.dwysokinski.me/twhelp/core/internal/bundb/internal/model" "gitea.dwysokinski.me/twhelp/core/internal/domain" @@ -36,3 +39,55 @@ func (p *PlayerSnapshot) Create(ctx context.Context, params ...domain.CreatePlay return nil } + +func (p *PlayerSnapshot) List(ctx context.Context, params domain.ListPlayerSnapshotsParams) ([]domain.PlayerSnapshot, error) { + var snapshots []model.PlayerSnapshot + + if err := p.db.NewSelect(). + Model(&snapshots). + Order("server_key ASC", "date ASC"). + Apply(listPlayerSnapshotsParamsApplier{params}.apply). + Scan(ctx); err != nil && !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("couldn't select player snapshots from the db: %w", err) + } + + result := make([]domain.PlayerSnapshot, 0, len(snapshots)) + for _, snapshot := range snapshots { + result = append(result, snapshot.ToDomain()) + } + + return result, nil +} + +func (p *PlayerSnapshot) Delete(ctx context.Context, serverKey string, createdAtLTE time.Time) error { + if _, err := p.db.NewDelete(). + Model(&model.PlayerSnapshot{}). + Where("server_key = ?", serverKey). + Where("created_at <= ?", createdAtLTE). + Returning("NULL"). + Exec(ctx); err != nil { + return fmt.Errorf("couldn't delete player snapshots: %w", err) + } + + return nil +} + +type listPlayerSnapshotsParamsApplier struct { + params domain.ListPlayerSnapshotsParams +} + +func (l listPlayerSnapshotsParamsApplier) apply(q *bun.SelectQuery) *bun.SelectQuery { + return l.applyPagination(l.applyFilters(q)) +} + +func (l listPlayerSnapshotsParamsApplier) applyFilters(q *bun.SelectQuery) *bun.SelectQuery { + if l.params.ServerKeys != nil { + q = q.Where("ps.server_key IN (?)", bun.In(l.params.ServerKeys)) + } + + return q +} + +func (l listPlayerSnapshotsParamsApplier) applyPagination(q *bun.SelectQuery) *bun.SelectQuery { + return (paginationApplier{pagination: l.params.Pagination}).apply(q) +} diff --git a/internal/bundb/player_snapshot_test.go b/internal/bundb/player_snapshot_test.go index 8058b68..0f35db8 100644 --- a/internal/bundb/player_snapshot_test.go +++ b/internal/bundb/player_snapshot_test.go @@ -7,6 +7,8 @@ import ( "gitea.dwysokinski.me/twhelp/core/internal/bundb" "gitea.dwysokinski.me/twhelp/core/internal/domain" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/jackc/pgerrcode" "github.com/stretchr/testify/assert" "github.com/uptrace/bun/driver/pgdriver" @@ -59,6 +61,30 @@ func TestPlayerSnapshot_Create(t *testing.T) { } assert.NoError(t, repo.Create(context.Background(), params...)) + + snapshots, err := repo.List(context.Background(), domain.ListPlayerSnapshotsParams{ + ServerKeys: []string{fixture.server(t, "de188").Key}, + }) + assert.NoError(t, err) + for _, p := range params { + var found bool + for _, s := range snapshots { + if cmp.Equal(p, domain.CreatePlayerSnapshotParams{ + OpponentsDefeated: s.OpponentsDefeated, + PlayerID: s.PlayerID, + NumVillages: s.NumVillages, + Points: s.Points, + Rank: s.Rank, + TribeID: s.TribeID, + ServerKey: s.ServerKey, + Date: s.Date, + }, cmpopts.EquateApproxTime(24*time.Hour)) { + found = true + break + } + } + assert.True(t, found) + } }) t.Run("OK: len(params) == 0", func(t *testing.T) { @@ -102,7 +128,7 @@ func TestPlayerSnapshot_Create(t *testing.T) { assert.Equal(t, "player_snapshots_player_id_server_key_date_key", pgErr.Field('n')) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() playerRiou89 := fixture.player(t, "de188-riou89") @@ -127,7 +153,7 @@ func TestPlayerSnapshot_Create(t *testing.T) { assert.Equal(t, "player_snapshots_server_key_fkey", pgErr.Field('n')) }) - t.Run("ERR: player must exist in the db", func(t *testing.T) { + t.Run("ERR: player must exist", func(t *testing.T) { t.Parallel() playerRiou89 := fixture.player(t, "de188-riou89") @@ -152,3 +178,151 @@ func TestPlayerSnapshot_Create(t *testing.T) { assert.Equal(t, "player_snapshots_player_id_server_key_fkey", pgErr.Field('n')) }) } + +func TestPlayerSnapshot_List(t *testing.T) { + t.Parallel() + + db := newDB(t) + fixture := loadFixtures(t, db) + repo := bundb.NewPlayerSnapshot(db) + snapshots := fixture.playerSnapshots(t) + + type expectedSnapshots struct { + id int64 + serverKey string + } + + allSnapshots := make([]expectedSnapshots, 0, len(snapshots)) + for _, p := range snapshots { + allSnapshots = append(allSnapshots, expectedSnapshots{ + id: p.ID, + serverKey: p.ServerKey, + }) + } + + //nolint:prealloc + var snapshotsDE188 []expectedSnapshots + for _, p := range snapshots { + if p.ServerKey != "de188" { + continue + } + + snapshotsDE188 = append(snapshotsDE188, expectedSnapshots{ + id: p.ID, + serverKey: p.ServerKey, + }) + } + + tests := []struct { + name string + params domain.ListPlayerSnapshotsParams + expectedSnapshots []expectedSnapshots + }{ + { + name: "Empty struct", + params: domain.ListPlayerSnapshotsParams{}, + expectedSnapshots: allSnapshots, + }, + { + name: "ServerKey=[de188]", + params: domain.ListPlayerSnapshotsParams{ + ServerKeys: []string{"de188"}, + }, + expectedSnapshots: snapshotsDE188, + }, + { + name: "ServerKey=[pl169],Limit=2", + params: domain.ListPlayerSnapshotsParams{ + ServerKeys: []string{"pl169"}, + Pagination: domain.Pagination{ + Limit: 2, + }, + }, + expectedSnapshots: []expectedSnapshots{ + { + id: 100000, + serverKey: "pl169", + }, + { + id: 100001, + serverKey: "pl169", + }, + }, + }, + { + name: "ServerKey=[pl169],Offset=1", + params: domain.ListPlayerSnapshotsParams{ + ServerKeys: []string{"pl169"}, + Pagination: domain.Pagination{ + Offset: 1, + }, + }, + expectedSnapshots: []expectedSnapshots{ + { + id: 100001, + serverKey: "pl169", + }, + { + id: 100050, + serverKey: "pl169", + }, + }, + }, + { + name: "ServerKey=[pl169],Offset=1,Limit=1", + params: domain.ListPlayerSnapshotsParams{ + ServerKeys: []string{"pl169"}, + Pagination: domain.Pagination{ + Limit: 1, + Offset: 1, + }, + }, + expectedSnapshots: []expectedSnapshots{ + { + id: 100001, + serverKey: "pl169", + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + res, err := repo.List(context.Background(), tt.params) + assert.NoError(t, err) + assert.Len(t, res, len(tt.expectedSnapshots)) + for _, expSnapshot := range tt.expectedSnapshots { + found := false + for _, snapshot := range res { + if snapshot.ID == expSnapshot.id && snapshot.ServerKey == expSnapshot.serverKey { + found = true + break + } + } + assert.True(t, found, "snapshot (id=%d,serverkey=%s) not found", expSnapshot.id, expSnapshot.serverKey) + } + }) + } +} + +func TestPlayerSnapshot_Delete(t *testing.T) { + t.Parallel() + + db := newDB(t) + fixture := loadFixtures(t, db) + repo := bundb.NewPlayerSnapshot(db) + + snapshotsBeforeDelete, err := repo.List(context.Background(), domain.ListPlayerSnapshotsParams{}) + assert.NoError(t, err) + assert.Greater(t, len(snapshotsBeforeDelete), 0) + + assert.NoError(t, repo.Delete(context.Background(), fixture.server(t, "pl169").Key, time.Date(2021, time.September, 3, 23, 0, 0, 0, time.UTC))) + + snapshotsAfterDelete, err := repo.List(context.Background(), domain.ListPlayerSnapshotsParams{}) + assert.NoError(t, err) + assert.Equal(t, len(snapshotsBeforeDelete)-2, len(snapshotsAfterDelete)) +} diff --git a/internal/bundb/player_test.go b/internal/bundb/player_test.go index 6d9a035..c33011e 100644 --- a/internal/bundb/player_test.go +++ b/internal/bundb/player_test.go @@ -142,7 +142,7 @@ func TestPlayer_CreateOrUpdate(t *testing.T) { assert.NoError(t, repo.CreateOrUpdate(context.Background())) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() err := repo.CreateOrUpdate(context.Background(), domain.CreatePlayerParams{ diff --git a/internal/bundb/testdata/fixture.yml b/internal/bundb/testdata/fixture.yml index f41da82..7d0ef66 100644 --- a/internal/bundb/testdata/fixture.yml +++ b/internal/bundb/testdata/fixture.yml @@ -7283,3 +7283,95 @@ old_tribe_id: 31 points: 5123 created_at: 2022-04-22T15:00:10.000Z +- model: PlayerSnapshot + rows: + - rank_att: 5 + score_att: 38213157 + rank_def: 2 + score_def: 51137731 + rank_sup: 104 + score_sup: 2282683 + rank_total: 1 + score_total: 91633571 + _id: pl169-chcesz-remont-2021-09-02 + id: 100000 + player_id: 6180190 + server_key: pl169 + num_villages: 665 + points: 7298055 + rank: 1 + tribe_id: 27 + date: 2021-09-02T00:00:00.000Z + created_at: 2021-09-02T21:01:03.000Z + - rank_att: 5 + score_att: 38213157 + rank_def: 2 + score_def: 51137731 + rank_sup: 104 + score_sup: 2282683 + rank_total: 1 + score_total: 91633571 + _id: pl169-chcesz-remont-2021-09-03 + id: 100001 + player_id: 6180190 + server_key: pl169 + num_villages: 665 + points: 7298055 + rank: 1 + tribe_id: 27 + date: 2021-09-03T00:00:00.000Z + created_at: 2021-09-03T21:01:03.000Z + - rank_att: 13 + score_att: 23124653 + rank_def: 112 + score_def: 5547007 + rank_sup: 10 + score_sup: 10651344 + rank_total: 23 + score_total: 39323004 + _id: pl169-maddov-2021-09-08 + id: 100050 + player_id: 8419570 + server_key: pl169 + num_villages: 643 + points: 6292398 + rank: 2 + tribe_id: 2 + date: 2021-09-08T00:00:00.000Z + created_at: 2021-09-08T20:01:11.000Z + - rank_att: 10 + score_att: 27638462 + rank_def: 35 + score_def: 5555868 + rank_sup: 3 + score_sup: 9815277 + rank_total: 16 + score_total: 43009607 + _id: de188-1scoo-2021-06-25 + id: 100100 + player_id: 1577279214 + server_key: de188 + num_villages: 878 + points: 9199775 + rank: 1 + tribe_id: 772 + date: 2021-06-25T00:00:00.000Z + created_at: 2021-06-25T11:00:53.000Z + - rank_att: 10 + score_att: 27638462 + rank_def: 35 + score_def: 5555868 + rank_sup: 3 + score_sup: 9815277 + rank_total: 16 + score_total: 43009607 + _id: de188-1scoo-2021-06-26 + id: 100101 + player_id: 1577279214 + server_key: de188 + num_villages: 878 + points: 9199775 + rank: 1 + tribe_id: 772 + date: 2021-06-26T00:00:00.000Z + created_at: 2021-06-26T11:00:53.000Z \ No newline at end of file diff --git a/internal/bundb/tribe_change_test.go b/internal/bundb/tribe_change_test.go index 334babe..5dd458c 100644 --- a/internal/bundb/tribe_change_test.go +++ b/internal/bundb/tribe_change_test.go @@ -61,7 +61,7 @@ func TestTribeChange_Create(t *testing.T) { assert.NoError(t, repo.Create(context.Background())) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() serverKey := "random" @@ -83,7 +83,7 @@ func TestTribeChange_Create(t *testing.T) { assert.Equal(t, "tribe_changes_server_key_fkey", pgErr.Field('n')) }) - t.Run("ERR: player must exist in the db", func(t *testing.T) { + t.Run("ERR: player must exist", func(t *testing.T) { t.Parallel() playerRiou89 := fixture.player(t, "de188-riou89") diff --git a/internal/bundb/tribe_snapshot_test.go b/internal/bundb/tribe_snapshot_test.go index d84116c..0920fcc 100644 --- a/internal/bundb/tribe_snapshot_test.go +++ b/internal/bundb/tribe_snapshot_test.go @@ -112,7 +112,7 @@ func TestTribeSnapshot_Create(t *testing.T) { assert.Equal(t, "tribe_snapshots_tribe_id_server_key_date_key", pgErr.Field('n')) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() serverKey := "random" @@ -140,7 +140,7 @@ func TestTribeSnapshot_Create(t *testing.T) { assert.Equal(t, "tribe_snapshots_server_key_fkey", pgErr.Field('n')) }) - t.Run("ERR: tribe must exist in the db", func(t *testing.T) { + t.Run("ERR: tribe must exist", func(t *testing.T) { t.Parallel() tribeTT := fixture.tribe(t, "pl169-tt") diff --git a/internal/bundb/tribe_test.go b/internal/bundb/tribe_test.go index 8782d4e..9d4d659 100644 --- a/internal/bundb/tribe_test.go +++ b/internal/bundb/tribe_test.go @@ -142,7 +142,7 @@ func TestTribe_CreateOrUpdate(t *testing.T) { assert.NoError(t, repo.CreateOrUpdate(context.Background())) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() err := repo.CreateOrUpdate(context.Background(), domain.CreateTribeParams{ diff --git a/internal/bundb/village_test.go b/internal/bundb/village_test.go index 957030e..45e0201 100644 --- a/internal/bundb/village_test.go +++ b/internal/bundb/village_test.go @@ -96,7 +96,7 @@ func TestVillage_CreateOrUpdate(t *testing.T) { assert.NoError(t, repo.CreateOrUpdate(context.Background())) }) - t.Run("ERR: server must exist in the db", func(t *testing.T) { + t.Run("ERR: server must exist", func(t *testing.T) { t.Parallel() err := repo.CreateOrUpdate(context.Background(), domain.CreateVillageParams{ diff --git a/internal/domain/snapshot.go b/internal/domain/snapshot.go index 1974c7e..f880523 100644 --- a/internal/domain/snapshot.go +++ b/internal/domain/snapshot.go @@ -28,6 +28,11 @@ type CreatePlayerSnapshotParams struct { Date time.Time } +type ListPlayerSnapshotsParams struct { + ServerKeys []string + Pagination Pagination +} + type TribeSnapshot struct { OpponentsDefeated diff --git a/internal/service/job.go b/internal/service/job.go index 8ba6fe6..4027cb6 100644 --- a/internal/service/job.go +++ b/internal/service/job.go @@ -29,12 +29,17 @@ type SnapshotPublisher interface { CmdCreateTribes(ctx context.Context, payloads ...domain.CreateSnapshotsCmdPayload) error } +type CleanUper interface { + CleanUp(ctx context.Context, srv domain.Server) error +} + type Job struct { versionSvc VersionLister serverSvc ServerLister serverPublisher ServerPublisher ennoblementPublisher EnnoblementPublisher snapshotPublisher SnapshotPublisher + cleaners []CleanUper } func NewJob( @@ -43,6 +48,7 @@ func NewJob( serverPublisher ServerPublisher, ennoblementPublisher EnnoblementPublisher, snapshotPublisher SnapshotPublisher, + cleaners []CleanUper, ) *Job { return &Job{ versionSvc: versionSvc, @@ -50,6 +56,7 @@ func NewJob( serverPublisher: serverPublisher, ennoblementPublisher: ennoblementPublisher, snapshotPublisher: snapshotPublisher, + cleaners: cleaners, } } @@ -201,12 +208,49 @@ func (j *Job) publishCreateSnapshotsCmds( return nil } +func (j *Job) CleanUp(ctx context.Context) error { + versions, err := j.versionSvc.List(ctx) + if err != nil { + return fmt.Errorf("VersionService: %w", err) + } + + for _, v := range versions { + servers, err := j.serverSvc.List(ctx, domain.ListServersParams{ + Special: domain.NullBool{ + Bool: false, + Valid: true, + }, + Open: domain.NullBool{ + Bool: false, + Valid: true, + }, + VersionCodes: []string{v.Code}, + Pagination: domain.Pagination{ + Limit: serverMaxLimit, + }, + }) + if err != nil { + return fmt.Errorf("%s: ServerService.List: %w", v.Code, err) + } + + for _, srv := range servers { + for _, c := range j.cleaners { + if err := c.CleanUp(ctx, srv); err != nil { + return fmt.Errorf("%s: %w", srv.Key, err) + } + } + } + } + + return nil +} + func newCreateSnapshotsCmdPayloads(servers []domain.Server, date time.Time) []domain.CreateSnapshotsCmdPayload { payloads := make([]domain.CreateSnapshotsCmdPayload, 0, len(servers)) - for _, s := range servers { + for _, srv := range servers { payloads = append(payloads, domain.CreateSnapshotsCmdPayload{ - Key: s.Key, - VersionCode: s.VersionCode, + Key: srv.Key, + VersionCode: srv.VersionCode, Date: date, }) } diff --git a/internal/service/player_snapshot.go b/internal/service/player_snapshot.go index 30dce7e..4ba2733 100644 --- a/internal/service/player_snapshot.go +++ b/internal/service/player_snapshot.go @@ -16,6 +16,7 @@ type PlayerLister interface { //counterfeiter:generate -o internal/mock/player_snapshot_repository.gen.go . PlayerSnapshotRepository type PlayerSnapshotRepository interface { Create(ctx context.Context, params ...domain.CreatePlayerSnapshotParams) error + Delete(ctx context.Context, serverKey string, createdAtLTE time.Time) error } type PlayerSnapshot struct { @@ -78,3 +79,16 @@ func (p *PlayerSnapshot) Create(ctx context.Context, key string, date time.Time) return nil } + +func (p *PlayerSnapshot) CleanUp(ctx context.Context, srv domain.Server) error { + if srv.PlayerSnapshotsCreatedAt.After(time.Now().Add(-30 * 24 * time.Hour)) { // 30 days + return nil + } + + // delete snapshots older than 6 months + if err := p.repo.Delete(ctx, srv.Key, time.Now().Add(-180*24*time.Hour)); err != nil { + return fmt.Errorf("couldn't delete old player snapshots: %w", err) + } + + return nil +} diff --git a/internal/service/player_snapshot_test.go b/internal/service/player_snapshot_test.go index 68bf0af..e82a60e 100644 --- a/internal/service/player_snapshot_test.go +++ b/internal/service/player_snapshot_test.go @@ -82,3 +82,29 @@ func TestPlayerSnapshot_Create(t *testing.T) { assert.Equal(t, date, params[i].Date) } } + +func TestPlayerSnapshot_CleanUp(t *testing.T) { + t.Parallel() + + serverKey := "pl151" + playerSvc := &mock.FakePlayerLister{} + repo := &mock.FakePlayerSnapshotRepository{} + repo.DeleteReturns(nil) + + svc := service.NewPlayerSnapshot(repo, playerSvc) + + assert.NoError(t, svc.CleanUp(context.Background(), domain.Server{ + Key: serverKey, + PlayerSnapshotsCreatedAt: time.Now(), + })) + assert.Equal(t, 0, repo.DeleteCallCount()) // only servers with PlayerSnapshotsCreatedAt < now - 30 days + + assert.NoError(t, svc.CleanUp(context.Background(), domain.Server{ + Key: serverKey, + PlayerSnapshotsCreatedAt: time.Now().Add(-30 * 24 * time.Hour), + })) + require.Equal(t, 1, repo.DeleteCallCount()) + _, argsServerKey, argsCreatedAtLTE := repo.DeleteArgsForCall(0) + assert.Equal(t, serverKey, argsServerKey) + assert.WithinDuration(t, time.Now().Add(-180*24*time.Hour), argsCreatedAtLTE, time.Second) +} diff --git a/k8s/base/api.yml b/k8s/base/api.yml index b6a5eb7..dd10405 100644 --- a/k8s/base/api.yml +++ b/k8s/base/api.yml @@ -30,8 +30,6 @@ spec: value: "10" - name: DB_MAX_IDLE_CONNECTIONS value: "5" - - name: OTEL_ENABLED - value: "false" - name: API_SWAGGER_ENABLED value: "true" livenessProbe: diff --git a/k8s/base/ennoblement-consumer.yml b/k8s/base/ennoblement-consumer.yml index fc7b6f8..a8c5fdc 100644 --- a/k8s/base/ennoblement-consumer.yml +++ b/k8s/base/ennoblement-consumer.yml @@ -32,8 +32,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" livenessProbe: exec: command: [ "cat", "/tmp/healthy" ] diff --git a/k8s/base/jobs.yml b/k8s/base/jobs.yml index 88caaf2..e83c452 100644 --- a/k8s/base/jobs.yml +++ b/k8s/base/jobs.yml @@ -33,8 +33,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" resources: requests: cpu: 50m @@ -79,8 +77,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" resources: requests: cpu: 50m @@ -125,8 +121,50 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: twhelp-job-clean-up +spec: + schedule: "35 0 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + jobTemplate: + spec: + parallelism: 1 + template: + spec: + restartPolicy: Never + containers: + - name: twhelp-job-clean-up + image: twhelp + args: ["job", "cleanup"] + env: + - name: APP_MODE + value: development + - name: DB_DSN + valueFrom: + secretKeyRef: + name: twhelp-secret + key: db-dsn + - name: DB_MAX_OPEN_CONNECTIONS + value: "2" + - name: DB_MAX_IDLE_CONNECTIONS + value: "2" + - name: AMQP_URI + valueFrom: + secretKeyRef: + name: twhelp-secret + key: amqp-uri resources: requests: cpu: 50m diff --git a/k8s/base/player-consumer.yml b/k8s/base/player-consumer.yml index 0a2cc9b..590ee60 100644 --- a/k8s/base/player-consumer.yml +++ b/k8s/base/player-consumer.yml @@ -32,8 +32,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" livenessProbe: exec: command: [ "cat", "/tmp/healthy" ] diff --git a/k8s/base/server-consumer.yml b/k8s/base/server-consumer.yml index d0a70c5..6d2093a 100644 --- a/k8s/base/server-consumer.yml +++ b/k8s/base/server-consumer.yml @@ -32,8 +32,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" livenessProbe: exec: command: [ "cat", "/tmp/healthy" ] diff --git a/k8s/base/tribe-consumer.yml b/k8s/base/tribe-consumer.yml index 6e0d446..f20111a 100644 --- a/k8s/base/tribe-consumer.yml +++ b/k8s/base/tribe-consumer.yml @@ -32,8 +32,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" livenessProbe: exec: command: [ "cat", "/tmp/healthy" ] diff --git a/k8s/base/village-consumer.yml b/k8s/base/village-consumer.yml index 3aab129..0075876 100644 --- a/k8s/base/village-consumer.yml +++ b/k8s/base/village-consumer.yml @@ -32,8 +32,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" livenessProbe: exec: command: [ "cat", "/tmp/healthy" ] diff --git a/k8s/overlays/prod/api.yml b/k8s/overlays/prod/api.yml index 8180d22..273975c 100644 --- a/k8s/overlays/prod/api.yml +++ b/k8s/overlays/prod/api.yml @@ -20,8 +20,6 @@ spec: value: "12" - name: DB_MAX_IDLE_CONNECTIONS value: "8" - - name: OTEL_ENABLED - value: "false" - name: API_SWAGGER_ENABLED value: "true" - name: API_SWAGGER_HOST diff --git a/k8s/overlays/prod/ennoblement-consumer.yml b/k8s/overlays/prod/ennoblement-consumer.yml index 4d851b0..0b4b4e4 100644 --- a/k8s/overlays/prod/ennoblement-consumer.yml +++ b/k8s/overlays/prod/ennoblement-consumer.yml @@ -26,5 +26,3 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" \ No newline at end of file diff --git a/k8s/overlays/prod/jobs.yml b/k8s/overlays/prod/jobs.yml index 84f9155..1442c92 100644 --- a/k8s/overlays/prod/jobs.yml +++ b/k8s/overlays/prod/jobs.yml @@ -28,8 +28,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" --- apiVersion: batch/v1 kind: CronJob @@ -61,8 +59,6 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" --- apiVersion: batch/v1 kind: CronJob @@ -94,8 +90,37 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: twhelp-job-clean-up +spec: + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: twhelp-job-clean-up + image: twhelp + env: + - name: APP_MODE + value: production + - name: DB_DSN + valueFrom: + secretKeyRef: + name: twhelp-secret + key: db-dsn + - name: DB_MAX_OPEN_CONNECTIONS + value: "2" + - name: DB_MAX_IDLE_CONNECTIONS + value: "2" + - name: AMQP_URI + valueFrom: + secretKeyRef: + name: twhelp-secret + key: amqp-uri --- apiVersion: batch/v1 kind: Job diff --git a/k8s/overlays/prod/player-consumer.yml b/k8s/overlays/prod/player-consumer.yml index e71ed5a..b700520 100644 --- a/k8s/overlays/prod/player-consumer.yml +++ b/k8s/overlays/prod/player-consumer.yml @@ -25,5 +25,3 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" diff --git a/k8s/overlays/prod/server-consumer.yml b/k8s/overlays/prod/server-consumer.yml index e956f2d..5d82dac 100644 --- a/k8s/overlays/prod/server-consumer.yml +++ b/k8s/overlays/prod/server-consumer.yml @@ -25,5 +25,3 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" diff --git a/k8s/overlays/prod/tribe-consumer.yml b/k8s/overlays/prod/tribe-consumer.yml index 6269ce5..319c686 100644 --- a/k8s/overlays/prod/tribe-consumer.yml +++ b/k8s/overlays/prod/tribe-consumer.yml @@ -25,5 +25,3 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false" diff --git a/k8s/overlays/prod/village-consumer.yml b/k8s/overlays/prod/village-consumer.yml index 1b8355b..aa6ed40 100644 --- a/k8s/overlays/prod/village-consumer.yml +++ b/k8s/overlays/prod/village-consumer.yml @@ -26,5 +26,3 @@ spec: secretKeyRef: name: twhelp-secret key: amqp-uri - - name: OTEL_ENABLED - value: "false"