feat: add a new integration test - TestDataCleanup (#38)

Reviewed-on: twhelp/corev3#38
2024-03-28 06:54:38 +00:00 · 2024-03-28 06:54:38 +00:00 · b2b8a5b802
parent aeee5df100
commit b2b8a5b802
5 changed files with 638 additions and 0 deletions
--- a/internal/port/consumer_data_cleanup_test.go
+++ b/internal/port/consumer_data_cleanup_test.go
@ -0,0 +1,230 @@
+package port_test
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"path"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"gitea.dwysokinski.me/twhelp/corev3/internal/adapter"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/app"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/bun/buntest"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/domain"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/port"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/watermill/watermillamqptest"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/watermill/watermillmsg"
+	"gitea.dwysokinski.me/twhelp/corev3/internal/watermill/watermilltest"
+	"github.com/ThreeDotsLabs/watermill"
+	"github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp"
+	"github.com/brianvoe/gofakeit/v7"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDataCleanup(t *testing.T) {
+	t.Parallel()
+
+	if testing.Short() {
+		t.Skip("skipping long-running test")
+	}
+
+	ctxTimeout, cancel := context.WithTimeout(context.Background(), 150*time.Second)
+	defer cancel()
+
+	ctx, stop := signal.NotifyContext(ctxTimeout, os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	filesys := os.DirFS("./testdata/cleanup")
+
+	// bun
+	db := postgres.NewDB(t)
+	buntest.NewFixture(db).Load(t, ctx, filesys, "fixture.yml")
+
+	// watermill
+	marshaler := watermillmsg.JSONMarshaler{NewUUID: watermill.NewUUID}
+	generateExchangeAndRoutingKeyName := func(topic string) string {
+		return topic + "_data_cleanup"
+	}
+	rmqConn := rabbitMQ.NewConnection(t)
+	nopLogger := watermill.NopLogger{}
+	tribePub, tribeSub := watermillamqptest.NewPubSub(
+		t,
+		rmqConn,
+		amqp.GenerateQueueNameTopicNameWithSuffix("_data_cleanup_tribe"),
+		generateExchangeAndRoutingKeyName,
+	)
+	_, playerSub := watermillamqptest.NewPubSub(
+		t,
+		rmqConn,
+		amqp.GenerateQueueNameTopicNameWithSuffix("_data_cleanup_player"),
+		generateExchangeAndRoutingKeyName,
+	)
+	_, ennoblementSub := watermillamqptest.NewPubSub(
+		t,
+		rmqConn,
+		amqp.GenerateQueueNameTopicNameWithSuffix("_data_cleanup_ennoblement"),
+		generateExchangeAndRoutingKeyName,
+	)
+
+	// events/commands
+	cmdCleanUpTopic := gofakeit.UUID()
+
+	// adapters
+	serverRepo := adapter.NewServerBunRepository(db)
+	ennoblementRepo := adapter.NewEnnoblementBunRepository(db)
+	tribeSnapshotRepo := adapter.NewTribeSnapshotBunRepository(db)
+	playerSnapshotRepo := adapter.NewPlayerSnapshotBunRepository(db)
+	dataCleanupPublisher := adapter.NewDataCleanupPublisher(tribePub, marshaler, cmdCleanUpTopic)
+
+	// services
+	serverSvc := app.NewServerService(serverRepo, nil, nil)
+	ennoblementSvc := app.NewEnnoblementService(ennoblementRepo, nil, nil)
+	tribeSnapshotSvc := app.NewTribeSnapshotService(tribeSnapshotRepo, nil, nil)
+	playerSnapshotSvc := app.NewPlayerSnapshotService(playerSnapshotRepo, nil, nil)
+	dataCleanupSvc := app.NewDataCleanupService(serverSvc, dataCleanupPublisher)
+
+	watermilltest.RunRouterWithContext(
+		t,
+		ctx,
+		port.NewTribeWatermillConsumer(
+			nil,
+			tribeSnapshotSvc,
+			tribeSub,
+			nopLogger,
+			marshaler,
+			"",
+			"",
+			"",
+			cmdCleanUpTopic,
+		),
+		port.NewPlayerWatermillConsumer(
+			nil,
+			playerSnapshotSvc,
+			playerSub,
+			nopLogger,
+			marshaler,
+			"",
+			"",
+			cmdCleanUpTopic,
+		),
+		port.NewEnnoblementWatermillConsumer(ennoblementSvc, ennoblementSub, nopLogger, marshaler, "", cmdCleanUpTopic),
+	)
+
+	require.NoError(t, dataCleanupSvc.CleanUp(ctx))
+
+	var wg sync.WaitGroup
+
+	var expectedEnnoblements []int
+	readJSONFile(t, filesys, path.Join("expected", "ennoblements.json"), &expectedEnnoblements)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		assert.EventuallyWithTf(t, func(collect *assert.CollectT) {
+			var ids []int
+
+			params := domain.NewListEnnoblementsParams()
+			require.NoError(collect, params.SetSort([]domain.EnnoblementSort{
+				domain.EnnoblementSortServerKeyASC,
+				domain.EnnoblementSortCreatedAtASC,
+				domain.EnnoblementSortIDASC,
+			}))
+
+			for {
+				res, err := ennoblementRepo.List(ctx, params)
+				require.NoError(collect, err)
+
+				for _, e := range res.Ennoblements() {
+					ids = append(ids, e.ID())
+				}
+
+				if res.Next().IsZero() {
+					break
+				}
+
+				require.NoError(collect, params.SetCursor(res.Next()))
+			}
+
+			assert.Equal(collect, expectedEnnoblements, ids)
+		}, 30*time.Second, 500*time.Millisecond, "ennoblements")
+	}()
+
+	var expectedTribeSnapshots []int
+	readJSONFile(t, filesys, path.Join("expected", "tribe-snapshots.json"), &expectedTribeSnapshots)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		assert.EventuallyWithTf(t, func(collect *assert.CollectT) {
+			var ids []int
+
+			params := domain.NewListTribeSnapshotsParams()
+			require.NoError(collect, params.SetSort([]domain.TribeSnapshotSort{
+				domain.TribeSnapshotSortServerKeyASC,
+				domain.TribeSnapshotSortDateASC,
+				domain.TribeSnapshotSortIDASC,
+			}))
+
+			for {
+				res, err := tribeSnapshotRepo.List(ctx, params)
+				require.NoError(collect, err)
+
+				for _, ts := range res.TribeSnapshots() {
+					ids = append(ids, ts.ID())
+				}
+
+				if res.Next().IsZero() {
+					break
+				}
+
+				require.NoError(collect, params.SetCursor(res.Next()))
+			}
+
+			assert.Equal(collect, expectedTribeSnapshots, ids)
+		}, 30*time.Second, 500*time.Millisecond, "tribe snapshots")
+	}()
+
+	var expectedPlayerSnapshots []int
+	readJSONFile(t, filesys, path.Join("expected", "player-snapshots.json"), &expectedPlayerSnapshots)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		assert.EventuallyWithTf(t, func(collect *assert.CollectT) {
+			var ids []int
+
+			params := domain.NewListPlayerSnapshotsParams()
+			require.NoError(collect, params.SetSort([]domain.PlayerSnapshotSort{
+				domain.PlayerSnapshotSortServerKeyASC,
+				domain.PlayerSnapshotSortDateASC,
+				domain.PlayerSnapshotSortIDASC,
+			}))
+
+			for {
+				res, err := playerSnapshotRepo.List(ctx, params)
+				require.NoError(collect, err)
+
+				for _, ps := range res.PlayerSnapshots() {
+					ids = append(ids, ps.ID())
+				}
+
+				if res.Next().IsZero() {
+					break
+				}
+
+				require.NoError(collect, params.SetCursor(res.Next()))
+			}
+
+			assert.Equal(collect, expectedPlayerSnapshots, ids)
+		}, 30*time.Second, 500*time.Millisecond, "player snapshots")
+	}()
+
+	wg.Wait()
+}
--- a/internal/port/testdata/cleanup/expected/ennoblements.json
+++ b/internal/port/testdata/cleanup/expected/ennoblements.json
@ -0,0 +1 @@
+[220041,2200407,2200408,22004089]
--- a/internal/port/testdata/cleanup/expected/player-snapshots.json
+++ b/internal/port/testdata/cleanup/expected/player-snapshots.json
@ -0,0 +1 @@
+[100001,1000000,1000001,10000000]
--- a/internal/port/testdata/cleanup/expected/tribe-snapshots.json
+++ b/internal/port/testdata/cleanup/expected/tribe-snapshots.json
@ -0,0 +1 @@
+[10001,100000,100001,1000000]
--- a/internal/port/testdata/cleanup/fixture.yml
+++ b/internal/port/testdata/cleanup/fixture.yml
@ -0,0 +1,405 @@
+- model: Server
+  rows:
+    - key: pl181 # data from this server should be cleaned up
+      url: https://pl181.plemiona.pl/
+      open: false
+      special: false
+      num_players: 144
+      num_tribes: 26
+      num_villages: 59581
+      num_player_villages: 58558
+      num_barbarian_villages: 1023
+      num_bonus_villages: 3715
+      created_at: 2022-09-30T08:00:04.504556Z
+      version_code: pl
+    - key: pl182 # data from this server shouldn't be cleaned up because open=true
+      url: https://pl182.plemiona.pl/
+      open: true
+      special: false
+      num_players: 144
+      num_tribes: 26
+      num_villages: 39702
+      num_player_villages: 39697
+      num_barbarian_villages: 5
+      num_bonus_villages: 2086
+      created_at: 2022-09-30T08:00:04.504556Z
+      version_code: pl
+    - key: pl185 # data from this server should not be cleaned because of a recent data update
+      url: https://pl185.plemiona.pl/
+      open: false
+      special: false
+      num_players: 458
+      num_tribes: 88
+      num_villages: 84069
+      num_player_villages: 83057
+      num_barbarian_villages: 1012
+      num_bonus_villages: 13379
+      created_at:
+      version_code: pl
+      player_data_updated_at: "{{ now }}"
+      player_snapshots_created_at: "{{ now }}"
+      tribe_data_updated_at: "{{ now }}"
+      tribe_snapshots_created_at: "{{ now }}"
+      village_data_updated_at: "{{ now }}"
+      ennoblement_data_updated_at: "{{ now }}"
+- model: Ennoblement
+  rows:
+    # pl181
+    - id: 220040
+      server_key: pl181
+      village_id: 1112
+      new_owner_id: 848928624
+      new_tribe_id: 0
+      old_owner_id: 0
+      old_tribe_id: 0
+      points: 9505
+      created_at: 2023-03-25T05:43:43Z
+    - id: 220041
+      server_key: pl181
+      village_id: 1112
+      new_owner_id: 699396429
+      new_tribe_id: 1312
+      old_owner_id: 0
+      old_tribe_id: 0
+      points: 9392
+      created_at: "{{ now }}"
+    # pl182
+    - id: 2200407
+      server_key: pl182
+      village_id: 1112
+      new_owner_id: 1179
+      new_tribe_id: 181
+      old_owner_id: 0
+      old_tribe_id: 0
+      points: 123
+      created_at: 2022-09-20T06:25:55Z
+    - id: 2200408
+      server_key: pl182
+      village_id: 1112
+      new_owner_id: 698496770
+      new_tribe_id: 145
+      old_owner_id: 698593426
+      old_tribe_id: 145
+      points: 4754
+      created_at: "{{ now }}"
+    # pl185
+    - id: 22004089
+      server_key: pl185
+      village_id: 1112
+      new_owner_id: 698321605
+      new_tribe_id: 489
+      old_owner_id: 699779452
+      old_tribe_id: 489
+      points: 3999
+      created_at: 2023-03-22T07:43:02Z
+- model: Tribe
+  rows:
+    # pl181
+    - rank_att: 1
+      score_att: 730488339
+      rank_def: 1
+      score_def: 303300128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 1
+      score_total: 1033788467
+      id: 1
+      server_key: pl181
+      name: Obdachlose Otter
+      tag: ObOtt
+      num_members: 20
+      num_villages: 8419
+      points: 86151161
+      all_points: 86151161
+      rank: 1
+      dominance: 58.367997781475324
+      created_at: 2021-03-07T11:00:51.000Z
+      profile_url: https://de188.die-staemme.de/game.php?screen=info_ally&id=772
+    # pl182
+    - rank_att: 1
+      score_att: 730488339
+      rank_def: 1
+      score_def: 303300128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 1
+      score_total: 1033788467
+      id: 1
+      server_key: pl182
+      name: Obdachlose Otter
+      tag: ObOtt
+      num_members: 20
+      num_villages: 8419
+      points: 86151161
+      all_points: 86151161
+      rank: 1
+      dominance: 58.367997781475324
+      created_at: 2021-03-07T11:00:51.000Z
+      profile_url: https://de188.die-staemme.de/game.php?screen=info_ally&id=772
+    # pl185
+    - rank_att: 1
+      score_att: 730488339
+      rank_def: 1
+      score_def: 303300128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 1
+      score_total: 1033788467
+      id: 1
+      server_key: pl185
+      name: Obdachlose Otter
+      tag: ObOtt
+      num_members: 20
+      num_villages: 8419
+      points: 86151161
+      all_points: 86151161
+      rank: 1
+      dominance: 58.367997781475324
+      created_at: 2021-03-07T11:00:51.000Z
+      profile_url: https://de188.die-staemme.de/game.php?screen=info_ally&id=772
+- model: Player
+  rows:
+    # pl181
+    - rank_att: 10
+      score_att: 27638462
+      rank_def: 35
+      score_def: 5555868
+      rank_sup: 3
+      score_sup: 9815277
+      rank_total: 16
+      score_total: 43009607
+      id: 698152377
+      server_key: pl181
+      name: 1scoo
+      num_villages: 878
+      points: 9199775
+      rank: 1
+      tribe_id: 772
+      created_at: 2021-06-25T11:00:53Z
+      profile_url: https://www.internalbenchmark.com/e-markets/proactive/user-centric
+    # pl182
+    - rank_att: 10
+      score_att: 27638462
+      rank_def: 35
+      score_def: 5555868
+      rank_sup: 3
+      score_sup: 9815277
+      rank_total: 16
+      score_total: 43009607
+      id: 2483601
+      server_key: pl182
+      name: 1scoo
+      num_villages: 878
+      points: 9199775
+      rank: 1
+      tribe_id: 772
+      created_at: 2021-06-25T11:00:53Z
+      profile_url: https://www.internalbenchmark.com/e-markets/proactive/user-centric
+    # pl185
+    - rank_att: 10
+      score_att: 27638462
+      rank_def: 35
+      score_def: 5555868
+      rank_sup: 3
+      score_sup: 9815277
+      rank_total: 16
+      score_total: 43009607
+      id: 7523589
+      server_key: pl185
+      name: 1scoo
+      num_villages: 878
+      points: 9199775
+      rank: 1
+      tribe_id: 772
+      created_at: 2021-06-25T11:00:53Z
+      profile_url: https://www.internalbenchmark.com/e-markets/proactive/user-centric
+- model: PlayerSnapshot
+  rows:
+    # pl181
+    - rank_att: 5
+      score_att: 38213157
+      rank_def: 2
+      score_def: 51137731
+      rank_sup: 104
+      score_sup: 2282683
+      rank_total: 1
+      score_total: 91633571
+      id: 100000
+      player_id: 698152377
+      server_key: pl181
+      num_villages: 665
+      points: 7298055
+      rank: 1
+      tribe_id: 47
+      date: 2023-09-01T00:00:00.000Z
+      created_at: 2022-09-17T06:03:40Z
+    - rank_att: 5
+      score_att: 38213157
+      rank_def: 2
+      score_def: 51137731
+      rank_sup: 104
+      score_sup: 2282683
+      rank_total: 1
+      score_total: 91633571
+      id: 100001
+      player_id: 698152377
+      server_key: pl181
+      num_villages: 665
+      points: 7298055
+      rank: 1
+      tribe_id: 47
+      date: "{{ now }}"
+      created_at: "{{ now }}"
+    # pl182
+    - rank_att: 5
+      score_att: 38213157
+      rank_def: 2
+      score_def: 51137731
+      rank_sup: 104
+      score_sup: 2282683
+      rank_total: 1
+      score_total: 91633571
+      id: 1000000
+      player_id: 2483601
+      server_key: pl182
+      num_villages: 665
+      points: 7298055
+      rank: 1
+      tribe_id: 65
+      date: 2023-09-01T00:00:00.000Z
+      created_at: 2022-09-17T06:03:40Z
+    - rank_att: 5
+      score_att: 38213157
+      rank_def: 2
+      score_def: 51137731
+      rank_sup: 104
+      score_sup: 2282683
+      rank_total: 1
+      score_total: 91633571
+      id: 1000001
+      player_id: 2483601
+      server_key: pl182
+      num_villages: 665
+      points: 7298055
+      rank: 1
+      tribe_id: 65
+      date: "{{ now }}"
+      created_at: "{{ now }}"
+    # pl185
+    - rank_att: 5
+      score_att: 38213157
+      rank_def: 2
+      score_def: 51137731
+      rank_sup: 104
+      score_sup: 2282683
+      rank_total: 1
+      score_total: 91633571
+      id: 10000000
+      player_id: 7523589
+      server_key: pl185
+      num_villages: 665
+      points: 7298055
+      rank: 1
+      tribe_id: 1768
+      date: "{{ now }}"
+      created_at: "{{ now }}"
+- model: TribeSnapshot
+  rows:
+    # pl181
+    - rank_att: 1
+      score_att: 669292167
+      rank_def: 5
+      score_def: 199124128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 2
+      score_total: 868416295
+      id: 10000
+      tribe_id: 1
+      server_key: pl181
+      num_members: 66
+      num_villages: 14480
+      points: 114277979
+      all_points: 143350058
+      rank: 1
+      dominance: 29.76912481240106
+      date: 2023-09-01T00:00:00.000Z
+      created_at: 2022-09-17T06:03:40Z
+    - rank_att: 1
+      score_att: 669292167
+      rank_def: 5
+      score_def: 199124128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 2
+      score_total: 868416295
+      id: 10001
+      tribe_id: 1
+      server_key: pl181
+      num_members: 66
+      num_villages: 14480
+      points: 114277979
+      all_points: 143350058
+      rank: 1
+      dominance: 29.76912481240106
+      date: "{{ now }}"
+      created_at: "{{ now }}"
+    # pl182
+    - rank_att: 1
+      score_att: 669292167
+      rank_def: 5
+      score_def: 199124128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 2
+      score_total: 868416295
+      id: 100000
+      tribe_id: 1
+      server_key: pl182
+      num_members: 66
+      num_villages: 14480
+      points: 114277979
+      all_points: 143350058
+      rank: 1
+      dominance: 29.76912481240106
+      date: 2023-09-01T00:00:00.000Z
+      created_at: 2022-09-17T06:03:40Z
+    - rank_att: 1
+      score_att: 669292167
+      rank_def: 5
+      score_def: 199124128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 2
+      score_total: 868416295
+      id: 100001
+      tribe_id: 1
+      server_key: pl182
+      num_members: 66
+      num_villages: 14480
+      points: 114277979
+      all_points: 143350058
+      rank: 1
+      dominance: 29.76912481240106
+      date: "{{ now }}"
+      created_at: "{{ now }}"
+      # pl185
+    - rank_att: 1
+      score_att: 669292167
+      rank_def: 5
+      score_def: 199124128
+      rank_sup: 0
+      score_sup: 0
+      rank_total: 2
+      score_total: 868416295
+      id: 1000000
+      tribe_id: 1
+      server_key: pl185
+      num_members: 66
+      num_villages: 14480
+      points: 114277979
+      all_points: 143350058
+      rank: 1
+      dominance: 29.76912481240106
+      date: 2023-09-01T00:00:00.000Z
+      created_at: 2022-09-17T06:03:40Z