twhelp/dcbot
1
0
Fork 0
Code Issues 1 Pull Requests 8 Packages Projects Releases Wiki Activity

feat: auto clean up old groups (#38)
All checks were successful
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/tag Build is passing
Details

Browse Source 
Reviewed-on: #38
This commit is contained in:
Dawid Wysokiński 2022-10-31 05:52:20 +00:00
parent 2c6dcda547
commit 5e99f68a91
12 changed files with 532 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/dcbot/internal/run/run.go
View File

@ -47,7 +47,12 @@ func New() *cli.Command {
monitorRepo := bundb.NewMonitor(db)
choiceSvc := service.NewChoice(client)
groupSvc := service.NewGroup(groupRepo, client, cfg.MaxGroupsPerServer)
groupSvc := service.NewGroup(
groupRepo,
client,
logger,
cfg.MaxGroupsPerServer,
)
monitorSvc := service.NewMonitor(
monitorRepo,
groupRepo,

28
internal/bundb/group.go
View File

@ -130,6 +130,22 @@ func (g *Group) Delete(ctx context.Context, id, serverID string) error {
return nil
}
func (g *Group) DeleteMany(ctx context.Context, ids ...string) error {
if len(ids) == 0 {
return nil
}
_, err := g.db.NewDelete().
Model(&model.Group{}).
Returning("NULL").
Where("id IN (?)", bun.In(ids)).
Exec(ctx)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("couldn't delete groups: %w", err)
}
return nil
}
type updateGroupsParamsApplier struct {
params domain.UpdateGroupParams
}
@ -182,5 +198,17 @@ func (l listGroupsParamsApplier) apply(q *bun.SelectQuery) *bun.SelectQuery {
}
}
if l.params.VersionCode.Valid {
q = q.Where("version_code = ?", l.params.VersionCode.String)
}
if l.params.ServerKeys != nil {
q = q.Where("server_key IN (?)", bun.In(l.params.ServerKeys))
}
if !l.params.CreatedAtLTE.IsZero() {
q = q.Where("created_at <= ?", l.params.CreatedAtLTE)
}
return q
}

82
internal/bundb/group_test.go
View File

@ -211,6 +211,44 @@ func TestGroup_List(t *testing.T) {
"abeb6c8e-70b6-445c-989f-890cd2a1f87a",
},
},
{
name: "VersionCode=pl",
params: domain.ListGroupsParams{
VersionCode: domain.NullString{
String: "pl",
Valid: true,
},
},
expectedGroups: []string{
"d56ad37f-2637-48ea-98f8-79627f3fcc96",
"429b790e-7186-4106-b531-4cc4931ce2ba",
"abeb6c8e-70b6-445c-989f-890cd2a1f87a",
"0be82203-4ca3-4b4c-a0c8-3a70099d88f7",
"982a9765-471c-43e8-9abb-1e4ee4f03738",
},
},
{
name: "VersionCode=pl,ServerKeys=[pl180]",
params: domain.ListGroupsParams{
VersionCode: domain.NullString{
String: "pl",
Valid: true,
},
ServerKeys: []string{"pl180"},
},
expectedGroups: []string{
"982a9765-471c-43e8-9abb-1e4ee4f03738",
},
},
{
name: "CreatedAtLTE=2022-03-15T15:00:10.000Z",
params: domain.ListGroupsParams{
CreatedAtLTE: time.Date(2022, time.March, 15, 15, 00, 10, 0, time.UTC),
},
expectedGroups: []string{
"d56ad37f-2637-48ea-98f8-79627f3fcc96",
},
},
}
for _, tt := range tests {
@ -363,11 +401,53 @@ func TestGroup_Delete(t *testing.T) {
})
}
func TestGroup_DeleteMany(t *testing.T) {
t.Parallel()
db := newDB(t)
fixture := loadFixtures(t, db)
groupRepo := bundb.NewGroup(db)
monitorRepo := bundb.NewMonitor(db)
group1 := getGroupFromFixture(t, fixture, "group-1-server-1")
group2 := getGroupFromFixture(t, fixture, "group-2-server-1")
ids := []string{group1.ID.String(), group2.ID.String()}
serverIDs := []string{group1.ServerID}
t.Run("OK", func(t *testing.T) {
t.Parallel()
beforeDelete, err := groupRepo.List(context.Background(), domain.ListGroupsParams{
ServerIDs: serverIDs,
})
assert.NoError(t, err)
assert.NoError(t, groupRepo.DeleteMany(context.Background(), ids...))
afterDelete, err := groupRepo.List(context.Background(), domain.ListGroupsParams{
ServerIDs: serverIDs,
})
assert.NoError(t, err)
assert.Len(t, afterDelete, len(beforeDelete)-len(ids))
// monitors should also be deleted
for _, id := range ids {
monitors, err := monitorRepo.List(context.Background(), id)
assert.NoError(t, err)
assert.Len(t, monitors, 0)
}
})
t.Run("OK: 0 ids", func(t *testing.T) {
t.Parallel()
assert.NoError(t, groupRepo.DeleteMany(context.Background()))
})
}
func getAllGroupsFromFixture(tb testing.TB, fixture *dbfixture.Fixture) []model.Group {
tb.Helper()
//nolint:lll
ids := []string{"group-1-server-1", "group-2-server-1", "group-1-server-2", "group-2-server-2"}
ids := []string{"group-1-server-1", "group-2-server-1", "group-1-server-2", "group-2-server-2", "group-3-server-1", "group-3-server-2"}
groups := make([]model.Group, 0, len(ids))
for _, id := range ids {

16
internal/bundb/testdata/fixture.yml vendored
View File

@ -28,6 +28,22 @@
channel_gains: 1555
channel_losses: 1235
created_at: 2022-03-19T15:03:10.000Z
- _id: group-3-server-1
id: 982a9765-471c-43e8-9abb-1e4ee4f03738
server_id: server-3
server_key: pl180
version_code: pl
channel_gains: 1555
channel_losses: 1235
created_at: 2022-03-23T18:03:10.000Z
- _id: group-3-server-2
id: f3a0a5d9-07fe-4770-8b43-79cbe10bb310
server_id: server-3
server_key: en130
version_code: en
channel_gains: 1555
channel_losses: 1235
created_at: 2022-04-23T18:03:10.000Z
- model: Monitor
rows:
- _id: monitor-1-group-1-server-1

28
internal/discord/bot.go
View File

@ -17,6 +17,7 @@ type GroupService interface {
SetChannelLosses(ctx context.Context, id, serverID, channel string) (domain.Group, error)
SetInternals(ctx context.Context, id, serverID string, internals bool) (domain.Group, error)
SetBarbarians(ctx context.Context, id, serverID string, barbarians bool) (domain.Group, error)
CleanUp(ctx context.Context) error
List(ctx context.Context, params domain.ListGroupsParams) ([]domain.Group, error)
Delete(ctx context.Context, id, serverID string) error
}
@ -122,14 +123,27 @@ func (b *Bot) registerCommand(cmd command) error {
}
func (b *Bot) initCron() error {
_, err := b.c.AddJob("@every 1m", &executeMonitorsJob{
svc: b.monitorSvc,
s: b.s,
logger: b.logger,
})
if err != nil {
return err
jobs := []struct {
spec string
job cron.Job
}{
{
spec: "@every 1m",
job: &executeMonitorsJob{svc: b.monitorSvc, s: b.s, logger: b.logger},
},
{
spec: "0 */8 * * *",
job: &cleanUpGroupsJob{svc: b.groupSvc, logger: b.logger},
},
}
for _, j := range jobs {
_, err := b.c.AddJob(j.spec, j.job)
if err != nil {
return err
}
}
return nil
}

30
internal/discord/job_groups_cleanup.go Normal file
View File

@ -0,0 +1,30 @@
package discord
import (
"context"
"time"
"go.uber.org/zap"
)
const (
cleanUpGroupsJobTimeout = 10 * time.Minute
)
type cleanUpGroupsJob struct {
svc GroupService
logger *zap.Logger
}
func (j *cleanUpGroupsJob) Run() {
ctx, cancel := context.WithTimeout(context.Background(), cleanUpGroupsJobTimeout)
defer cancel()
start := time.Now()
if err := j.svc.CleanUp(ctx); err != nil {
j.logger.Error("something went wrong while deleting old groups", zap.Error(err))
return
}
j.logger.Info("old groups have been deleted", zap.Duration("duration", time.Since(start)))
}

5
internal/domain/group.go
View File

@ -106,8 +106,11 @@ func (u UpdateGroupParams) IsZero() bool {
}
type ListGroupsParams struct {
ServerIDs []string
ServerIDs []string // DC server IDs
VersionCode NullString
ServerKeys []string
EnabledNotifications NullBool // check if ChannelGains != null && ChannelLosses != null
CreatedAtLTE time.Time
}
type GroupLimitReachedError struct {

134
internal/service/group.go
View File

@ -4,9 +4,11 @@ import (
"context"
"errors"
"fmt"
"time"
"gitea.dwysokinski.me/twhelp/dcbot/internal/domain"
"gitea.dwysokinski.me/twhelp/dcbot/internal/twhelp"
"go.uber.org/zap"
)
//counterfeiter:generate -o internal/mock/group_repository.gen.go . GroupRepository
@ -16,18 +18,21 @@ type GroupRepository interface {
List(ctx context.Context, params domain.ListGroupsParams) ([]domain.Group, error)
Get(ctx context.Context, id, serverID string) (domain.Group, error)
Delete(ctx context.Context, id, serverID string) error
DeleteMany(ctx context.Context, id ...string) error
}
type Group struct {
repo GroupRepository
client TWHelpClient
logger *zap.Logger
maxGroupsPerServer int
}
func NewGroup(repo GroupRepository, client TWHelpClient, maxGroupsPerServer int) *Group {
func NewGroup(repo GroupRepository, client TWHelpClient, logger *zap.Logger, maxGroupsPerServer int) *Group {
return &Group{
repo: repo,
client: client,
logger: logger,
maxGroupsPerServer: maxGroupsPerServer,
}
}
@ -59,6 +64,29 @@ func (g *Group) Create(ctx context.Context, params domain.CreateGroupParams) (do
return group, nil
}
func (g *Group) checkTWServer(ctx context.Context, versionCode, serverKey string) error {
server, err := g.client.GetServer(ctx, versionCode, serverKey)
if err != nil {
var apiErr twhelp.APIError
if !errors.As(err, &apiErr) {
return fmt.Errorf("TWHelpClient.GetServer: %w", err)
}
return domain.ServerDoesNotExistError{
VersionCode: versionCode,
Key: serverKey,
}
}
if !server.Open {
return domain.ServerIsClosedError{
VersionCode: versionCode,
Key: serverKey,
}
}
return nil
}
func (g *Group) SetChannelGains(ctx context.Context, id, serverID, channel string) (domain.Group, error) {
return g.update(ctx, id, serverID, domain.UpdateGroupParams{
ChannelGains: domain.NullString{
@ -126,23 +154,97 @@ func (g *Group) Delete(ctx context.Context, id, serverID string) error {
return nil
}
func (g *Group) checkTWServer(ctx context.Context, versionCode, serverKey string) error {
server, err := g.client.GetServer(ctx, versionCode, serverKey)
if err != nil {
var apiErr twhelp.APIError
if !errors.As(err, &apiErr) {
return fmt.Errorf("TWHelpClient.GetServer: %w", err)
}
return domain.ServerDoesNotExistError{
VersionCode: versionCode,
Key: serverKey,
}
func (g *Group) CleanUp(ctx context.Context) error {
if err := g.deleteAllWithDisabledNotifications(ctx); err != nil {
return err
}
if !server.Open {
return domain.ServerIsClosedError{
VersionCode: versionCode,
Key: serverKey,
if err := g.deleteAllWithClosedTWServers(ctx); err != nil {
return err
}
return nil
}
func (g *Group) deleteAllWithDisabledNotifications(ctx context.Context) error {
groups, err := g.repo.List(ctx, domain.ListGroupsParams{
EnabledNotifications: domain.NullBool{
Bool: false,
Valid: true,
},
CreatedAtLTE: time.Now().Add(-24 * time.Hour),
})
if err != nil {
return fmt.Errorf("GroupRepository.List: %w", err)
}
ids := make([]string, 0, len(groups))
for _, group := range groups {
ids = append(ids, group.ID)
}
if err = g.repo.DeleteMany(ctx, ids...); err != nil {
return fmt.Errorf("GroupRepository.DeleteMany: %w", err)
}
return nil
}
func (g *Group) deleteAllWithClosedTWServers(ctx context.Context) error {
versions, err := g.client.ListVersions(ctx)
if err != nil {
return fmt.Errorf("TWHelpClient.ListVersions: %w", err)
}
for _, v := range versions {
servers, err := g.client.ListServers(ctx, v.Code, twhelp.ListServersQueryParams{
Open: twhelp.NullBool{
Bool: false,
Valid: true,
},
})
if err != nil {
g.logger.Warn("failed to fetch closed servers", zap.Error(err), zap.String("versionCode", v.Code))
continue
}
if len(servers) == 0 {
continue
}
params := domain.ListGroupsParams{
VersionCode: domain.NullString{
String: v.Code,
Valid: true,
},
ServerKeys: make([]string, 0, len(servers)),
}
for _, s := range servers {
params.ServerKeys = append(params.ServerKeys, s.Key)
}
groups, err := g.repo.List(ctx, params)
if err != nil {
g.logger.Warn("failed to list groups", zap.Error(err), zap.String("versionCode", v.Code))
continue
}
if len(groups) == 0 {
continue
}
ids := make([]string, 0, len(groups))
for _, group := range groups {
ids = append(ids, group.ID)
}
if err = g.repo.DeleteMany(ctx, ids...); err != nil {
g.logger.Warn(
"failed to delete groups",
zap.Error(err),
zap.String("versionCode", v.Code),
)
continue
}
}

172
internal/service/group_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func TestGroup_Create(t *testing.T) {
@ -57,7 +58,7 @@ func TestGroup_Create(t *testing.T) {
}, nil
})
g, err := service.NewGroup(repo, client, 1).Create(context.Background(), params)
g, err := service.NewGroup(repo, client, zap.NewNop(), 1).Create(context.Background(), params)
assert.NoError(t, err)
assert.NotEmpty(t, g.ID)
assert.Equal(t, params.ServerID(), g.ServerID)
@ -78,7 +79,7 @@ func TestGroup_Create(t *testing.T) {
repo := &mock.FakeGroupRepository{}
repo.ListReturns(make([]domain.Group, maxGroupsPerServer), nil)
g, err := service.NewGroup(repo, nil, maxGroupsPerServer).Create(context.Background(), params)
g, err := service.NewGroup(repo, nil, zap.NewNop(), maxGroupsPerServer).Create(context.Background(), params)
assert.ErrorIs(t, err, domain.GroupLimitReachedError{
Current: maxGroupsPerServer,
Limit: maxGroupsPerServer,
@ -100,7 +101,7 @@ func TestGroup_Create(t *testing.T) {
}
})
g, err := service.NewGroup(repo, client, 1).Create(context.Background(), params)
g, err := service.NewGroup(repo, client, zap.NewNop(), 1).Create(context.Background(), params)
assert.ErrorIs(t, err, domain.ServerDoesNotExistError{
VersionCode: params.VersionCode(),
Key: params.ServerKey(),
@ -123,7 +124,7 @@ func TestGroup_Create(t *testing.T) {
}, nil
})
g, err := service.NewGroup(repo, client, 1).Create(context.Background(), params)
g, err := service.NewGroup(repo, client, zap.NewNop(), 1).Create(context.Background(), params)
assert.ErrorIs(t, err, domain.ServerIsClosedError{
VersionCode: params.VersionCode(),
Key: params.ServerKey(),
@ -131,3 +132,166 @@ func TestGroup_Create(t *testing.T) {
assert.Zero(t, g)
})
}
func TestGroup_CleanUp(t *testing.T) {
t.Parallel()
t.Run("OK", func(t *testing.T) {
t.Parallel()
repo := &mock.FakeGroupRepository{}
groupsWithDisabledNotifications := []domain.Group{
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: "",
ChannelLosses: "",
Internals: true,
Barbarians: true,
ServerKey: "pl181",
VersionCode: "pl",
CreatedAt: time.Now().Add(-48 * time.Hour),
},
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: "",
ChannelLosses: "",
Internals: true,
Barbarians: true,
ServerKey: "pl181",
VersionCode: "pl",
CreatedAt: time.Now().Add(-45 * time.Hour),
},
}
repo.ListReturnsOnCall(0, groupsWithDisabledNotifications, nil)
groupsPL := []domain.Group{
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: uuid.NewString(),
ChannelLosses: "",
Internals: true,
Barbarians: true,
ServerKey: "pl181",
VersionCode: "pl",
CreatedAt: time.Now().Add(-120 * time.Hour),
},
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: uuid.NewString(),
ChannelLosses: uuid.NewString(),
Internals: true,
Barbarians: true,
ServerKey: "pl181",
VersionCode: "pl",
CreatedAt: time.Now().Add(-11 * time.Hour),
},
}
repo.ListReturnsOnCall(1, groupsPL, nil)
groupsUK := []domain.Group{
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: uuid.NewString(),
ChannelLosses: uuid.NewString(),
Internals: true,
Barbarians: true,
ServerKey: "uk51",
VersionCode: "uk",
CreatedAt: time.Now().Add(-150 * time.Hour),
},
{
ID: uuid.NewString(),
ServerID: uuid.NewString(),
ChannelGains: uuid.NewString(),
ChannelLosses: uuid.NewString(),
Internals: true,
Barbarians: true,
ServerKey: "uk52",
VersionCode: "uk",
CreatedAt: time.Now().Add(-200 * time.Hour),
},
}
repo.ListReturnsOnCall(2, groupsUK, nil)
repo.DeleteManyReturns(nil)
client := &mock.FakeTWHelpClient{}
versions := []twhelp.Version{
{
Code: "pl",
Host: "www.plemiona.pl",
Name: "Poland",
Timezone: "Europe/Warsaw",
},
{
Code: "uk",
Host: "www.tribalwars.co.uk",
Name: "United Kingdom",
Timezone: "Europe/London",
},
{
Code: "tr",
Host: "www.klanlar.org",
Name: "Turkey",
Timezone: "Europe/Istanbul",
},
}
client.ListVersionsReturns(versions, nil)
serversPL := []twhelp.Server{
{
Key: "pl181",
URL: "https://pl181.plemiona.pl",
Open: false,
},
}
client.ListServersReturnsOnCall(0, serversPL, nil)
serversUK := []twhelp.Server{
{
Key: "uk51",
URL: "https://uk51.tribalwars.co.uk",
Open: false,
},
{
Key: "uk52",
URL: "https://uk52.tribalwars.co.uk",
Open: false,
},
}
client.ListServersReturnsOnCall(1, serversUK, nil)
client.ListServersReturnsOnCall(1, serversUK, nil)
client.ListServersReturnsOnCall(2, nil, nil) // Turkey
assert.NoError(t, service.NewGroup(repo, client, zap.NewNop(), 1).CleanUp(context.Background()))
require.Equal(t, 3, repo.ListCallCount())
require.Equal(t, 3, repo.DeleteManyCallCount())
for _, tt := range []struct {
i int
groups []domain.Group
}{
{
i: 0,
groups: groupsWithDisabledNotifications,
},
{
i: 1,
groups: groupsPL,
},
{
i: 2,
groups: groupsUK,
},
} {
_, ids := repo.DeleteManyArgsForCall(tt.i)
assert.Len(t, ids, len(tt.groups))
for i, id := range ids {
assert.Equal(t, tt.groups[i].ID, id)
}
}
require.Equal(t, 1, client.ListVersionsCallCount())
require.Equal(t, 3, client.ListServersCallCount())
})
}

5
internal/service/service.go
View File

@ -11,6 +11,11 @@ import (
//counterfeiter:generate -o internal/mock/twhelp_client.gen.go . TWHelpClient
type TWHelpClient interface {
ListVersions(ctx context.Context) ([]twhelp.Version, error)
ListServers(
ctx context.Context,
version string,
params twhelp.ListServersQueryParams,
) ([]twhelp.Server, error)
GetServer(ctx context.Context, version, server string) (twhelp.Server, error)
GetTribeByID(ctx context.Context, version, server string, id int64) (twhelp.Tribe, error)
GetTribeByTag(ctx context.Context, version, server, tag string) (twhelp.Tribe, error)

54
internal/twhelp/client.go
View File

@ -16,6 +16,7 @@ const (
defaultTimeout = 10 * time.Second
endpointListVersions = "/api/v1/versions"
endpointListServers = "/api/v1/versions/%s/servers"
endpointGetServer = "/api/v1/versions/%s/servers/%s"
endpointGetTribeByID = "/api/v1/versions/%s/servers/%s/tribes/%d"
endpointGetTribeByTag = "/api/v1/versions/%s/servers/%s/tribes/tag/%s"
@ -64,6 +65,39 @@ func (c *Client) ListVersions(ctx context.Context) ([]Version, error) {
return resp.Data, nil
}
type ListServersQueryParams struct {
Limit int32
Offset int32
Open NullBool
}
func (c *Client) ListServers(
ctx context.Context,
version string,
params ListServersQueryParams,
) ([]Server, error) {
q := url.Values{}
if params.Limit > 0 {
q.Set("limit", strconv.Itoa(int(params.Limit)))
}
if params.Offset > 0 {
q.Set("offset", strconv.Itoa(int(params.Offset)))
}
if params.Open.Valid {
q.Set("open", strconv.FormatBool(params.Open.Bool))
}
var resp listServersResp
if err := c.getJSON(ctx, fmt.Sprintf(endpointListServers, version)+"?"+q.Encode(), &resp); err != nil {
return nil, err
}
return resp.Data, nil
}
func (c *Client) GetServer(ctx context.Context, version, server string) (Server, error) {
var resp getServerResp
if err := c.getJSON(ctx, fmt.Sprintf(endpointGetServer, version, server), &resp); err != nil {
@ -105,22 +139,26 @@ type ListEnnoblementsQueryParams struct {
Sort []ListEnnoblementsSort
}
func (c *Client) ListEnnoblements(ctx context.Context, version, server string, queryParams ListEnnoblementsQueryParams) ([]Ennoblement, error) {
func (c *Client) ListEnnoblements(
ctx context.Context,
version, server string,
params ListEnnoblementsQueryParams,
) ([]Ennoblement, error) {
q := url.Values{}
if queryParams.Limit > 0 {
q.Set("limit", strconv.Itoa(int(queryParams.Limit)))
if params.Limit > 0 {
q.Set("limit", strconv.Itoa(int(params.Limit)))
}
if queryParams.Offset > 0 {
q.Set("offset", strconv.Itoa(int(queryParams.Offset)))
if params.Offset > 0 {
q.Set("offset", strconv.Itoa(int(params.Offset)))
}
if !queryParams.Since.IsZero() {
q.Set("since", queryParams.Since.Format(time.RFC3339))
if !params.Since.IsZero() {
q.Set("since", params.Since.Format(time.RFC3339))
}
for _, s := range queryParams.Sort {
for _, s := range params.Sort {
q.Add("sort", s.String())
}

9
internal/twhelp/twhelp.go
View File

@ -53,6 +53,10 @@ type getServerResp struct {
Data Server `json:"data"`
}
type listServersResp struct {
Data []Server `json:"data"`
}
type Tribe struct {
ID int64 `json:"id"`
Tag string `json:"tag"`
@ -152,3 +156,8 @@ type Ennoblement struct {
type listEnnoblementsResp struct {
Data []Ennoblement `json:"data"`
}
type NullBool struct {
Bool bool
Valid bool // Valid is true if Bool is not NULL
}