Merge pull request #4 from tribalwarshelp/bump-shared

Bump shared
This commit is contained in:
Dawid Wysokiński 2021-05-06 15:59:50 +02:00 committed by GitHub
commit e38704663c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 231 additions and 281 deletions

View File

@ -12,23 +12,23 @@ RUN go mod download
# Copy the source from the current directory to the Working Directory inside the container
COPY . .
RUN go build -o main .
RUN go build -o twcron .
######## Start a new stage from scratch #######
FROM alpine:latest
FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/
# Copy the Pre-built binary file from the previous stage
COPY --from=builder /app/main .
COPY --from=builder /app/twcron .
ENV MODE=production
ENV APP_MODE=production
EXPOSE 8080
RUN apk add --no-cache tzdata
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.2.1/wait ./wait
RUN chmod +x ./wait
CMD ./wait && ./main
CMD ./wait && ./twcron

View File

@ -3,13 +3,13 @@
Features:
- Adds automatically new servers.
- Fetches and updates servers data (players, tribes, ODA, ODD, ODS, OD, conquers, configs).
- Fetches and updates server data (players, tribes, ODA, ODD, ODS, OD, conquers, configs).
- Saves daily player/tribe stats, player/tribe history, tribe changes, player name changes, server stats.
- Cleans the database from old player/tribe stats, player/tribe history.
- Clears database from old player/tribe stats, player/tribe history.
## Development
**Required env variables:**
**Required ENV variables:**
```
DB_USER=your_db_user

7
go.mod
View File

@ -3,8 +3,9 @@ module github.com/tribalwarshelp/cron
go 1.16
require (
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9
github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83
github.com/Kichiyaki/appmode v0.0.0-20210502105643-0a26207c548d
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210502060056-ad595ba7b858
github.com/Kichiyaki/goutil v0.0.0-20210504132659-3d843a787db7
github.com/go-pg/pg/v10 v10.9.1
github.com/go-redis/redis/v8 v8.8.2
github.com/go-redis/redis_rate/v9 v9.1.1 // indirect
@ -13,7 +14,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.8.1
github.com/tribalwarshelp/shared v0.0.0-20210501144726-b8f533bc97be
github.com/tribalwarshelp/shared v0.0.0-20210505113409-79d8dc192942
github.com/vmihailenco/taskq/v3 v3.2.3
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
)

16
go.sum
View File

@ -2,10 +2,17 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9 h1:S/08K0AD4bXYeSPJKei8ZbumDy1JNARZsgYbNZgr9Tk=
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw=
github.com/Kichiyaki/appmode v0.0.0-20210502105643-0a26207c548d h1:ApX13STtfJc2YPH5D2JnBa6+4AM2vt7a81so/MPr/bA=
github.com/Kichiyaki/appmode v0.0.0-20210502105643-0a26207c548d/go.mod h1:41p1KTy/fiVocPnJR2h/iXh2NvWWVBdNoZrN8TWVXUI=
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210502060056-ad595ba7b858 h1:O7EPPY5lWaKbYB/5yJzE8WMesismUYuje7gOemo1UNo=
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210502060056-ad595ba7b858/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw=
github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM=
github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83/go.mod h1:+iGkf5HfOVeRVd9K7qQDucIl+/Kt3MyenMa90b/O/c4=
github.com/Kichiyaki/gopgutil/v10 v10.0.0-20210505093434-655fa2df248f h1:/kJmv8B59cMHdTmsko0RyAOeRC9WpsQPDmLtRXOAU6c=
github.com/Kichiyaki/gopgutil/v10 v10.0.0-20210505093434-655fa2df248f/go.mod h1:MSAEhr8oeK+Rhjhqyl31/8/AI88thYky80OyD8mheDA=
github.com/Kichiyaki/goutil v0.0.0-20210502095630-318d17091eab/go.mod h1:+HhI932Xb0xrCodNcCv5GPiCjLYhDxWhCtlEqMIJhB4=
github.com/Kichiyaki/goutil v0.0.0-20210504132659-3d843a787db7 h1:OU3ZA5H8fHTzaYIw9UBfH3gtWRL0XmnczlhH3E2PjV4=
github.com/Kichiyaki/goutil v0.0.0-20210504132659-3d843a787db7/go.mod h1:+HhI932Xb0xrCodNcCv5GPiCjLYhDxWhCtlEqMIJhB4=
github.com/aws/aws-sdk-go v1.35.28 h1:S2LuRnfC8X05zgZLC8gy/Sb82TGv2Cpytzbzz7tkeHc=
github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
github.com/bsm/redislock v0.7.0 h1:RL7aZJhCKkuBjQbnSTKCeedTRifBWxd/ffP+GZ599Mo=
@ -111,7 +118,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -122,8 +128,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
github.com/tribalwarshelp/shared v0.0.0-20210501144726-b8f533bc97be h1:NSdGqXTiWYk67US87sqE8sHlGEsL/dLIfQpb0jDPbTI=
github.com/tribalwarshelp/shared v0.0.0-20210501144726-b8f533bc97be/go.mod h1:CDQvesBYmSyGDl5X37xfa+ub55ZbikrHDuIZ4AcfM8I=
github.com/tribalwarshelp/shared v0.0.0-20210505113409-79d8dc192942 h1:4N3ajmOaZCFz04TpqtrcJnRquzSRsMCWlLwUYE926FA=
github.com/tribalwarshelp/shared v0.0.0-20210505113409-79d8dc192942/go.mod h1:GBnSKQrxL8Nmi3MViIzZVbyP9+ugd28gWArsSvw1iVU=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
github.com/vmihailenco/msgpack/v5 v5.0.0/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo=

View File

@ -6,9 +6,9 @@ import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/robfig/cron/v3"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/cron/internal/cron/queue"
"github.com/tribalwarshelp/cron/internal/cron/tasks"
@ -44,7 +44,7 @@ func New(cfg *Config) (*Cron, error) {
}
func (c *Cron) init() error {
var versions []*models.Version
var versions []*twmodel.Version
if err := c.db.Model(&versions).DistinctOn("timezone").Select(); err != nil {
return errors.Wrap(err, "Cron.init: couldn't load versions")
}

View File

@ -1,14 +1,13 @@
package tasks
import (
"github.com/tribalwarshelp/shared/tw/dataloader"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
"net/http"
"time"
"github.com/tribalwarshelp/shared/models"
)
func countPlayerVillages(villages []*models.Village) int {
func countPlayerVillages(villages []*twmodel.Village) int {
count := 0
for _, village := range villages {
if village.PlayerID != 0 {
@ -39,15 +38,15 @@ func newHTTPClient() *http.Client {
}
}
func newDataloader(url string) dataloader.DataLoader {
return dataloader.New(&dataloader.Config{
func newServerDataLoader(url string) twdataloader.ServerDataLoader {
return twdataloader.NewServerDataLoader(&twdataloader.ServerDataLoaderConfig{
BaseURL: url,
Client: newHTTPClient(),
})
}
type playersSearchableByID struct {
players []*models.Player
players []*twmodel.Player
}
func (searchable playersSearchableByID) getID(index int) int {
@ -59,7 +58,7 @@ func (searchable playersSearchableByID) len() int {
}
type tribesSearchableByID struct {
tribes []*models.Tribe
tribes []*twmodel.Tribe
}
func (searchable tribesSearchableByID) getID(index int) int {
@ -71,7 +70,7 @@ func (searchable tribesSearchableByID) len() int {
}
type villagesSearchableByID struct {
villages []*models.Village
villages []*twmodel.Village
}
func (searchable villagesSearchableByID) getID(index int) int {
@ -83,7 +82,7 @@ func (searchable villagesSearchableByID) len() int {
}
type ennoblementsSearchableByNewOwnerID struct {
ennoblements []*models.Ennoblement
ennoblements []*twmodel.Ennoblement
}
func (searchable ennoblementsSearchableByNewOwnerID) getID(index int) int {

View File

@ -2,9 +2,9 @@ package tasks
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/shared/tw/twurlbuilder"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -14,11 +14,11 @@ type taskDeleteNonExistentVillages struct {
}
func (t *taskDeleteNonExistentVillages) execute() error {
var servers []*models.Server
var servers []*twmodel.Server
err := t.db.
Model(&servers).
Relation("Version").
Where("status = ?", models.ServerStatusOpen).
Where("status = ?", twmodel.ServerStatusOpen).
Relation("Version").
Select()
if err != nil {
@ -36,7 +36,7 @@ func (t *taskDeleteNonExistentVillages) execute() error {
Get(TaskNameServerDeleteNonExistentVillages).
WithArgs(
context.Background(),
fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host),
twurlbuilder.BuildServerURL(server.Key, server.Version.Host),
s,
),
)

View File

@ -2,64 +2,70 @@ package tasks
import (
"context"
"fmt"
phpserialize "github.com/Kichiyaki/go-php-serialize"
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tribalwarshelp/shared/models"
"io/ioutil"
"net/http"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/cron/internal/cron/queue"
"github.com/tribalwarshelp/cron/internal/db"
)
const (
endpointGetServers = "/backend/get_servers.php"
"github.com/tribalwarshelp/cron/internal/postgres"
)
type taskLoadServersAndUpdateData struct {
*task
}
func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error {
type serverWithURL struct {
*twmodel.Server `pg:",inherit"`
url string
}
func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
if err := t.validatePayload(version); err != nil {
log.Debug(err)
return nil
}
entry := log.WithField("host", version.Host)
entry.Infof("taskLoadServersAndUpdateData.execute: %s: Loading servers", version.Host)
data, err := t.getServers(version)
loadedServers, err := twdataloader.
NewVersionDataLoader(&twdataloader.VersionDataLoaderConfig{
Host: version.Host,
Client: newHTTPClient(),
}).
LoadServers()
if err != nil {
log.Errorln(err)
return err
}
var serverKeys []string
var servers []*models.Server
for serverKey := range data {
if version.SpecialServers.Contains(serverKey) {
var servers []*serverWithURL
for _, loadedServer := range loadedServers {
if version.SpecialServers.Contains(loadedServer.Key) {
continue
}
server := &models.Server{
Key: serverKey,
Status: models.ServerStatusOpen,
server := &twmodel.Server{
Key: loadedServer.Key,
Status: twmodel.ServerStatusOpen,
VersionCode: version.Code,
Version: version,
}
if err := db.CreateSchema(t.db, server); err != nil {
if err := postgres.CreateSchema(t.db, server); err != nil {
logrus.Warn(errors.Wrapf(err, "taskLoadServersAndUpdateData.execute: %s: couldn't create the schema", server.Key))
continue
}
servers = append(servers, server)
serverKeys = append(serverKeys, serverKey)
servers = append(servers, &serverWithURL{
Server: server,
url: loadedServer.URL,
})
serverKeys = append(serverKeys, server.Key)
}
if len(servers) > 0 {
if _, err := t.db.Model(&servers).
OnConflict("(key) DO UPDATE").
Set("status = ?", models.ServerStatusOpen).
Set("status = ?", twmodel.ServerStatusOpen).
Set("version_code = EXCLUDED.version_code").
Returning("*").
Insert(); err != nil {
@ -69,8 +75,8 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error {
}
}
if _, err := t.db.Model(&models.Server{}).
Set("status = ?", models.ServerStatusClosed).
if _, err := t.db.Model(&twmodel.Server{}).
Set("status = ?", twmodel.ServerStatusClosed).
Where("key NOT IN (?) AND version_code = ?", pg.In(serverKeys), version.Code).
Update(); err != nil {
err = errors.Wrap(err, "taskLoadServersAndUpdateData.execute: couldn't update server statuses")
@ -79,44 +85,16 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error {
}
for _, server := range servers {
s := server
t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), data[s.Key], s))
t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), server.url, server.Server))
}
entry.Infof("%s: Servers have been loaded", version.Host)
return nil
}
func (t *taskLoadServersAndUpdateData) validatePayload(version *models.Version) error {
func (t *taskLoadServersAndUpdateData) validatePayload(version *twmodel.Version) error {
if version == nil {
return errors.New("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil")
return errors.New("taskLoadServersAndUpdateData.validatePayload: Expected *twmodel.Version, got nil")
}
return nil
}
func (t *taskLoadServersAndUpdateData) getServers(version *models.Version) (map[string]string, error) {
resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers))
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't load servers", version.Host)
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the response body", version.Host)
}
body, err := phpserialize.Decode(string(bodyBytes))
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the response body into the go value", version.Host)
}
result := make(map[string]string)
for serverKey, url := range body.(map[interface{}]interface{}) {
serverKeyStr := serverKey.(string)
urlStr := url.(string)
if serverKeyStr != "" && urlStr != "" {
result[serverKeyStr] = urlStr
}
}
return result, nil
}

View File

@ -3,7 +3,7 @@ package tasks
import (
"context"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -13,7 +13,7 @@ type taskLoadVersionsAndUpdateServerData struct {
}
func (t *taskLoadVersionsAndUpdateServerData) execute() error {
var versions []*models.Version
var versions []*twmodel.Version
log.Debug("taskLoadVersionsAndUpdateServerData.execute: Loading versions...")
if err := t.db.Model(&versions).Relation("SpecialServers").Select(); err != nil {
err = errors.Wrap(err, "taskLoadVersionsAndUpdateServerData.execute: couldn't load versions")

View File

@ -3,15 +3,15 @@ package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/dataloader"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
)
type taskServerDeleteNonExistentVillages struct {
*task
}
func (t *taskServerDeleteNonExistentVillages) execute(url string, server *models.Server) error {
func (t *taskServerDeleteNonExistentVillages) execute(url string, server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -20,7 +20,7 @@ func (t *taskServerDeleteNonExistentVillages) execute(url string, server *models
entry.Infof("taskServerDeleteNonExistentVillages.execute: %s: Deleting non-existent villages...", server.Key)
err := (&workerDeleteNonExistentVillages{
db: t.db.WithParam("SERVER", pg.Safe(server.Key)),
dataloader: newDataloader(url),
dataloader: newServerDataLoader(url),
server: server,
}).delete()
if err != nil {
@ -32,9 +32,9 @@ func (t *taskServerDeleteNonExistentVillages) execute(url string, server *models
return nil
}
func (t *taskServerDeleteNonExistentVillages) validatePayload(server *models.Server) error {
func (t *taskServerDeleteNonExistentVillages) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil")
return errors.New("taskUpdateServerData.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -42,8 +42,8 @@ func (t *taskServerDeleteNonExistentVillages) validatePayload(server *models.Ser
type workerDeleteNonExistentVillages struct {
db *pg.DB
dataloader dataloader.DataLoader
server *models.Server
dataloader twdataloader.ServerDataLoader
server *twmodel.Server
}
func (w *workerDeleteNonExistentVillages) delete() error {
@ -54,9 +54,9 @@ func (w *workerDeleteNonExistentVillages) delete() error {
var idsToDelete []int
searchableByVillageID := &villagesSearchableByID{villages}
if err := w.db.
Model(&models.Village{}).
Model(&twmodel.Village{}).
Column("id").
ForEach(func(village *models.Village) error {
ForEach(func(village *twmodel.Village) error {
index := searchByID(searchableByVillageID, village.ID)
if index < 0 {
idsToDelete = append(idsToDelete, village.ID)
@ -68,7 +68,7 @@ func (w *workerDeleteNonExistentVillages) delete() error {
totalDeleted := 0
if len(idsToDelete) > 0 {
result, err := w.db.Model(&models.Village{}).Where("id = ANY(?)", pg.Array(idsToDelete)).Delete()
result, err := w.db.Model(&twmodel.Village{}).Where("id = ANY(?)", pg.Array(idsToDelete)).Delete()
if err != nil {
return errors.Wrap(err, "workerDeleteNonExistentVillages.delete")
}

View File

@ -2,9 +2,9 @@ package tasks
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/shared/tw/twurlbuilder"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -14,11 +14,11 @@ type taskUpdateEnnoblements struct {
}
func (t *taskUpdateEnnoblements) execute() error {
var servers []*models.Server
var servers []*twmodel.Server
err := t.db.
Model(&servers).
Relation("Version").
Where("status = ?", models.ServerStatusOpen).
Where("status = ?", twmodel.ServerStatusOpen).
Select()
if err != nil {
err = errors.Wrap(err, "taskUpdateEnnoblements.execute")
@ -31,7 +31,7 @@ func (t *taskUpdateEnnoblements) execute() error {
err := t.queue.Add(
queue.EnnoblementsQueue,
Get(TaskUpdateServerEnnoblements).
WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s),
WithArgs(context.Background(), twurlbuilder.BuildServerURL(server.Key, server.Version.Host), s),
)
if err != nil {
log.Warn(

View File

@ -3,7 +3,7 @@ package tasks
import (
"context"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
"github.com/tribalwarshelp/cron/internal/cron/queue"
@ -23,12 +23,12 @@ func (t *taskUpdateHistory) execute(timezone string) error {
}
year, month, day := time.Now().In(location).Date()
date := time.Date(year, month, day, 1, 30, 0, 0, location)
var servers []*models.Server
var servers []*twmodel.Server
err = t.db.
Model(&servers).
Where(
"status = ? AND (history_updated_at IS NULL OR history_updated_at < ?) AND timezone = ?",
models.ServerStatusOpen,
twmodel.ServerStatusOpen,
date,
timezone,
).

View File

@ -5,8 +5,8 @@ import (
"github.com/go-pg/pg/v10"
"github.com/go-pg/pg/v10/orm"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/dataloader"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
)
@ -14,7 +14,7 @@ type taskUpdateServerData struct {
*task
}
func (t *taskUpdateServerData) execute(url string, server *models.Server) error {
func (t *taskUpdateServerData) execute(url string, server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -24,7 +24,7 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error
entry.Infof("taskUpdateServerData.execute: %s: Update of the server data has started...", server.Key)
err := (&workerUpdateServerData{
db: t.db.WithParam("SERVER", pg.Safe(server.Key)),
dataloader: newDataloader(url),
dataloader: newServerDataLoader(url),
server: server,
}).update()
if err != nil {
@ -42,9 +42,9 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error
return nil
}
func (t *taskUpdateServerData) validatePayload(server *models.Server) error {
func (t *taskUpdateServerData) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil")
return errors.New("taskUpdateServerData.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -52,20 +52,20 @@ func (t *taskUpdateServerData) validatePayload(server *models.Server) error {
type workerUpdateServerData struct {
db *pg.DB
dataloader dataloader.DataLoader
server *models.Server
dataloader twdataloader.ServerDataLoader
server *twmodel.Server
}
type loadPlayersResult struct {
ids []int
players []*models.Player
playersToServer []*models.PlayerToServer
players []*twmodel.Player
playersToServer []*twmodel.PlayerToServer
deletedPlayers []int
numberOfPlayers int
}
func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeated) (loadPlayersResult, error) {
var ennoblements []*models.Ennoblement
func (w *workerUpdateServerData) loadPlayers(od map[int]*twmodel.OpponentsDefeated) (loadPlayersResult, error) {
var ennoblements []*twmodel.Ennoblement
result := loadPlayersResult{}
if err := w.db.
Model(&ennoblements).
@ -83,7 +83,7 @@ func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeate
result.numberOfPlayers = len(result.players)
now := time.Now()
result.playersToServer = make([]*models.PlayerToServer, result.numberOfPlayers)
result.playersToServer = make([]*twmodel.PlayerToServer, result.numberOfPlayers)
result.ids = make([]int, result.numberOfPlayers)
searchableByNewOwnerID := &ennoblementsSearchableByNewOwnerID{ennoblements}
for index, player := range result.players {
@ -99,7 +99,7 @@ func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeate
player.DailyGrowth = calcPlayerDailyGrowth(diffInDays, player.Points)
}
result.playersToServer[index] = &models.PlayerToServer{
result.playersToServer[index] = &twmodel.PlayerToServer{
PlayerID: player.ID,
ServerKey: w.server.Key,
}
@ -109,10 +109,10 @@ func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeate
searchablePlayers := &playersSearchableByID{result.players}
if err := w.db.
Model(&models.Player{}).
Model(&twmodel.Player{}).
Column("id").
Where("exists = true").
ForEach(func(player *models.Player) error {
ForEach(func(player *twmodel.Player) error {
if index := searchByID(searchablePlayers, player.ID); index < 0 {
result.deletedPlayers = append(result.deletedPlayers, player.ID)
}
@ -126,12 +126,12 @@ func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeate
type loadTribesResult struct {
ids []int
tribes []*models.Tribe
tribes []*twmodel.Tribe
deletedTribes []int
numberOfTribes int
}
func (w *workerUpdateServerData) loadTribes(od map[int]*models.OpponentsDefeated, numberOfVillages int) (loadTribesResult, error) {
func (w *workerUpdateServerData) loadTribes(od map[int]*twmodel.OpponentsDefeated, numberOfVillages int) (loadTribesResult, error) {
var err error
result := loadTribesResult{}
result.tribes, err = w.dataloader.LoadTribes()
@ -156,10 +156,10 @@ func (w *workerUpdateServerData) loadTribes(od map[int]*models.OpponentsDefeated
searchableTribes := &tribesSearchableByID{result.tribes}
if err := w.db.
Model(&models.Tribe{}).
Model(&twmodel.Tribe{}).
Column("id").
Where("exists = true").
ForEach(func(tribe *models.Tribe) error {
ForEach(func(tribe *twmodel.Tribe) error {
if index := searchByID(searchableTribes, tribe.ID); index < 0 {
result.deletedTribes = append(result.deletedTribes, tribe.ID)
}
@ -171,8 +171,8 @@ func (w *workerUpdateServerData) loadTribes(od map[int]*models.OpponentsDefeated
return result, nil
}
func (w *workerUpdateServerData) calculateODifference(od1 models.OpponentsDefeated, od2 models.OpponentsDefeated) models.OpponentsDefeated {
return models.OpponentsDefeated{
func (w *workerUpdateServerData) calculateODifference(od1 twmodel.OpponentsDefeated, od2 twmodel.OpponentsDefeated) twmodel.OpponentsDefeated {
return twmodel.OpponentsDefeated{
RankAtt: (od1.RankAtt - od2.RankAtt) * -1,
ScoreAtt: od1.ScoreAtt - od2.ScoreAtt,
RankDef: (od1.RankDef - od2.RankDef) * -1,
@ -185,16 +185,16 @@ func (w *workerUpdateServerData) calculateODifference(od1 models.OpponentsDefeat
}
func (w *workerUpdateServerData) calculateTodaysTribeStats(
tribes []*models.Tribe,
history []*models.TribeHistory,
) []*models.DailyTribeStats {
var todaysStats []*models.DailyTribeStats
tribes []*twmodel.Tribe,
history []*twmodel.TribeHistory,
) []*twmodel.DailyTribeStats {
var todaysStats []*twmodel.DailyTribeStats
searchableTribes := &tribesSearchableByID{tribes}
for _, historyRecord := range history {
if index := searchByID(searchableTribes, historyRecord.TribeID); index != -1 {
tribe := tribes[index]
todaysStats = append(todaysStats, &models.DailyTribeStats{
todaysStats = append(todaysStats, &twmodel.DailyTribeStats{
TribeID: tribe.ID,
Members: tribe.TotalMembers - historyRecord.TotalMembers,
Villages: tribe.TotalVillages - historyRecord.TotalVillages,
@ -212,16 +212,16 @@ func (w *workerUpdateServerData) calculateTodaysTribeStats(
}
func (w *workerUpdateServerData) calculateDailyPlayerStats(
players []*models.Player,
history []*models.PlayerHistory,
) []*models.DailyPlayerStats {
var todaysStats []*models.DailyPlayerStats
players []*twmodel.Player,
history []*twmodel.PlayerHistory,
) []*twmodel.DailyPlayerStats {
var todaysStats []*twmodel.DailyPlayerStats
searchablePlayers := &playersSearchableByID{players}
for _, historyRecord := range history {
if index := searchByID(searchablePlayers, historyRecord.PlayerID); index != -1 {
player := players[index]
todaysStats = append(todaysStats, &models.DailyPlayerStats{
todaysStats = append(todaysStats, &twmodel.DailyPlayerStats{
PlayerID: player.ID,
Villages: player.TotalVillages - historyRecord.TotalVillages,
Points: player.Points - historyRecord.Points,
@ -281,7 +281,7 @@ func (w *workerUpdateServerData) update() error {
defer cancel()
return w.db.RunInTransaction(ctx, func(tx *pg.Tx) error {
if len(tribesResult.deletedTribes) > 0 {
if _, err := tx.Model(&models.Tribe{}).
if _, err := tx.Model(&twmodel.Tribe{}).
Where("tribe.id = ANY (?)", pg.Array(tribesResult.deletedTribes)).
Set("exists = false").
Set("deleted_at = now()").
@ -309,7 +309,7 @@ func (w *workerUpdateServerData) update() error {
return errors.Wrap(err, "couldn't insert tribes")
}
var tribesHistory []*models.TribeHistory
var tribesHistory []*twmodel.TribeHistory
if err := tx.
Model(&tribesHistory).
DistinctOn("tribe_id").
@ -339,7 +339,7 @@ func (w *workerUpdateServerData) update() error {
}
if len(playersResult.deletedPlayers) > 0 {
if _, err := tx.Model(&models.Player{}).
if _, err := tx.Model(&twmodel.Player{}).
Where("player.id = ANY (?)", pg.Array(playersResult.deletedPlayers)).
Set("exists = false").
Set("deleted_at = now()").
@ -365,7 +365,7 @@ func (w *workerUpdateServerData) update() error {
return errors.Wrap(err, "couldn't insert players")
}
var playerHistory []*models.PlayerHistory
var playerHistory []*twmodel.PlayerHistory
if err := tx.Model(&playerHistory).
DistinctOn("player_id").
Column("player_history.*").

View File

@ -3,15 +3,15 @@ package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/dataloader"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
)
type taskUpdateServerEnnoblements struct {
*task
}
func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server) error {
func (t *taskUpdateServerEnnoblements) execute(url string, server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -20,7 +20,7 @@ func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server
entry.Debugf("%s: update of the ennoblements has started...", server.Key)
err := (&workerUpdateServerEnnoblements{
db: t.db.WithParam("SERVER", pg.Safe(server.Key)),
dataloader: newDataloader(url),
dataloader: newServerDataLoader(url),
}).update()
if err != nil {
err = errors.Wrap(err, "taskUpdateServerEnnoblements.execute")
@ -32,9 +32,9 @@ func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server
return nil
}
func (t *taskUpdateServerEnnoblements) validatePayload(server *models.Server) error {
func (t *taskUpdateServerEnnoblements) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.Errorf("taskUpdateServerEnnoblements.validatePayload: Expected *models.Server, got nil")
return errors.Errorf("taskUpdateServerEnnoblements.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -42,11 +42,11 @@ func (t *taskUpdateServerEnnoblements) validatePayload(server *models.Server) er
type workerUpdateServerEnnoblements struct {
db *pg.DB
dataloader dataloader.DataLoader
dataloader twdataloader.ServerDataLoader
}
func (w *workerUpdateServerEnnoblements) loadEnnoblements() ([]*models.Ennoblement, error) {
lastEnnoblement := &models.Ennoblement{}
func (w *workerUpdateServerEnnoblements) loadEnnoblements() ([]*twmodel.Ennoblement, error) {
lastEnnoblement := &twmodel.Ennoblement{}
if err := w.db.
Model(lastEnnoblement).
Limit(1).
@ -55,7 +55,7 @@ func (w *workerUpdateServerEnnoblements) loadEnnoblements() ([]*models.Ennobleme
return nil, errors.Wrapf(err, "couldn't load last ennoblement")
}
return w.dataloader.LoadEnnoblements(&dataloader.LoadEnnoblementsConfig{
return w.dataloader.LoadEnnoblements(&twdataloader.LoadEnnoblementsConfig{
EnnobledAtGT: lastEnnoblement.EnnobledAt,
})
}

View File

@ -3,7 +3,7 @@ package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
)
@ -11,7 +11,7 @@ type taskUpdateServerHistory struct {
*task
}
func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server) error {
func (t *taskUpdateServerHistory) execute(timezone string, server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -39,9 +39,9 @@ func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server
return nil
}
func (t *taskUpdateServerHistory) validatePayload(server *models.Server) error {
func (t *taskUpdateServerHistory) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.New("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil")
return errors.New("taskUpdateServerHistory.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -49,21 +49,21 @@ func (t *taskUpdateServerHistory) validatePayload(server *models.Server) error {
type workerUpdateServerHistory struct {
db *pg.DB
server *models.Server
server *twmodel.Server
location *time.Location
}
func (w *workerUpdateServerHistory) update() error {
var players []*models.Player
var players []*twmodel.Player
if err := w.db.Model(&players).Where("exists = true").Select(); err != nil {
return errors.Wrap(err, "couldn't load players")
}
now := time.Now().In(w.location)
createDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
var ph []*models.PlayerHistory
var ph []*twmodel.PlayerHistory
for _, player := range players {
ph = append(ph, &models.PlayerHistory{
ph = append(ph, &twmodel.PlayerHistory{
OpponentsDefeated: player.OpponentsDefeated,
PlayerID: player.ID,
TotalVillages: player.TotalVillages,
@ -74,13 +74,13 @@ func (w *workerUpdateServerHistory) update() error {
})
}
var tribes []*models.Tribe
var tribes []*twmodel.Tribe
if err := w.db.Model(&tribes).Where("exists = true").Select(); err != nil {
return errors.Wrap(err, "couldn't load tribes")
}
var th []*models.TribeHistory
var th []*twmodel.TribeHistory
for _, tribe := range tribes {
th = append(th, &models.TribeHistory{
th = append(th, &twmodel.TribeHistory{
OpponentsDefeated: tribe.OpponentsDefeated,
TribeID: tribe.ID,
TotalMembers: tribe.TotalMembers,
@ -97,7 +97,7 @@ func (w *workerUpdateServerHistory) update() error {
if err != nil {
return err
}
defer func(s *models.Server) {
defer func(s *twmodel.Server) {
if err := tx.Close(); err != nil {
log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key))
}

View File

@ -3,7 +3,7 @@ package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
)
@ -11,7 +11,7 @@ type taskUpdateServerStats struct {
*task
}
func (t *taskUpdateServerStats) execute(timezone string, server *models.Server) error {
func (t *taskUpdateServerStats) execute(timezone string, server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -39,9 +39,9 @@ func (t *taskUpdateServerStats) execute(timezone string, server *models.Server)
return nil
}
func (t *taskUpdateServerStats) validatePayload(server *models.Server) error {
func (t *taskUpdateServerStats) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.New("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil")
return errors.New("taskUpdateServerStats.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -49,51 +49,51 @@ func (t *taskUpdateServerStats) validatePayload(server *models.Server) error {
type workerUpdateServerStats struct {
db *pg.DB
server *models.Server
server *twmodel.Server
location *time.Location
}
func (w *workerUpdateServerStats) prepare() (*models.ServerStats, error) {
activePlayers, err := w.db.Model(&models.Player{}).Where("exists = true").Count()
func (w *workerUpdateServerStats) prepare() (*twmodel.ServerStats, error) {
activePlayers, err := w.db.Model(&twmodel.Player{}).Where("exists = true").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count active players")
}
inactivePlayers, err := w.db.Model(&models.Player{}).Where("exists = false").Count()
inactivePlayers, err := w.db.Model(&twmodel.Player{}).Where("exists = false").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count inactive players")
}
players := activePlayers + inactivePlayers
activeTribes, err := w.db.Model(&models.Tribe{}).Where("exists = true").Count()
activeTribes, err := w.db.Model(&twmodel.Tribe{}).Where("exists = true").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count active tribes")
}
inactiveTribes, err := w.db.Model(&models.Tribe{}).Where("exists = false").Count()
inactiveTribes, err := w.db.Model(&twmodel.Tribe{}).Where("exists = false").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count inactive tribes")
}
tribes := activeTribes + inactiveTribes
barbarianVillages, err := w.db.Model(&models.Village{}).Where("player_id = 0").Count()
barbarianVillages, err := w.db.Model(&twmodel.Village{}).Where("player_id = 0").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count barbarian villages")
}
bonusVillages, err := w.db.Model(&models.Village{}).Where("bonus <> 0").Count()
bonusVillages, err := w.db.Model(&twmodel.Village{}).Where("bonus <> 0").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count bonus villages")
}
playerVillages, err := w.db.Model(&models.Village{}).Where("player_id <> 0").Count()
playerVillages, err := w.db.Model(&twmodel.Village{}).Where("player_id <> 0").Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count player villages")
}
villages, err := w.db.Model(&models.Village{}).Count()
villages, err := w.db.Model(&twmodel.Village{}).Count()
if err != nil {
return nil, errors.Wrap(err, "cannot count villages")
}
now := time.Now().In(w.location)
createDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
return &models.ServerStats{
return &twmodel.ServerStats{
ActivePlayers: activePlayers,
InactivePlayers: inactivePlayers,
Players: players,
@ -120,7 +120,7 @@ func (w *workerUpdateServerStats) update() error {
if err != nil {
return err
}
defer func(s *models.Server) {
defer func(s *twmodel.Server) {
if err := tx.Close(); err != nil {
log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key))
}

View File

@ -3,7 +3,7 @@ package tasks
import (
"context"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
"github.com/tribalwarshelp/cron/internal/cron/queue"
@ -23,12 +23,12 @@ func (t *taskUpdateStats) execute(timezone string) error {
}
year, month, day := time.Now().In(location).Date()
date := time.Date(year, month, day, 1, 45, 0, 0, location)
var servers []*models.Server
var servers []*twmodel.Server
err = t.db.
Model(&servers).
Where(
"status = ? AND (stats_updated_at IS NULL OR stats_updated_at < ?) AND timezone = ?",
models.ServerStatusOpen,
twmodel.ServerStatusOpen,
date,
location.String(),
).

View File

@ -3,7 +3,7 @@ package tasks
import (
"context"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -13,7 +13,7 @@ type taskVacuum struct {
}
func (t *taskVacuum) execute() error {
var servers []*models.Server
var servers []*twmodel.Server
err := t.db.
Model(&servers).
Select()

View File

@ -3,7 +3,7 @@ package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
"github.com/tribalwarshelp/shared/tw/twmodel"
"time"
)
@ -15,7 +15,7 @@ type taskVacuumServerDB struct {
*task
}
func (t *taskVacuumServerDB) execute(server *models.Server) error {
func (t *taskVacuumServerDB) execute(server *twmodel.Server) error {
if err := t.validatePayload(server); err != nil {
log.Debug(err)
return nil
@ -36,9 +36,9 @@ func (t *taskVacuumServerDB) execute(server *models.Server) error {
return nil
}
func (t *taskVacuumServerDB) validatePayload(server *models.Server) error {
func (t *taskVacuumServerDB) validatePayload(server *twmodel.Server) error {
if server == nil {
return errors.New("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil")
return errors.New("taskVacuumServerDB.validatePayload: Expected *twmodel.Server, got nil")
}
return nil
@ -46,7 +46,7 @@ func (t *taskVacuumServerDB) validatePayload(server *models.Server) error {
type workerVacuumServerDB struct {
db *pg.DB
server *models.Server
server *twmodel.Server
}
func (w *workerVacuumServerDB) vacuum() error {
@ -54,16 +54,16 @@ func (w *workerVacuumServerDB) vacuum() error {
if err != nil {
return err
}
defer func(s *models.Server) {
defer func(s *twmodel.Server) {
if err := tx.Close(); err != nil {
log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key))
}
}(w.server)
withNonExistentPlayers := w.db.Model(&models.Player{}).Column("id").Where("exists = false and NOW() - deleted_at > '14 days'")
withNonExistentTribes := w.db.Model(&models.Tribe{}).Column("id").Where("exists = false and NOW() - deleted_at > '1 days'")
withNonExistentPlayers := w.db.Model(&twmodel.Player{}).Column("id").Where("exists = false and NOW() - deleted_at > '14 days'")
withNonExistentTribes := w.db.Model(&twmodel.Tribe{}).Column("id").Where("exists = false and NOW() - deleted_at > '1 days'")
_, err = tx.Model(&models.PlayerHistory{}).
_, err = tx.Model(&twmodel.PlayerHistory{}).
With("players", withNonExistentPlayers).
Where("player_id IN (Select id FROM players) OR player_history.create_date < ?", time.Now().Add(-1*day*180)).
Delete()
@ -71,7 +71,7 @@ func (w *workerVacuumServerDB) vacuum() error {
return errors.Wrap(err, "couldn't delete the old player history records")
}
_, err = tx.Model(&models.TribeHistory{}).
_, err = tx.Model(&twmodel.TribeHistory{}).
With("tribes", withNonExistentTribes).
Where("tribe_id IN (Select id FROM tribes) OR tribe_history.create_date < ?", time.Now().Add(-1*day*180)).
Delete()
@ -79,7 +79,7 @@ func (w *workerVacuumServerDB) vacuum() error {
return errors.Wrap(err, "couldn't delete the old tribe history records")
}
_, err = tx.Model(&models.DailyPlayerStats{}).
_, err = tx.Model(&twmodel.DailyPlayerStats{}).
With("players", withNonExistentPlayers).
Where("player_id IN (Select id FROM players) OR daily_player_stats.create_date < ?", time.Now().Add(-1*day*180)).
Delete()
@ -87,7 +87,7 @@ func (w *workerVacuumServerDB) vacuum() error {
return errors.Wrap(err, "couldn't delete the old player stats records")
}
_, err = tx.Model(&models.DailyTribeStats{}).
_, err = tx.Model(&twmodel.DailyTribeStats{}).
With("tribes", withNonExistentTribes).
Where("tribe_id IN (Select id FROM tribes) OR daily_tribe_stats.create_date < ?", time.Now().Add(-1*day*180)).
Delete()

View File

@ -113,10 +113,7 @@ func Get(taskName string) *taskq.Task {
}
func validateConfig(cfg *Config) error {
if cfg == nil {
return errors.New("Config hasn't been provided")
}
if cfg.DB == nil {
if cfg == nil || cfg.DB == nil {
return errors.New("cfg.DB is required")
}
if cfg.Queue == nil {

View File

@ -1,35 +1,34 @@
package db
package postgres
import (
"fmt"
gopglogrusquerylogger "github.com/Kichiyaki/go-pg-logrus-query-logger/v10"
"github.com/Kichiyaki/go-pg-logrus-query-logger/v10"
"github.com/Kichiyaki/goutil/envutil"
"github.com/go-pg/pg/v10"
"github.com/go-pg/pg/v10/orm"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tribalwarshelp/shared/models"
envutils "github.com/tribalwarshelp/cron/internal/utils/env"
"github.com/tribalwarshelp/shared/tw/twmodel"
)
var log = logrus.WithField("package", "db")
var log = logrus.WithField("package", "postgres")
type Config struct {
LogQueries bool
}
func New(cfg *Config) (*pg.DB, error) {
func Connect(cfg *Config) (*pg.DB, error) {
db := pg.Connect(prepareOptions())
if cfg != nil && cfg.LogQueries {
db.AddQueryHook(gopglogrusquerylogger.QueryLogger{
db.AddQueryHook(querylogger.Logger{
Log: log,
MaxQueryLength: 5000,
})
}
if err := prepareDB(db); err != nil {
return nil, errors.Wrap(err, "New")
return nil, errors.Wrap(err, "Connect")
}
return db, nil
@ -37,11 +36,11 @@ func New(cfg *Config) (*pg.DB, error) {
func prepareOptions() *pg.Options {
return &pg.Options{
User: envutils.GetenvString("DB_USER"),
Password: envutils.GetenvString("DB_PASSWORD"),
Database: envutils.GetenvString("DB_NAME"),
Addr: envutils.GetenvString("DB_HOST") + ":" + envutils.GetenvString("DB_PORT"),
PoolSize: envutils.GetenvInt("DB_POOL_SIZE"),
User: envutil.GetenvString("DB_USER"),
Password: envutil.GetenvString("DB_PASSWORD"),
Database: envutil.GetenvString("DB_NAME"),
Addr: envutil.GetenvString("DB_HOST") + ":" + envutil.GetenvString("DB_PORT"),
PoolSize: envutil.GetenvInt("DB_POOL_SIZE"),
}
}
@ -57,11 +56,11 @@ func prepareDB(db *pg.DB) error {
}()
dbModels := []interface{}{
(*models.SpecialServer)(nil),
(*models.Server)(nil),
(*models.Version)(nil),
(*models.PlayerToServer)(nil),
(*models.PlayerNameChange)(nil),
(*twmodel.SpecialServer)(nil),
(*twmodel.Server)(nil),
(*twmodel.Version)(nil),
(*twmodel.PlayerToServer)(nil),
(*twmodel.PlayerNameChange)(nil),
}
for _, model := range dbModels {
@ -105,7 +104,7 @@ func prepareDB(db *pg.DB) error {
return errors.Wrap(err, "Couldn't commit changes")
}
var servers []*models.Server
var servers []*twmodel.Server
if err := db.Model(&servers).Select(); err != nil {
return errors.Wrap(err, "Couldn't load servers")
}
@ -119,7 +118,7 @@ func prepareDB(db *pg.DB) error {
return nil
}
func CreateSchema(db *pg.DB, server *models.Server) error {
func CreateSchema(db *pg.DB, server *twmodel.Server) error {
return createSchema(db, server, false)
}
@ -135,7 +134,7 @@ func SchemaExists(db *pg.DB, schemaName string) bool {
return exists
}
func createSchema(db *pg.DB, server *models.Server, init bool) error {
func createSchema(db *pg.DB, server *twmodel.Server, init bool) error {
if !init && SchemaExists(db, server.Key) {
return nil
}
@ -155,16 +154,16 @@ func createSchema(db *pg.DB, server *models.Server, init bool) error {
}
dbModels := []interface{}{
(*models.Tribe)(nil),
(*models.Player)(nil),
(*models.Village)(nil),
(*models.Ennoblement)(nil),
(*models.ServerStats)(nil),
(*models.TribeHistory)(nil),
(*models.PlayerHistory)(nil),
(*models.TribeChange)(nil),
(*models.DailyPlayerStats)(nil),
(*models.DailyTribeStats)(nil),
(*twmodel.Tribe)(nil),
(*twmodel.Player)(nil),
(*twmodel.Village)(nil),
(*twmodel.Ennoblement)(nil),
(*twmodel.ServerStats)(nil),
(*twmodel.TribeHistory)(nil),
(*twmodel.PlayerHistory)(nil),
(*twmodel.TribeChange)(nil),
(*twmodel.DailyPlayerStats)(nil),
(*twmodel.DailyTribeStats)(nil),
}
for _, model := range dbModels {

View File

@ -1,4 +1,4 @@
package db
package postgres
const (
allSpecialServersPGInsertStatements = `

View File

@ -1,30 +0,0 @@
package envutils
import (
"os"
"strconv"
)
func GetenvInt(key string) int {
str := GetenvString(key)
if str == "" {
return 0
}
i, err := strconv.Atoi(str)
if err != nil {
return 0
}
return i
}
func GetenvBool(key string) bool {
str := GetenvString(key)
if str == "" {
return false
}
return str == "true" || str == "1"
}
func GetenvString(key string) string {
return os.Getenv(key)
}

28
main.go
View File

@ -2,6 +2,8 @@ package main
import (
"context"
"github.com/Kichiyaki/appmode"
"github.com/Kichiyaki/goutil/envutil"
"github.com/go-redis/redis/v8"
"github.com/pkg/errors"
"os"
@ -10,11 +12,9 @@ import (
"time"
"github.com/sirupsen/logrus"
"github.com/tribalwarshelp/shared/mode"
twhelpcron "github.com/tribalwarshelp/cron/internal/cron"
"github.com/tribalwarshelp/cron/internal/db"
envutils "github.com/tribalwarshelp/cron/internal/utils/env"
"github.com/tribalwarshelp/cron/internal/postgres"
"github.com/joho/godotenv"
"github.com/robfig/cron/v3"
@ -38,7 +38,7 @@ func main() {
}
}()
dbConn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")})
dbConn, err := postgres.Connect(&postgres.Config{LogQueries: envutil.GetenvBool("LOG_DB_QUERIES")})
if err != nil {
logrus.Fatal(errors.Wrap(err, "Couldn't connect to the db"))
}
@ -50,9 +50,9 @@ func main() {
c, err := twhelpcron.New(&twhelpcron.Config{
DB: dbConn,
RunOnInit: envutils.GetenvBool("RUN_ON_INIT"),
RunOnInit: envutil.GetenvBool("RUN_ON_INIT"),
Redis: redisClient,
WorkerLimit: envutils.GetenvInt("WORKER_LIMIT"),
WorkerLimit: envutil.GetenvInt("WORKER_LIMIT"),
Opts: []cron.Option{
cron.WithChain(
cron.SkipIfStillRunning(
@ -68,7 +68,7 @@ func main() {
logrus.Fatal(err)
}
logrus.Info("Cron is running!")
logrus.Info("Cron is up and running!")
channel := make(chan os.Signal, 1)
signal.Notify(channel, os.Interrupt, os.Kill, syscall.SIGTERM, syscall.SIGINT)
@ -86,7 +86,7 @@ func setupENVs() error {
return errors.Wrap(err, "setupENVs")
}
if mode.Get() == mode.DevelopmentMode {
if appmode.Equals(appmode.DevelopmentMode) {
err := godotenv.Load(".env.local")
if err != nil {
return errors.Wrap(err, "setupENVs")
@ -97,12 +97,12 @@ func setupENVs() error {
}
func setupLogger() {
if mode.Get() == mode.DevelopmentMode {
if appmode.Equals(appmode.DevelopmentMode) {
logrus.SetLevel(logrus.DebugLevel)
}
timestampFormat := "2006-01-02 15:04:05"
if mode.Get() == mode.ProductionMode {
if appmode.Equals(appmode.ProductionMode) {
customFormatter := new(logrus.JSONFormatter)
customFormatter.TimestampFormat = timestampFormat
logrus.SetFormatter(customFormatter)
@ -116,10 +116,10 @@ func setupLogger() {
func initializeRedis() (redis.UniversalClient, error) {
client := redis.NewClient(&redis.Options{
Addr: envutils.GetenvString("REDIS_ADDR"),
Username: envutils.GetenvString("REDIS_USERNAME"),
Password: envutils.GetenvString("REDIS_PASSWORD"),
DB: envutils.GetenvInt("REDIS_DB"),
Addr: envutil.GetenvString("REDIS_ADDR"),
Username: envutil.GetenvString("REDIS_USERNAME"),
Password: envutil.GetenvString("REDIS_PASSWORD"),
DB: envutil.GetenvInt("REDIS_DB"),
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()