BREAKING CHANGE - run updateStats and updateHistory separately for each version
This commit is contained in:
parent
30fe501f2f
commit
10f735075c
|
|
@ -27,6 +27,7 @@ COPY --from=builder /app/main .
|
|||
ENV MODE=production
|
||||
EXPOSE 8080
|
||||
|
||||
RUN apk add --no-cache tzdata
|
||||
ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.2.1/wait ./wait
|
||||
RUN chmod +x ./wait
|
||||
|
||||
|
|
|
|||
46
cron/cron.go
46
cron/cron.go
|
|
@ -3,6 +3,7 @@ package cron
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/tribalwarshelp/shared/models"
|
||||
"github.com/tribalwarshelp/shared/utils"
|
||||
|
||||
"github.com/go-pg/pg/v10"
|
||||
|
|
@ -20,7 +21,7 @@ type Config struct {
|
|||
|
||||
func Attach(c *cron.Cron, cfg Config) error {
|
||||
if cfg.DB == nil {
|
||||
return fmt.Errorf("cfg.DB cannot be nil, expected go-pg database")
|
||||
return fmt.Errorf("cfg.DB cannot be nil, expected *pg.DB")
|
||||
}
|
||||
|
||||
h := &handler{cfg.DB, cfg.MaxConcurrentWorkers}
|
||||
|
|
@ -28,28 +29,49 @@ func Attach(c *cron.Cron, cfg Config) error {
|
|||
return err
|
||||
}
|
||||
|
||||
versions := []*models.Version{}
|
||||
if err := cfg.DB.Model(&versions).Select(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updateServerData := utils.TrackExecutionTime(log, h.updateServerData, "updateServerData")
|
||||
updateHistory := utils.TrackExecutionTime(log, h.updateHistory, "updateHistory")
|
||||
vacuumDatabase := utils.TrackExecutionTime(log, h.vacuumDatabase, "vacuumDatabase")
|
||||
updateStats := utils.TrackExecutionTime(log, h.updateStats, "updateStats")
|
||||
updateHistoryFuncs := []func(){}
|
||||
updateStatsFuncs := []func(){}
|
||||
for _, version := range versions {
|
||||
updateHistory := utils.TrackExecutionTime(log,
|
||||
createFnWithTimezone(version.Timezone, h.updateHistory),
|
||||
fmt.Sprintf("%s: updateHistory", version.Timezone))
|
||||
updateHistoryFuncs = append(updateHistoryFuncs, updateHistory)
|
||||
|
||||
updateStats := utils.TrackExecutionTime(log,
|
||||
createFnWithTimezone(version.Timezone, h.updateStats),
|
||||
fmt.Sprintf("%s: updateStats", version.Timezone))
|
||||
updateStatsFuncs = append(updateStatsFuncs, updateStats)
|
||||
|
||||
if _, err := c.AddFunc(fmt.Sprintf("CRON_TZ=%s 30 1 * * *", version.Timezone), updateHistory); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.AddFunc(fmt.Sprintf("CRON_TZ=%s 45 1 * * *", version.Timezone), updateStats); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := c.AddFunc("0 * * * *", updateServerData); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.AddFunc("30 0 * * *", updateHistory); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.AddFunc("30 1 * * *", vacuumDatabase); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.AddFunc("30 2 * * *", updateStats); err != nil {
|
||||
if _, err := c.AddFunc("20 1 * * *", vacuumDatabase); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.RunOnStartup {
|
||||
go func() {
|
||||
updateServerData()
|
||||
vacuumDatabase()
|
||||
updateHistory()
|
||||
updateStats()
|
||||
for _, fn := range updateHistoryFuncs {
|
||||
fn()
|
||||
}
|
||||
for _, fn := range updateStatsFuncs {
|
||||
fn()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -234,13 +233,18 @@ func (h *handler) updateServerData() {
|
|||
wg.Wait()
|
||||
}
|
||||
|
||||
func (h *handler) updateHistory() {
|
||||
func (h *handler) updateHistory(location *time.Location) {
|
||||
servers := []*models.Server{}
|
||||
now := time.Now()
|
||||
t1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 30, 0, 0, time.UTC)
|
||||
t1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 30, 0, 0, location)
|
||||
log = log.WithField("timezone", location.String())
|
||||
err := h.db.
|
||||
Model(&servers).
|
||||
Where("status = ? AND (history_updated_at < ? OR history_updated_at IS NULL)", models.ServerStatusOpen, t1).
|
||||
Where("status = ? AND (history_updated_at < ? OR history_updated_at IS NULL) AND timezone = ?",
|
||||
models.ServerStatusOpen,
|
||||
t1,
|
||||
location.String()).
|
||||
Relation("Version").
|
||||
Select()
|
||||
if err != nil {
|
||||
log.Errorln(errors.Wrap(err, "updateHistory"))
|
||||
|
|
@ -251,7 +255,7 @@ func (h *handler) updateHistory() {
|
|||
Info("updateHistory: servers loaded")
|
||||
|
||||
var wg sync.WaitGroup
|
||||
p := newPool(runtime.NumCPU())
|
||||
p := newPool(h.maxConcurrentWorkers)
|
||||
|
||||
for _, server := range servers {
|
||||
p.waitForWorker()
|
||||
|
|
@ -278,19 +282,26 @@ func (h *handler) updateHistory() {
|
|||
wg.Wait()
|
||||
}
|
||||
|
||||
func (h *handler) updateServerStats(t time.Time) error {
|
||||
func (h *handler) updateStats(location *time.Location) {
|
||||
servers := []*models.Server{}
|
||||
now := time.Now()
|
||||
t := time.Date(now.Year(), now.Month(), now.Day(), 0, 45, 0, 0, location)
|
||||
err := h.db.
|
||||
Model(&servers).
|
||||
Where("status = ? AND (stats_updated_at < ? OR stats_updated_at IS NULL)", models.ServerStatusOpen, t).
|
||||
Where("status = ? AND (stats_updated_at < ? OR stats_updated_at IS NULL) AND timezone = ?",
|
||||
models.ServerStatusOpen,
|
||||
t,
|
||||
location.String()).
|
||||
Relation("Version").
|
||||
Select()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "updateServerStats")
|
||||
log.Errorf(errors.Wrap(err, "updateServerStats").Error())
|
||||
return
|
||||
}
|
||||
log.WithField("numberOfServers", len(servers)).Info("updateServerStats: servers loaded")
|
||||
|
||||
var wg sync.WaitGroup
|
||||
p := newPool(runtime.NumCPU())
|
||||
p := newPool(h.maxConcurrentWorkers)
|
||||
|
||||
for _, server := range servers {
|
||||
p.waitForWorker()
|
||||
|
|
@ -315,15 +326,6 @@ func (h *handler) updateServerStats(t time.Time) error {
|
|||
}
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *handler) updateStats() {
|
||||
now := time.Now()
|
||||
t1 := time.Date(now.Year(), now.Month(), now.Day(), 1, 30, 0, 0, time.UTC)
|
||||
if err := h.updateServerStats(t1); err != nil {
|
||||
log.Error(errors.Wrap(err, "updateStats"))
|
||||
}
|
||||
}
|
||||
|
||||
func (h *handler) vacuumDatabase() {
|
||||
|
|
@ -337,7 +339,7 @@ func (h *handler) vacuumDatabase() {
|
|||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
p := newPool(runtime.NumCPU())
|
||||
p := newPool(h.maxConcurrentWorkers)
|
||||
|
||||
for _, server := range servers {
|
||||
p.waitForWorker()
|
||||
|
|
|
|||
|
|
@ -78,3 +78,13 @@ func calcPlayerDailyGrowth(diffInDays, points int) int {
|
|||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func createFnWithTimezone(timezone string, fn func(location *time.Location)) func() {
|
||||
tz, err := time.LoadLocation(timezone)
|
||||
if err != nil {
|
||||
tz = time.UTC
|
||||
}
|
||||
return func() {
|
||||
fn(tz)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,13 +28,13 @@ const (
|
|||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('hu', 'Hungary', 'klanhaboru.hu', 'Europe/Budapest') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('it', 'Italy', 'tribals.it', 'Europe/Rome') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('fr', 'France', 'guerretribale.fr', 'Europe/Paris') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('us', 'United States', 'tribalwars.us', 'America/Washington') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('us', 'United States', 'tribalwars.us', 'America/New_York') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('nl', 'The Netherlands', 'tribalwars.nl', 'Europe/Amsterdam') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('es', 'Spain', 'guerrastribales.es', 'Europe/Madrid') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('ro', 'Romania', 'triburile.ro', 'Europe/Bucharest') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('gr', 'Greece', 'fyletikesmaxes.gr', 'Europe/Athens') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('br', 'Brazil', 'tribalwars.com.br', 'America/Sao_Paulo') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('tr', 'Turkey', 'klanlar.org', 'Europe/Instanbul') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('tr', 'Turkey', 'klanlar.org', 'Europe/Istanbul') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('cs', 'Czech Republic', 'divokekmeny.cz', 'Europe/Prague') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('ru', 'Russia', 'voyna-plemyon.ru', 'Europe/Moscow') ON CONFLICT (code) DO NOTHING;
|
||||
INSERT INTO public.versions (code, name, host, timezone) VALUES ('ch', 'Switerzland', 'staemme.ch', 'Europe/Zurich') ON CONFLICT (code) DO NOTHING;
|
||||
|
|
|
|||
5
go.mod
5
go.mod
|
|
@ -5,14 +5,11 @@ go 1.14
|
|||
require (
|
||||
github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83
|
||||
github.com/go-pg/pg/v10 v10.3.2
|
||||
github.com/go-pg/pgext v0.2.0
|
||||
github.com/joho/godotenv v1.3.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
github.com/tribalwarshelp/shared v0.0.0-20201230135308-316413cf7f53
|
||||
github.com/vmihailenco/tagparser v0.1.2 // indirect
|
||||
go.opentelemetry.io/otel v0.13.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
|
||||
golang.org/x/net v0.0.0-20201009032441-dbdefad45b89 // indirect
|
||||
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd // indirect
|
||||
)
|
||||
|
|
|
|||
5
go.sum
5
go.sum
|
|
@ -15,6 +15,8 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
|
|||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/go-pg/pg/v10 v10.3.2 h1:2GgewrvOE3G0eQH5V6NhAx5vzI+1If0cupnF9Ls3znY=
|
||||
github.com/go-pg/pg/v10 v10.3.2/go.mod h1:wY0cRYyO1JfUXBF2XjkbnNvhce3WyunFDhEvZXnHbP0=
|
||||
github.com/go-pg/pgext v0.2.0 h1:hN5ex/T86L2/zMXp/wTlVRqf39eGKprLeoqOM+bdbOo=
|
||||
github.com/go-pg/pgext v0.2.0/go.mod h1:0iPPtB2BIFZ2/Jzfr51HHcDEltm3tUVkXh4/su5u8kU=
|
||||
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
|
|
@ -70,6 +72,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/segmentio/encoding v0.2.0 h1:lmwUV/WjSX7pL1pbK4MEQBH/M7K2NrgYymeSc1pV1W8=
|
||||
github.com/segmentio/encoding v0.2.0/go.mod h1:MJjRE6bMDocliO2FyFC2Dusp+uYdBfHWh5Bw7QyExto=
|
||||
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -136,6 +140,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc=
|
||||
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
7
main.go
7
main.go
|
|
@ -5,6 +5,7 @@ import (
|
|||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
|
@ -13,6 +14,7 @@ import (
|
|||
_cron "github.com/tribalwarshelp/cron/cron"
|
||||
|
||||
"github.com/go-pg/pg/v10"
|
||||
"github.com/go-pg/pgext"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
|
@ -49,6 +51,11 @@ func main() {
|
|||
logrus.WithFields(dbFields).Fatalln(err)
|
||||
}
|
||||
}()
|
||||
if strings.ToUpper(os.Getenv("LOG_DB_QUERIES")) == "TRUE" {
|
||||
db.AddQueryHook(pgext.DebugHook{
|
||||
Verbose: true,
|
||||
})
|
||||
}
|
||||
logrus.WithFields(dbFields).Info("Connected to the database")
|
||||
|
||||
c := cron.New(cron.WithChain(
|
||||
|
|
|
|||
Reference in New Issue