rename one package (internal/cron/tasks -> internal/cron/task), correct 'package' field in logger initialization

This commit is contained in:
Dawid Wysokiński 2021-05-14 14:02:51 +02:00
parent 3313208707
commit 849ee6ae36
21 changed files with 201 additions and 202 deletions

View File

@ -11,7 +11,7 @@ import (
"github.com/robfig/cron/v3"
"github.com/tribalwarshelp/cron/internal/cron/queue"
"github.com/tribalwarshelp/cron/internal/cron/tasks"
"github.com/tribalwarshelp/cron/internal/cron/task"
)
type Cron struct {
@ -35,7 +35,7 @@ func New(cfg *Config) (*Cron, error) {
queue: q,
db: cfg.DB,
runOnInit: cfg.RunOnInit,
log: logrus.WithField("package", "cron"),
log: logrus.WithField("package", "internal/cron"),
}
if err := c.init(); err != nil {
return nil, err
@ -109,44 +109,44 @@ func (c *Cron) Stop() error {
}
func (c *Cron) updateServerData() {
err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background()))
err := c.queue.Add(queue.Main, task.Get(task.LoadVersionsAndUpdateServerData).WithArgs(context.Background()))
if err != nil {
c.logError("Cron.updateServerData", tasks.TaskNameLoadVersionsAndUpdateServerData, err)
c.logError("Cron.updateServerData", task.LoadVersionsAndUpdateServerData, err)
}
}
func (c *Cron) updateEnnoblements() {
err := c.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background()))
err := c.queue.Add(queue.Ennoblements, task.Get(task.UpdateEnnoblements).WithArgs(context.Background()))
if err != nil {
c.logError("Cron.updateEnnoblements", tasks.TaskUpdateEnnoblements, err)
c.logError("Cron.updateEnnoblements", task.UpdateEnnoblements, err)
}
}
func (c *Cron) updateHistory(timezone string) {
err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone))
err := c.queue.Add(queue.Main, task.Get(task.UpdateHistory).WithArgs(context.Background(), timezone))
if err != nil {
c.logError("Cron.updateHistory", tasks.TaskUpdateHistory, err)
c.logError("Cron.updateHistory", task.UpdateHistory, err)
}
}
func (c *Cron) updateStats(timezone string) {
err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone))
err := c.queue.Add(queue.Main, task.Get(task.UpdateStats).WithArgs(context.Background(), timezone))
if err != nil {
c.logError("Cron.updateStats", tasks.TaskUpdateStats, err)
c.logError("Cron.updateStats", task.UpdateStats, err)
}
}
func (c *Cron) vacuumDatabase() {
err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background()))
err := c.queue.Add(queue.Main, task.Get(task.Vacuum).WithArgs(context.Background()))
if err != nil {
c.logError("Cron.vacuumDatabase", tasks.TaskNameVacuum, err)
c.logError("Cron.vacuumDatabase", task.Vacuum, err)
}
}
func (c *Cron) deleteNonExistentVillages() {
err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameDeleteNonExistentVillages).WithArgs(context.Background()))
err := c.queue.Add(queue.Main, task.Get(task.DeleteNonExistentVillages).WithArgs(context.Background()))
if err != nil {
c.logError("Cron.deleteNonExistentVillages", tasks.TaskNameDeleteNonExistentVillages, err)
c.logError("Cron.deleteNonExistentVillages", task.DeleteNonExistentVillages, err)
}
}
@ -169,7 +169,7 @@ func initializeQueue(cfg *Config) (queue.Queue, error) {
if err != nil {
return nil, errors.Wrap(err, "initializeQueue: Couldn't create the task queue")
}
err = tasks.RegisterTasks(&tasks.Config{
err = task.RegisterTasks(&task.Config{
DB: cfg.DB,
Queue: q,
})

View File

@ -11,8 +11,8 @@ import (
)
const (
MainQueue = "main"
EnnoblementsQueue = "ennoblements"
Main = "main"
Ennoblements = "ennoblements"
)
type Queue interface {
@ -46,8 +46,8 @@ func New(cfg *Config) (Queue, error) {
func (q *queue) init(cfg *Config) error {
q.factory = redisq.NewFactory()
q.mainQueue = q.registerQueue(MainQueue, cfg.WorkerLimit)
q.ennoblementsQueue = q.registerQueue(EnnoblementsQueue, cfg.WorkerLimit)
q.mainQueue = q.registerQueue(Main, cfg.WorkerLimit)
q.ennoblementsQueue = q.registerQueue(Ennoblements, cfg.WorkerLimit)
return nil
}
@ -64,9 +64,9 @@ func (q *queue) registerQueue(name string, limit int) taskq.Queue {
func (q *queue) getQueueByName(name string) taskq.Queue {
switch name {
case MainQueue:
case Main:
return q.mainQueue
case EnnoblementsQueue:
case Ennoblements:
return q.ennoblementsQueue
}
return nil

View File

@ -0,0 +1,22 @@
package task
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
type Config struct {
DB *pg.DB
Queue queue.Queue
}
func validateConfig(cfg *Config) error {
if cfg == nil || cfg.DB == nil {
return errors.New("cfg.DB is required")
}
if cfg.Queue == nil {
return errors.New("cfg.Queue is required")
}
return nil
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/tribalwarshelp/shared/tw/twdataloader"

129
internal/cron/task/task.go Normal file
View File

@ -0,0 +1,129 @@
package task
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vmihailenco/taskq/v3"
"sync"
"time"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
const (
LoadVersionsAndUpdateServerData = "loadVersionsAndUpdateServerData"
LoadServersAndUpdateData = "loadServersAndUpdateData"
UpdateServerData = "updateServerData"
Vacuum = "vacuum"
VacuumServerDB = "vacuumServerDB"
UpdateEnnoblements = "updateEnnoblements"
UpdateServerEnnoblements = "updateServerEnnoblements"
UpdateHistory = "updateHistory"
UpdateServerHistory = "updateServerHistory"
UpdateStats = "updateStats"
UpdateServerStats = "updateServerStats"
DeleteNonExistentVillages = "deleteNonExistentVillages"
ServerDeleteNonExistentVillages = "serverDeleteNonExistentVillages"
defaultRetryLimit = 3
)
var log = logrus.WithField("package", "internal/cron/task")
type task struct {
db *pg.DB
queue queue.Queue
cachedLocations sync.Map
}
func (t *task) loadLocation(timezone string) (*time.Location, error) {
val, ok := t.cachedLocations.Load(timezone)
if ok {
return val.(*time.Location), nil
}
location, err := time.LoadLocation(timezone)
if err != nil {
return nil, errors.Wrap(err, "task.loadLocation")
}
t.cachedLocations.Store(timezone, location)
return location, nil
}
func RegisterTasks(cfg *Config) error {
if err := validateConfig(cfg); err != nil {
return errors.Wrap(err, "RegisterTasks")
}
t := &task{
db: cfg.DB,
queue: cfg.Queue,
}
options := []*taskq.TaskOptions{
{
Name: LoadVersionsAndUpdateServerData,
Handler: (&taskLoadVersionsAndUpdateServerData{t}).execute,
},
{
Name: LoadServersAndUpdateData,
Handler: (&taskLoadServersAndUpdateData{t}).execute,
},
{
Name: UpdateServerData,
Handler: (&taskUpdateServerData{t}).execute,
},
{
Name: Vacuum,
Handler: (&taskVacuum{t}).execute,
},
{
Name: VacuumServerDB,
Handler: (&taskVacuumServerDB{t}).execute,
},
{
Name: UpdateEnnoblements,
Handler: (&taskUpdateEnnoblements{t}).execute,
},
{
Name: UpdateServerEnnoblements,
Handler: (&taskUpdateServerEnnoblements{t}).execute,
},
{
Name: UpdateHistory,
Handler: (&taskUpdateHistory{t}).execute,
},
{
Name: UpdateServerHistory,
RetryLimit: defaultRetryLimit,
Handler: (&taskUpdateServerHistory{t}).execute,
},
{
Name: UpdateStats,
Handler: (&taskUpdateStats{t}).execute,
},
{
Name: UpdateServerStats,
Handler: (&taskUpdateServerStats{t}).execute,
},
{
Name: DeleteNonExistentVillages,
Handler: (&taskDeleteNonExistentVillages{t}).execute,
},
{
Name: ServerDeleteNonExistentVillages,
Handler: (&taskServerDeleteNonExistentVillages{t}).execute,
},
}
for _, taskOptions := range options {
opts := taskOptions
if opts.RetryLimit == 0 {
opts.RetryLimit = defaultRetryLimit
}
taskq.RegisterTask(opts)
}
return nil
}
func Get(taskName string) *taskq.Task {
return taskq.Tasks.Get(taskName)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -32,8 +32,8 @@ func (t *taskDeleteNonExistentVillages) execute() error {
for _, server := range servers {
s := server
err := t.queue.Add(
queue.MainQueue,
Get(TaskNameServerDeleteNonExistentVillages).
queue.Main,
Get(ServerDeleteNonExistentVillages).
WithArgs(
context.Background(),
twurlbuilder.BuildServerURL(server.Key, server.Version.Host),
@ -46,7 +46,7 @@ func (t *taskDeleteNonExistentVillages) execute() error {
err,
"taskDeleteNonExistentVillages.execute: %s: Couldn't add the task '%s' for this server",
server.Key,
TaskNameServerDeleteNonExistentVillages,
ServerDeleteNonExistentVillages,
),
)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -85,7 +85,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
}
for _, server := range servers {
t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), server.url, server.Server))
t.queue.Add(queue.Main, Get(UpdateServerData).WithArgs(context.Background(), server.url, server.Server))
}
entry.Infof("%s: Servers have been loaded", version.Host)

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -21,7 +21,7 @@ func (t *taskLoadVersionsAndUpdateServerData) execute() error {
return err
}
for _, version := range versions {
t.queue.Add(queue.MainQueue, Get(TaskNameLoadServersAndUpdateData).WithArgs(context.Background(), version))
t.queue.Add(queue.Main, Get(LoadServersAndUpdateData).WithArgs(context.Background(), version))
}
log.Debug("taskLoadVersionsAndUpdateServerData.execute: Versions have been loaded")
return nil

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/go-pg/pg/v10"

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -29,8 +29,8 @@ func (t *taskUpdateEnnoblements) execute() error {
for _, server := range servers {
s := server
err := t.queue.Add(
queue.EnnoblementsQueue,
Get(TaskUpdateServerEnnoblements).
queue.Ennoblements,
Get(UpdateServerEnnoblements).
WithArgs(context.Background(), twurlbuilder.BuildServerURL(server.Key, server.Version.Host), s),
)
if err != nil {
@ -39,7 +39,7 @@ func (t *taskUpdateEnnoblements) execute() error {
err,
"taskUpdateEnnoblements.execute: %s: Couldn't add the task '%s' for this server",
server.Key,
TaskUpdateServerEnnoblements,
UpdateServerEnnoblements,
),
)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -44,14 +44,14 @@ func (t *taskUpdateHistory) execute(timezone string) error {
Info("taskUpdateHistory.execute: Update of the history has started")
for _, server := range servers {
s := server
err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), timezone, s))
err := t.queue.Add(queue.Main, Get(UpdateServerHistory).WithArgs(context.Background(), timezone, s))
if err != nil {
log.Warn(
errors.Wrapf(
err,
"taskUpdateHistory.execute: %s: Couldn't add the task '%s' for this server",
server.Key,
TaskUpdateServerHistory,
UpdateServerHistory,
),
)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/go-pg/pg/v10"

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/go-pg/pg/v10"

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/go-pg/pg/v10"

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -44,14 +44,14 @@ func (t *taskUpdateStats) execute(timezone string) error {
Info("taskUpdateStats.execute: Update of the stats has started")
for _, server := range servers {
s := server
err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), timezone, s))
err := t.queue.Add(queue.Main, Get(UpdateServerStats).WithArgs(context.Background(), timezone, s))
if err != nil {
log.Warn(
errors.Wrapf(
err,
"taskUpdateStats.execute: %s: Couldn't add the task '%s' for this server",
server.Key,
TaskUpdateServerStats,
UpdateServerStats,
),
)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"context"
@ -25,14 +25,14 @@ func (t *taskVacuum) execute() error {
log.Infof("taskVacuum.execute: Start database vacumming...")
for _, server := range servers {
s := server
err := t.queue.Add(queue.MainQueue, Get(TaskNameVacuumServerDB).WithArgs(context.Background(), s))
err := t.queue.Add(queue.Main, Get(VacuumServerDB).WithArgs(context.Background(), s))
if err != nil {
log.Warn(
errors.Wrapf(
err,
"taskVacuum.execute: %s: Couldn't add the task '%s' for this server",
server.Key,
TaskUpdateServerEnnoblements,
UpdateServerEnnoblements,
),
)
}

View File

@ -1,4 +1,4 @@
package tasks
package task
import (
"github.com/go-pg/pg/v10"

View File

@ -1,29 +0,0 @@
package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"sync"
"time"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
type task struct {
db *pg.DB
queue queue.Queue
cachedLocations sync.Map
}
func (t *task) loadLocation(timezone string) (*time.Location, error) {
val, ok := t.cachedLocations.Load(timezone)
if ok {
return val.(*time.Location), nil
}
location, err := time.LoadLocation(timezone)
if err != nil {
return nil, errors.Wrap(err, "task.loadLocation")
}
t.cachedLocations.Store(timezone, location)
return location, nil
}

View File

@ -1,123 +0,0 @@
package tasks
import (
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vmihailenco/taskq/v3"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
const (
TaskNameLoadVersionsAndUpdateServerData = "loadVersionsAndUpdateServerData"
TaskNameLoadServersAndUpdateData = "loadServersAndUpdateData"
TaskNameUpdateServerData = "updateServerData"
TaskNameVacuum = "vacuum"
TaskNameVacuumServerDB = "vacuumServerDB"
TaskUpdateEnnoblements = "updateEnnoblements"
TaskUpdateServerEnnoblements = "updateServerEnnoblements"
TaskUpdateHistory = "updateHistory"
TaskUpdateServerHistory = "updateServerHistory"
TaskUpdateStats = "updateStats"
TaskUpdateServerStats = "updateServerStats"
TaskNameDeleteNonExistentVillages = "deleteNonExistentVillages"
TaskNameServerDeleteNonExistentVillages = "serverDeleteNonExistentVillages"
defaultRetryLimit = 3
)
var log = logrus.WithField("package", "cron/tasks")
type Config struct {
DB *pg.DB
Queue queue.Queue
}
func RegisterTasks(cfg *Config) error {
if err := validateConfig(cfg); err != nil {
return errors.Wrap(err, "RegisterTasks")
}
t := &task{
db: cfg.DB,
queue: cfg.Queue,
}
options := []*taskq.TaskOptions{
{
Name: TaskNameLoadVersionsAndUpdateServerData,
Handler: (&taskLoadVersionsAndUpdateServerData{t}).execute,
},
{
Name: TaskNameLoadServersAndUpdateData,
Handler: (&taskLoadServersAndUpdateData{t}).execute,
},
{
Name: TaskNameUpdateServerData,
Handler: (&taskUpdateServerData{t}).execute,
},
{
Name: TaskNameVacuum,
Handler: (&taskVacuum{t}).execute,
},
{
Name: TaskNameVacuumServerDB,
Handler: (&taskVacuumServerDB{t}).execute,
},
{
Name: TaskUpdateEnnoblements,
Handler: (&taskUpdateEnnoblements{t}).execute,
},
{
Name: TaskUpdateServerEnnoblements,
Handler: (&taskUpdateServerEnnoblements{t}).execute,
},
{
Name: TaskUpdateHistory,
Handler: (&taskUpdateHistory{t}).execute,
},
{
Name: TaskUpdateServerHistory,
RetryLimit: defaultRetryLimit,
Handler: (&taskUpdateServerHistory{t}).execute,
},
{
Name: TaskUpdateStats,
Handler: (&taskUpdateStats{t}).execute,
},
{
Name: TaskUpdateServerStats,
Handler: (&taskUpdateServerStats{t}).execute,
},
{
Name: TaskNameDeleteNonExistentVillages,
Handler: (&taskDeleteNonExistentVillages{t}).execute,
},
{
Name: TaskNameServerDeleteNonExistentVillages,
Handler: (&taskServerDeleteNonExistentVillages{t}).execute,
},
}
for _, taskOptions := range options {
opts := taskOptions
if opts.RetryLimit == 0 {
opts.RetryLimit = defaultRetryLimit
}
taskq.RegisterTask(opts)
}
return nil
}
func Get(taskName string) *taskq.Task {
return taskq.Tasks.Get(taskName)
}
func validateConfig(cfg *Config) error {
if cfg == nil || cfg.DB == nil {
return errors.New("cfg.DB is required")
}
if cfg.Queue == nil {
return errors.New("cfg.Queue is required")
}
return nil
}

View File

@ -11,7 +11,7 @@ import (
"github.com/tribalwarshelp/shared/tw/twmodel"
)
var log = logrus.WithField("package", "postgres")
var log = logrus.WithField("package", "internal/postgres")
type Config struct {
LogQueries bool