add a new task - taskServerDeleteNonExistentVillages

internal/cron/cron.go

@@ -71,6 +71,9 @@ func (c *Cron) init() error {
 	if _, err := c.AddFunc("20 1 * * *", c.vacuumDatabase); err != nil {
 		return err
 	}
+	if _, err := c.AddFunc("10 1 * * *", c.deleteNonExistentVillages); err != nil {
+		return err
+	}
 	if _, err := c.AddFunc("@every 1m", c.updateEnnoblements); err != nil {
 		return err
 	}
@@ -79,10 +82,10 @@ func (c *Cron) init() error {
 			c.updateServerData()
 			c.vacuumDatabase()
 			for _, fn := range updateHistoryFuncs {
-				go fn()
+				fn()
 			}
 			for _, fn := range updateStatsFuncs {
-				go fn()
+				fn()
 			}
 		}()
 	}
@@ -140,6 +143,13 @@ func (c *Cron) vacuumDatabase() {
 	}
 }
 
+func (c *Cron) deleteNonExistentVillages() {
+	err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameDeleteNonExistentVillages).WithArgs(context.Background()))
+	if err != nil {
+		c.logError("Cron.deleteNonExistentVillages", tasks.TaskNameDeleteNonExistentVillages, err)
+	}
+}
+
 func (c *Cron) logError(prefix string, taskName string, err error) {
 	c.log.Error(
 		errors.Wrapf(

internal/cron/tasks/helpers.go

@@ -46,45 +46,57 @@ func newDataloader(url string) dataloader.DataLoader {
 	})
 }
 
-type tribesSearchableByID struct {
-	tribes []*models.Tribe
-}
-
-func (searchable tribesSearchableByID) GetID(index int) int {
-	return searchable.tribes[index].ID
-}
-
-func (searchable tribesSearchableByID) Len() int {
-	return len(searchable.tribes)
-}
-
 type playersSearchableByID struct {
 	players []*models.Player
 }
 
-func (searchable playersSearchableByID) GetID(index int) int {
+func (searchable playersSearchableByID) getID(index int) int {
 	return searchable.players[index].ID
 }
 
-func (searchable playersSearchableByID) Len() int {
+func (searchable playersSearchableByID) len() int {
 	return len(searchable.players)
 }
 
+type tribesSearchableByID struct {
+	tribes []*models.Tribe
+}
+
+func (searchable tribesSearchableByID) getID(index int) int {
+	return searchable.tribes[index].ID
+}
+
+func (searchable tribesSearchableByID) len() int {
+	return len(searchable.tribes)
+}
+
+type villagesSearchableByID struct {
+	villages []*models.Village
+}
+
+func (searchable villagesSearchableByID) getID(index int) int {
+	return searchable.villages[index].ID
+}
+
+func (searchable villagesSearchableByID) len() int {
+	return len(searchable.villages)
+}
+
 type ennoblementsSearchableByNewOwnerID struct {
 	ennoblements []*models.Ennoblement
 }
 
-func (searchable ennoblementsSearchableByNewOwnerID) GetID(index int) int {
+func (searchable ennoblementsSearchableByNewOwnerID) getID(index int) int {
 	return searchable.ennoblements[index].NewOwnerID
 }
 
-func (searchable ennoblementsSearchableByNewOwnerID) Len() int {
+func (searchable ennoblementsSearchableByNewOwnerID) len() int {
 	return len(searchable.ennoblements)
 }
 
 type searchableByID interface {
-	GetID(index int) int
-	Len() int
+	getID(index int) int
+	len() int
 }
 
 func makePlayersSearchable(players []*models.Player) searchableByID {
@@ -101,19 +113,19 @@ func makeTribesSearchable(tribes []*models.Tribe) searchableByID {
 
 func searchByID(haystack searchableByID, id int) int {
 	low := 0
-	high := haystack.Len() - 1
+	high := haystack.len() - 1
 
 	for low <= high {
 		median := (low + high) / 2
 
-		if haystack.GetID(median) < id {
+		if haystack.getID(median) < id {
 			low = median + 1
 		} else {
 			high = median - 1
 		}
 	}
 
-	if low == haystack.Len() || haystack.GetID(low) != id {
+	if low == haystack.len() || haystack.getID(low) != id {
 		return -1
 	}
 

internal/cron/tasks/task_delete_non_existent_villages.go

@@ -2,6 +2,7 @@ package tasks
 
 import (
 	"context"
+	"fmt"
 	"github.com/pkg/errors"
 	"github.com/tribalwarshelp/shared/models"
 
@@ -16,6 +17,7 @@ func (t *taskDeleteNonExistentVillages) execute() error {
 	var servers []*models.Server
 	err := t.db.
 		Model(&servers).
+		Relation("Version").
 		Where("status = ?", models.ServerStatusOpen).
 		Relation("Version").
 		Select()
@@ -29,7 +31,15 @@ func (t *taskDeleteNonExistentVillages) execute() error {
 		Info("taskDeleteNonExistentVillages.execute: Servers have been loaded and added to the queue")
 	for _, server := range servers {
 		s := server
-		err := t.queue.Add(queue.MainQueue, Get(TaskNameServerDeleteNonExistentVillages).WithArgs(context.Background(), s))
+		err := t.queue.Add(
+			queue.MainQueue,
+			Get(TaskNameServerDeleteNonExistentVillages).
+				WithArgs(
+					context.Background(),
+					fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host),
+					s,
+				),
+		)
 		if err != nil {
 			log.Warn(
 				errors.Wrapf(

internal/cron/tasks/task_load_servers_and_update_data.go

@@ -89,7 +89,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error {
 
 func (t *taskLoadServersAndUpdateData) validatePayload(version *models.Version) error {
 	if version == nil {
-		return errors.Errorf("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil")
+		return errors.New("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil")
 	}
 	return nil
 }

internal/cron/tasks/task_server_delete_non_existent_villages.go

@@ -0,0 +1,77 @@
+package tasks
+
+import (
+	"github.com/go-pg/pg/v10"
+	"github.com/pkg/errors"
+	"github.com/tribalwarshelp/shared/models"
+	"github.com/tribalwarshelp/shared/tw/dataloader"
+)
+
+type taskServerDeleteNonExistentVillages struct {
+	*task
+}
+
+func (t *taskServerDeleteNonExistentVillages) execute(url string, server *models.Server) error {
+	if err := t.validatePayload(server); err != nil {
+		log.Debug(err)
+		return nil
+	}
+	entry := log.WithField("key", server.Key)
+	entry.Infof("taskServerDeleteNonExistentVillages.execute: %s: Deleting non-existent villages...", server.Key)
+	err := (&workerDeleteNonExistentVillages{
+		db:         t.db.WithParam("SERVER", pg.Safe(server.Key)),
+		dataloader: newDataloader(url),
+		server:     server,
+	}).delete()
+	if err != nil {
+		err = errors.Wrap(err, "taskServerDeleteNonExistentVillages.execute")
+		entry.Error(err)
+		return err
+	}
+	entry.Infof("taskServerDeleteNonExistentVillages.execute: %s: Non-existent villages have been deleted", server.Key)
+	return nil
+}
+
+func (t *taskServerDeleteNonExistentVillages) validatePayload(server *models.Server) error {
+	if server == nil {
+		return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil")
+	}
+
+	return nil
+}
+
+type workerDeleteNonExistentVillages struct {
+	db         *pg.DB
+	dataloader dataloader.DataLoader
+	server     *models.Server
+}
+
+func (w *workerDeleteNonExistentVillages) delete() error {
+	var villagesFromDB []*models.Village
+	if err := w.db.Model(&villagesFromDB).Column("id").Select(); err != nil {
+		return errors.Wrap(err, "workerDeleteNonExistentVillages.delete")
+	}
+	villages, err := w.dataloader.LoadVillages()
+	if err != nil {
+		return errors.Wrap(err, "workerDeleteNonExistentVillages.delete")
+	}
+	var idsToDelete []int
+	searchableByVillageID := &villagesSearchableByID{villages}
+	for _, village := range villagesFromDB {
+		index := searchByID(searchableByVillageID, village.ID)
+		if index < 0 {
+			idsToDelete = append(idsToDelete, village.ID)
+		}
+	}
+
+	totalDeleted := 0
+	if len(idsToDelete) > 0 {
+		result, err := w.db.Model(&models.Village{}).Where("id = ANY(?)", pg.Array(idsToDelete)).Delete()
+		if err != nil {
+			return errors.Wrap(err, "workerDeleteNonExistentVillages.delete")
+		}
+		totalDeleted = result.RowsAffected()
+	}
+	log.Debugf("%s: deleted %d villages", w.server.Key, totalDeleted)
+	return nil
+}

internal/cron/tasks/task_update_server_data.go

@@ -43,7 +43,7 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error
 
 func (t *taskUpdateServerData) validatePayload(server *models.Server) error {
 	if server == nil {
-		return errors.Errorf("taskUpdateServerData.validatePayload: Expected *models.Server, got nil")
+		return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil")
 	}
 
 	return nil
@@ -56,7 +56,7 @@ type workerUpdateServerData struct {
 }
 
 func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeated) ([]*models.Player, error) {
-	var ennoblements = []*models.Ennoblement{}
+	var ennoblements []*models.Ennoblement
 	err := w.db.Model(&ennoblements).DistinctOn("new_owner_id").Order("new_owner_id ASC", "ennobled_at ASC").Select()
 	if err != nil {
 		return nil, errors.Wrap(err, "workerUpdateServerData.loadPlayers: couldn't load ennoblements")
@@ -145,7 +145,7 @@ func (w *workerUpdateServerData) calculateTodaysTribeStats(
 
 func (w *workerUpdateServerData) calculateDailyPlayerStats(players []*models.Player,
 	history []*models.PlayerHistory) []*models.DailyPlayerStats {
-	todaysStats := []*models.DailyPlayerStats{}
+	var todaysStats []*models.DailyPlayerStats
 	searchablePlayers := makePlayersSearchable(players)
 
 	for _, historyRecord := range history {
@@ -168,42 +168,45 @@ func (w *workerUpdateServerData) calculateDailyPlayerStats(players []*models.Pla
 func (w *workerUpdateServerData) update() error {
 	pod, err := w.dataloader.LoadOD(false)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
+
 	tod, err := w.dataloader.LoadOD(true)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
 
 	villages, err := w.dataloader.LoadVillages()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
 	numberOfVillages := len(villages)
 
 	tribes, err := w.loadTribes(tod, countPlayerVillages(villages))
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
 	numberOfTribes := len(tribes)
 
 	players, err := w.loadPlayers(pod)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
 	numberOfPlayers := len(players)
 
 	cfg, err := w.dataloader.GetConfig()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
+
 	buildingCfg, err := w.dataloader.GetBuildingConfig()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
+
 	unitCfg, err := w.dataloader.GetUnitConfig()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "workerUpdateServerData.update")
 	}
 
 	tx, err := w.db.Begin()
@@ -217,7 +220,7 @@ func (w *workerUpdateServerData) update() error {
 	}(w.server)
 
 	if len(tribes) > 0 {
-		ids := []int{}
+		var ids []int
 		for _, tribe := range tribes {
 			ids = append(ids, tribe.ID)
 		}
@@ -244,7 +247,7 @@ func (w *workerUpdateServerData) update() error {
 			return errors.Wrap(err, "couldn't update non-existent tribes")
 		}
 
-		tribesHistory := []*models.TribeHistory{}
+		var tribesHistory []*models.TribeHistory
 		if err := w.db.Model(&tribesHistory).
 			DistinctOn("tribe_id").
 			Column("*").
@@ -272,7 +275,7 @@ func (w *workerUpdateServerData) update() error {
 	}
 
 	if len(players) > 0 {
-		ids := []int{}
+		var ids []int
 		for _, player := range players {
 			ids = append(ids, player.ID)
 		}
@@ -297,7 +300,7 @@ func (w *workerUpdateServerData) update() error {
 			return errors.Wrap(err, "couldn't update non-existent players")
 		}
 
-		playerHistory := []*models.PlayerHistory{}
+		var playerHistory []*models.PlayerHistory
 		if err := w.db.Model(&playerHistory).
 			DistinctOn("player_id").
 			Column("*").

internal/cron/tasks/task_update_server_history.go

@@ -41,7 +41,7 @@ func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server
 
 func (t *taskUpdateServerHistory) validatePayload(server *models.Server) error {
 	if server == nil {
-		return errors.Errorf("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil")
+		return errors.New("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil")
 	}
 
 	return nil

internal/cron/tasks/task_update_server_stats.go

@@ -41,7 +41,7 @@ func (t *taskUpdateServerStats) execute(timezone string, server *models.Server)
 
 func (t *taskUpdateServerStats) validatePayload(server *models.Server) error {
 	if server == nil {
-		return errors.Errorf("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil")
+		return errors.New("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil")
 	}
 
 	return nil

internal/cron/tasks/task_vacuum_server_db.go

@@ -38,7 +38,7 @@ func (t *taskVacuumServerDB) execute(server *models.Server) error {
 
 func (t *taskVacuumServerDB) validatePayload(server *models.Server) error {
 	if server == nil {
-		return errors.Errorf("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil")
+		return errors.New("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil")
 	}
 
 	return nil

internal/cron/tasks/tasks.go

@@ -92,6 +92,10 @@ func RegisterTasks(cfg *Config) error {
 			Name:    TaskNameDeleteNonExistentVillages,
 			Handler: (&taskDeleteNonExistentVillages{t}).execute,
 		},
+		{
+			Name:    TaskNameServerDeleteNonExistentVillages,
+			Handler: (&taskServerDeleteNonExistentVillages{t}).execute,
+		},
 	}
 	for _, taskOptions := range options {
 		opts := taskOptions