cron fetches conquers and saves server stats and history, bump github.com/tribalwarshelp/shared

2020-06-20 20:16:27 +02:00 · 2020-06-20 20:16:27 +02:00 · 5509ffded0
parent ed14e23638
commit 5509ffded0
6 changed files with 445 additions and 16 deletions
--- a/cron/handler.go
+++ b/cron/handler.go
@ -7,6 +7,7 @@ import (
 	"net/http"
 	"runtime"
 	"sync"
+	"time"

 	"github.com/tribalwarshelp/shared/models"

@ -20,6 +21,60 @@ import (

 const (
 	endpointGetServers = "/backend/get_servers.php"
+	serverPGFunctions  = `
+		CREATE OR REPLACE FUNCTION ?0.log_tribe_change()
+			RETURNS trigger AS
+		$BODY$
+		BEGIN
+			IF NEW.tribe_id <> OLD.tribe_id THEN
+				INSERT INTO ?0.tribe_changes(player_id,old_tribe_id,new_tribe_id,created_at)
+				VALUES(OLD.id,OLD.tribe_id,NEW.tribe_id,now());
+			END IF;
+
+			RETURN NEW;
+		END;
+		$BODY$
+		LANGUAGE plpgsql VOLATILE;
+		CREATE OR REPLACE FUNCTION ?0.get_old_and_new_owner_tribe_id()
+			RETURNS trigger AS
+		$BODY$
+		BEGIN
+			IF NEW.old_owner_id <> 0 THEN
+				SELECT tribe_id INTO NEW.old_owner_tribe_id
+					FROM ?0.players
+					WHERE id = NEW.old_owner_id;
+			END IF;
+			IF NEW.old_owner_tribe_id IS NULL THEN
+				NEW.old_owner_tribe_id = 0;
+			END IF;
+			IF NEW.new_owner_id <> 0 THEN
+				SELECT tribe_id INTO NEW.new_owner_tribe_id
+					FROM ?0.players
+					WHERE id = NEW.new_owner_id;
+			END IF;
+			IF NEW.new_owner_tribe_id IS NULL THEN
+				NEW.new_owner_tribe_id = 0;
+			END IF;
+
+			RETURN NEW;
+		END;
+		$BODY$
+		LANGUAGE plpgsql VOLATILE;
+	`
+	serverPGTriggers = `
+		DROP TRIGGER IF EXISTS ?0_tribe_changes ON ?0.players;
+		CREATE TRIGGER ?0_tribe_changes
+			AFTER UPDATE
+			ON ?0.players
+			FOR EACH ROW
+			EXECUTE PROCEDURE ?0.log_tribe_change();
+		DROP TRIGGER IF EXISTS ?0_update_ennoblement_old_and_new_owner_tribe_id ON ?0.ennoblements;
+		CREATE TRIGGER ?0_update_ennoblement_old_and_new_owner_tribe_id
+			BEFORE INSERT
+			ON ?0.ennoblements
+			FOR EACH ROW
+			EXECUTE PROCEDURE ?0.get_old_and_new_owner_tribe_id();
+	`
 )

 type handler struct {
@ -32,10 +87,20 @@ func Attach(c *cron.Cron, db *pg.DB) error {
 		return err
 	}

-	if _, err := c.AddFunc("@every 1h", h.updateData); err != nil {
+	if _, err := c.AddFunc("0 * * * *", h.updateServersData); err != nil {
 		return err
 	}
-	go h.updateData()
+	if _, err := c.AddFunc("30 0 * * *", h.updateServersHistory); err != nil {
+		return err
+	}
+	if _, err := c.AddFunc("30 1 * * *", h.updateStats); err != nil {
+		return err
+	}
+	go func() {
+		h.updateServersData()
+		h.updateServersHistory()
+		h.updateStats()
+	}()

 	return nil
 }
@ -79,6 +144,11 @@ func (h *handler) createSchema(key string) error {
 		(*models.Tribe)(nil),
 		(*models.Player)(nil),
 		(*models.Village)(nil),
+		(*models.Ennoblement)(nil),
+		(*models.ServerStats)(nil),
+		(*models.TribeHistory)(nil),
+		(*models.PlayerHistory)(nil),
+		(*models.TribeChange)(nil),
 	}

 	for _, model := range models {
@ -90,6 +160,15 @@ func (h *handler) createSchema(key string) error {
 		}
 	}

+	for _, statement := range []string{
+		serverPGFunctions,
+		serverPGTriggers,
+	} {
+		if _, err := tx.Exec(statement, pg.Safe(key)); err != nil {
+			return err
+		}
+	}
+
 	return tx.Commit()
 }

@ -136,13 +215,15 @@ func (h *handler) getServers() ([]*models.Server, map[string]string, error) {
 		}
 	}

-	if _, err := h.db.Model(&servers).
-		OnConflict("(key) DO UPDATE").
-		Set("status = ?", models.ServerStatusOpen).
-		Set("lang_version_tag = EXCLUDED.lang_version_tag").
-		Returning("*").
-		Insert(); err != nil {
-		return nil, nil, err
+	if len(servers) > 0 {
+		if _, err := h.db.Model(&servers).
+			OnConflict("(key) DO UPDATE").
+			Set("status = ?", models.ServerStatusOpen).
+			Set("lang_version_tag = EXCLUDED.lang_version_tag").
+			Returning("*").
+			Insert(); err != nil {
+			return nil, nil, err
+		}
 	}

 	if _, err := h.db.Model(&models.Server{}).
@ -155,7 +236,7 @@ func (h *handler) getServers() ([]*models.Server, map[string]string, error) {
 	return servers, urls, nil
 }

-func (h *handler) updateData() {
+func (h *handler) updateServersData() {
 	servers, urls, err := h.getServers()
 	if err != nil {
 		log.Println(err.Error())
@ -197,3 +278,96 @@ func (h *handler) updateData() {

 	wg.Wait()
 }
+
+func (h *handler) updateServersHistory() {
+	servers := []*models.Server{}
+	now := time.Now()
+	t1 := time.Date(now.Year(), now.Month(), now.Day(), 0, 30, 0, 0, time.UTC)
+	err := h.db.
+		Model(&servers).
+		Where("status = ? AND (history_updated_at < ? OR history_updated_at IS NULL)", models.ServerStatusOpen, t1).
+		Select()
+	if err != nil {
+		log.Println(errors.Wrap(err, "updateServersHistory"))
+		return
+	}
+
+	var wg sync.WaitGroup
+	max := runtime.NumCPU() * 5
+	count := 0
+
+	for _, server := range servers {
+		if count >= max {
+			wg.Wait()
+			count = 0
+		}
+		sh := &updateServerHistoryHandler{
+			db:     h.db.WithParam("SERVER", pg.Safe(server.Key)),
+			server: server,
+		}
+		count++
+		wg.Add(1)
+		go func(server *models.Server, sh *updateServerHistoryHandler) {
+			defer wg.Done()
+			log.Printf("%s: updating history", server.Key)
+			if err := sh.update(); err != nil {
+				log.Println(errors.Wrap(err, server.Key))
+				return
+			} else {
+				log.Printf("%s: history updated", server.Key)
+			}
+		}(server, sh)
+	}
+
+	wg.Wait()
+}
+
+func (h *handler) updateServersStats(t time.Time) error {
+	servers := []*models.Server{}
+	err := h.db.
+		Model(&servers).
+		Where("status = ? AND (stats_updated_at < ? OR stats_updated_at IS NULL)", models.ServerStatusOpen, t).
+		Select()
+	if err != nil {
+		return errors.Wrap(err, "updateServersStats")
+	}
+
+	var wg sync.WaitGroup
+	max := runtime.NumCPU() * 5
+	count := 0
+
+	for _, server := range servers {
+		if count >= max {
+			wg.Wait()
+			count = 0
+		}
+		sh := &updateServerStatsHandler{
+			db:     h.db.WithParam("SERVER", pg.Safe(server.Key)),
+			server: server,
+		}
+		count++
+		wg.Add(1)
+		go func(server *models.Server, sh *updateServerStatsHandler) {
+			defer wg.Done()
+			log.Printf("%s: updating stats", server.Key)
+			if err := sh.update(); err != nil {
+				log.Println(errors.Wrap(err, server.Key))
+				return
+			} else {
+				log.Printf("%s: stats updated", server.Key)
+			}
+		}(server, sh)
+	}
+
+	wg.Wait()
+	return nil
+}
+
+func (h *handler) updateStats() {
+	now := time.Now()
+	t1 := time.Date(now.Year(), now.Month(), now.Day(), 1, 30, 0, 0, time.UTC)
+	if err := h.updateServersStats(t1); err != nil {
+		log.Println(err)
+		return
+	}
+}
--- a/cron/update_server_data_handler.go
+++ b/cron/update_server_data_handler.go
@ -27,6 +27,7 @@ const (
 	endpointKillAttTribe   = "/map/kill_att_tribe.txt"
 	endpointKillDefTribe   = "/map/kill_def_tribe.txt"
 	endpointKillAllTribe   = "/map/kill_all_tribe.txt"
+	endpointConquer        = "/map/conquer.txt"
 )

 type updateServerDataHandler struct {
@ -295,6 +296,62 @@ func (h *updateServerDataHandler) getVillages() ([]*models.Village, error) {
 	return villages, nil
 }

+func (h *updateServerDataHandler) parseEnnoblementLine(line []string) (*models.Ennoblement, error) {
+	if len(line) != 4 {
+		return nil, fmt.Errorf("Invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)")
+	}
+	var err error
+	ennoblement := &models.Ennoblement{}
+	ennoblement.VillageID, err = strconv.Atoi(line[0])
+	if err != nil {
+		return nil, errors.Wrap(err, "ennoblement.VillageID")
+	}
+	timestamp, err := strconv.Atoi(line[1])
+	if err != nil {
+		return nil, errors.Wrap(err, "timestamp")
+	}
+	ennoblement.EnnobledAt = time.Unix(int64(timestamp), 0)
+	ennoblement.NewOwnerID, err = strconv.Atoi(line[2])
+	if err != nil {
+		return nil, errors.Wrap(err, "ennoblement.NewOwnerID")
+	}
+	ennoblement.OldOwnerID, err = strconv.Atoi(line[3])
+	if err != nil {
+		return nil, errors.Wrap(err, "ennoblement.OldOwnerID")
+	}
+
+	return ennoblement, nil
+}
+
+func (h *updateServerDataHandler) getEnnoblements() ([]*models.Ennoblement, error) {
+	url := h.baseURL + endpointConquer
+	lines, err := getCSVData(url, false)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to get data, url %s", url)
+	}
+
+	lastEnnoblement := &models.Ennoblement{}
+	if err := h.db.
+		Model(lastEnnoblement).
+		Limit(1).
+		Order("ennobled_at DESC").
+		Select(); err != nil && err != pg.ErrNoRows {
+		return nil, errors.Wrapf(err, "cannot load last ennoblement, url %s", url)
+	}
+
+	ennoblements := []*models.Ennoblement{}
+	for _, line := range lines {
+		ennoblement, err := h.parseEnnoblementLine(line)
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot parse line, url %s", url)
+		}
+		if ennoblement.EnnobledAt.After(lastEnnoblement.EnnobledAt) {
+			ennoblements = append(ennoblements, ennoblement)
+		}
+	}
+	return ennoblements, nil
+}
+
 func (h *updateServerDataHandler) getConfig() (*models.ServerConfig, error) {
 	url := h.baseURL + endpointConfig
 	cfg := &models.ServerConfig{}
@ -358,6 +415,10 @@ func (h *updateServerDataHandler) update() error {
 	if err != nil {
 		return err
 	}
+	ennoblements, err := h.getEnnoblements()
+	if err != nil {
+		return err
+	}

 	tx, err := h.db.Begin()
 	if err != nil {
@ -440,12 +501,20 @@ func (h *updateServerDataHandler) update() error {
 			return errors.Wrap(err, "cannot delete not existed villages")
 		}
 	}
+	if len(ennoblements) > 0 {
+		if _, err := tx.Model(&ennoblements).Insert(); err != nil {
+			return errors.Wrap(err, "cannot insert ennoblements")
+		}
+	}

-	h.server.Config = *cfg
-	h.server.UnitConfig = *unitCfg
-	h.server.BuildingConfig = *buildingCfg
-	h.server.DataUpdatedAt = time.Now()
-	if err := tx.Update(h.server); err != nil {
+	if _, err := tx.Model(h.server).
+		Set("data_updated_at = ?", time.Now()).
+		Set("unit_config = ?", unitCfg).
+		Set("building_config = ?", buildingCfg).
+		Set("config = ?", cfg).
+		Returning("*").
+		WherePK().
+		Update(); err != nil {
 		return errors.Wrap(err, "cannot update server")
 	}

--- a/cron/update_server_history_handler.go
+++ b/cron/update_server_history_handler.go
@ -0,0 +1,78 @@
+package cron
+
+import (
+	"time"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/pkg/errors"
+	"github.com/tribalwarshelp/shared/models"
+)
+
+type updateServerHistoryHandler struct {
+	db     *pg.DB
+	server *models.Server
+}
+
+func (h *updateServerHistoryHandler) update() error {
+	players := []*models.Player{}
+	if err := h.db.Model(&players).Where("exist = true").Select(); err != nil {
+		return errors.Wrap(err, "cannot load players")
+	}
+	ph := []*models.PlayerHistory{}
+	for _, player := range players {
+		ph = append(ph, &models.PlayerHistory{
+			OpponentsDefeated: player.OpponentsDefeated,
+			PlayerID:          player.ID,
+			TotalVillages:     player.TotalVillages,
+			Points:            player.Points,
+			Rank:              player.Rank,
+			TribeID:           player.TribeID,
+		})
+	}
+
+	tribes := []*models.Tribe{}
+	if err := h.db.Model(&tribes).Where("exist = true").Select(); err != nil {
+		return errors.Wrap(err, "cannot load tribes")
+	}
+	th := []*models.TribeHistory{}
+	for _, tribe := range tribes {
+		th = append(th, &models.TribeHistory{
+			OpponentsDefeated: tribe.OpponentsDefeated,
+			TribeID:           tribe.ID,
+			TotalMembers:      tribe.TotalMembers,
+			TotalVillages:     tribe.TotalVillages,
+			Points:            tribe.Points,
+			AllPoints:         tribe.AllPoints,
+			Rank:              tribe.Rank,
+		})
+	}
+
+	tx, err := h.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Close()
+
+	if len(ph) > 0 {
+		if _, err := h.db.Model(&ph).Insert(); err != nil {
+			return errors.Wrap(err, "cannot insert players history")
+		}
+	}
+
+	if len(th) > 0 {
+		if _, err := h.db.Model(&th).Insert(); err != nil {
+			return errors.Wrap(err, "cannot insert tribes history")
+		}
+	}
+
+	if _, err := tx.Model(h.server).
+		Set("history_updated_at = ?", time.Now()).
+		WherePK().
+		Returning("*").
+		Update(); err != nil {
+		return errors.Wrap(err, "cannot update server")
+
+	}
+
+	return tx.Commit()
+}
--- a/cron/update_server_stats_handler.go
+++ b/cron/update_server_stats_handler.go
@ -0,0 +1,96 @@
+package cron
+
+import (
+	"time"
+
+	"github.com/go-pg/pg/v10"
+	"github.com/pkg/errors"
+	"github.com/tribalwarshelp/shared/models"
+)
+
+type updateServerStatsHandler struct {
+	db     *pg.DB
+	server *models.Server
+}
+
+func (h *updateServerStatsHandler) prepare() (*models.ServerStats, error) {
+	activePlayers, err := h.db.Model(&models.Player{}).Where("exist = true").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count active players")
+	}
+	inactivePlayers, err := h.db.Model(&models.Player{}).Where("exist = false").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count inactive players")
+	}
+	players := activePlayers + inactivePlayers
+
+	activeTribes, err := h.db.Model(&models.Tribe{}).Where("exist = true").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count active tribes")
+	}
+	inactiveTribes, err := h.db.Model(&models.Tribe{}).Where("exist = false").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count inactive tribes")
+	}
+	tribes := activeTribes + inactiveTribes
+
+	barbarianVillages, err := h.db.Model(&models.Village{}).Where("player_id = 0 AND bonus = 0").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count barbarian villages")
+	}
+	bonusVillages, err := h.db.Model(&models.Village{}).Where("bonus <> 0").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count bonus villages")
+	}
+	playerVillages, err := h.db.Model(&models.Village{}).Where("player_id <> 0").Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count player villages")
+	}
+	villages, err := h.db.Model(&models.Village{}).Count()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot count villages")
+	}
+
+	return &models.ServerStats{
+		ActivePlayers:   activePlayers,
+		InactivePlayers: inactivePlayers,
+		Players:         players,
+
+		ActiveTribes:   activeTribes,
+		InactiveTribes: inactiveTribes,
+		Tribes:         tribes,
+
+		BarbarianVillages: barbarianVillages,
+		BonusVillages:     bonusVillages,
+		PlayerVillages:    playerVillages,
+		Villages:          villages,
+	}, nil
+}
+
+func (h *updateServerStatsHandler) update() error {
+	stats, err := h.prepare()
+	if err != nil {
+		return err
+	}
+
+	tx, err := h.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Close()
+
+	if err := tx.Insert(stats); err != nil {
+		return errors.Wrap(err, "cannot insert server stats")
+	}
+
+	_, err = tx.Model(h.server).
+		Set("stats_updated_at = ?", time.Now()).
+		WherePK().
+		Returning("*").
+		Update()
+	if err != nil {
+		return errors.Wrap(err, "cannot update server")
+	}
+
+	return tx.Commit()
+}
--- a/go.mod
+++ b/go.mod
@ -8,6 +8,6 @@ require (
 	github.com/joho/godotenv v1.3.0
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/tribalwarshelp/shared v0.0.0-20200619163026-783d3512bc3e
+	github.com/tribalwarshelp/shared v0.0.0-20200620172120-2529c050265b
 	golang.org/x/net v0.0.0-20200528225125-3c3fba18258b // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -94,6 +94,18 @@ github.com/tribalwarshelp/shared v0.0.0-20200619155726-3476188924c3 h1:vJyTmEdxs
 github.com/tribalwarshelp/shared v0.0.0-20200619155726-3476188924c3/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
 github.com/tribalwarshelp/shared v0.0.0-20200619163026-783d3512bc3e h1:u54k/lorToKUmhruBLLLtykF4MBC3qu5xrzDTk2WLjY=
 github.com/tribalwarshelp/shared v0.0.0-20200619163026-783d3512bc3e/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200619170116-8109f286e39e h1:lPVjeNPbLYC8MJ4lVWaGsppYJyYb80r39hD5bsFscZY=
+github.com/tribalwarshelp/shared v0.0.0-20200619170116-8109f286e39e/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200619170430-9ae1e58e717a h1:DvZaFa0q43ME/j9vBQMgI2v23Re/Dx11pxqpXw5NzC4=
+github.com/tribalwarshelp/shared v0.0.0-20200619170430-9ae1e58e717a/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200620141729-df8e201d51a1 h1:gkpV0q6mQF5s3mNyT5/4MfTGh/+bFBDDX8KqpPi18Bw=
+github.com/tribalwarshelp/shared v0.0.0-20200620141729-df8e201d51a1/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200620142258-4b7c12cbdd87 h1:6ByuFR3HRKgYAi70fNjYhBCSz040HrFgcmgsPT9ozAA=
+github.com/tribalwarshelp/shared v0.0.0-20200620142258-4b7c12cbdd87/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200620153121-bcad77573c9d h1:nyDzPccg/nZjiOKy/VoShyotSMtZMGOpUPNydWE2zVA=
+github.com/tribalwarshelp/shared v0.0.0-20200620153121-bcad77573c9d/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
+github.com/tribalwarshelp/shared v0.0.0-20200620172120-2529c050265b h1:uYxRxhuIXIcPLU3/MoKOkOn8/vOKMkM06XHxeEkIebQ=
+github.com/tribalwarshelp/shared v0.0.0-20200620172120-2529c050265b/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
 github.com/vmihailenco/bufpool v0.1.5/go.mod h1:fL9i/PRTuS7AELqAHwSU1Zf1c70xhkhGe/cD5ud9pJk=
 github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
 github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=