From e70c73a3fa872c1cd16fae6b99541316b70c7536 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 2 May 2021 09:44:49 +0200 Subject: [PATCH] refactor, add two new packages tw/urlbuilder and tw/twutil tw/dataloader: - Move endpoints to the new file - use errors.New instead of fmt.Errorf - new error messages --- tw/dataloader/dataloader.go | 120 +++++++----------- tw/dataloader/endpoints.go | 30 +++++ .../version_code_from_server_key.go | 2 +- tw/{urls.go => urlbuilder/build_url.go} | 20 +-- utils/track_execution_time.go | 21 --- 5 files changed, 89 insertions(+), 104 deletions(-) create mode 100644 tw/dataloader/endpoints.go rename tw/{ => twutil}/version_code_from_server_key.go (92%) rename tw/{urls.go => urlbuilder/build_url.go} (52%) delete mode 100644 utils/track_execution_time.go diff --git a/tw/dataloader/dataloader.go b/tw/dataloader/dataloader.go index c35f7f4..c4af55e 100644 --- a/tw/dataloader/dataloader.go +++ b/tw/dataloader/dataloader.go @@ -14,36 +14,8 @@ import ( "time" "github.com/pkg/errors" - "github.com/tribalwarshelp/shared/models" -) -const ( - EndpointConfig = "/interface.php?func=get_config" - EndpointUnitConfig = "/interface.php?func=get_unit_info" - EndpointBuildingConfig = "/interface.php?func=get_building_info" - EndpointPlayer = "/map/player.txt.gz" - EndpointPlayerNotGzipped = "/map/player.txt" - EndpointTribe = "/map/ally.txt.gz" - EndpointTribeNotGzipped = "/map/ally.txt" - EndpointVillage = "/map/village.txt.gz" - EndpointVillageNotGzipped = "/map/village.txt" - EndpointKillAtt = "/map/kill_att.txt.gz" - EndpointKillAttNotGzipped = "/map/kill_att.txt" - EndpointKillDef = "/map/kill_def.txt.gz" - EndpointKillDefNotGzipped = "/map/kill_def.txt" - EndpointKillSup = "/map/kill_sup.txt.gz" - EndpointKillSupNotGzipped = "/map/kill_sup.txt" - EndpointKillAll = "/map/kill_all.txt.gz" - EndpointKillAllNotGzipped = "/map/kill_all.txt" - EndpointKillAttTribe = "/map/kill_att_tribe.txt.gz" - EndpointKillAttTribeNotGzipped = "/map/kill_att_tribe.txt" - EndpointKillDefTribe = "/map/kill_def_tribe.txt.gz" - EndpointKillDefTribeNotGzipped = "/map/kill_def_tribe.txt" - EndpointKillAllTribe = "/map/kill_all_tribe.txt.gz" - EndpointKillAllTribeNotGzipped = "/map/kill_all_tribe.txt" - EndpointConquer = "/map/conquer.txt.gz" - EndpointConquerNotGzipped = "/map/conquer.txt" - EndpointGetConquer = "/interface.php?func=get_conquer&since=%d" + "github.com/tribalwarshelp/shared/models" ) type DataLoader interface { @@ -90,7 +62,7 @@ type parsedODLine struct { func (d *dataLoader) parseODLine(line []string) (*parsedODLine, error) { if len(line) != 3 { - return nil, fmt.Errorf("Invalid line format (should be rank,id,score)") + return nil, errors.New("invalid line format (should be rank,id,score)") } p := &parsedODLine{} var err error @@ -111,51 +83,51 @@ func (d *dataLoader) parseODLine(line []string) (*parsedODLine, error) { func (d *dataLoader) LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, error) { m := make(map[int]*models.OpponentsDefeated) - urls := []string{ + formattedURLs := []string{ fmt.Sprintf("%s%s", d.baseURL, EndpointKillAll), fmt.Sprintf("%s%s", d.baseURL, EndpointKillAtt), fmt.Sprintf("%s%s", d.baseURL, EndpointKillDef), fmt.Sprintf("%s%s", d.baseURL, EndpointKillSup), } if tribe { - urls = []string{ + formattedURLs = []string{ fmt.Sprintf("%s%s", d.baseURL, EndpointKillAllTribe), fmt.Sprintf("%s%s", d.baseURL, EndpointKillAttTribe), fmt.Sprintf("%s%s", d.baseURL, EndpointKillDefTribe), "", } } - for _, url := range urls { - if url == "" { + for _, formattedURL := range formattedURLs { + if formattedURL == "" { continue } - lines, err := d.getCSVData(url, true) + lines, err := d.getCSVData(formattedURL, true) if err != nil { //fallback to not gzipped file - lines, err = d.getCSVData(strings.ReplaceAll(url, ".gz", ""), false) + lines, err = d.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false) if err != nil { - return nil, errors.Wrapf(err, "cannot get data, url %s", url) + return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) } } for _, line := range lines { parsed, err := d.parseODLine(line) if err != nil { - return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ",")) + return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } if _, ok := m[parsed.ID]; !ok { m[parsed.ID] = &models.OpponentsDefeated{} } - switch url { - case urls[0]: + switch formattedURL { + case formattedURLs[0]: m[parsed.ID].RankTotal = parsed.Rank m[parsed.ID].ScoreTotal = parsed.Score - case urls[1]: + case formattedURLs[1]: m[parsed.ID].RankAtt = parsed.Rank m[parsed.ID].ScoreAtt = parsed.Score - case urls[2]: + case formattedURLs[2]: m[parsed.ID].RankDef = parsed.Rank m[parsed.ID].ScoreDef = parsed.Score - case urls[3]: + case formattedURLs[3]: m[parsed.ID].RankSup = parsed.Rank m[parsed.ID].ScoreSup = parsed.Score } @@ -166,7 +138,7 @@ func (d *dataLoader) LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, erro func (d *dataLoader) parsePlayerLine(line []string) (*models.Player, error) { if len(line) != 6 { - return nil, fmt.Errorf("Invalid line format (should be id,name,tribeid,villages,points,rank)") + return nil, errors.New("Invalid line format (should be id,name,tribeid,villages,points,rank)") } var err error @@ -203,20 +175,20 @@ func (d *dataLoader) parsePlayerLine(line []string) (*models.Player, error) { } func (d *dataLoader) LoadPlayers() ([]*models.Player, error) { - url := d.baseURL + EndpointPlayer - lines, err := d.getCSVData(url, true) + formattedURL := d.baseURL + EndpointPlayer + lines, err := d.getCSVData(formattedURL, true) if err != nil { lines, err = d.getCSVData(d.baseURL+EndpointPlayerNotGzipped, false) if err != nil { - return nil, errors.Wrapf(err, "cannot get data, url %s", url) + return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL) } } - players := []*models.Player{} + var players []*models.Player for _, line := range lines { player, err := d.parsePlayerLine(line) if err != nil { - return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ",")) + return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } players = append(players, player) } @@ -226,7 +198,7 @@ func (d *dataLoader) LoadPlayers() ([]*models.Player, error) { func (d *dataLoader) parseTribeLine(line []string) (*models.Tribe, error) { if len(line) != 8 { - return nil, fmt.Errorf("Invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)") + return nil, errors.New("invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)") } var err error @@ -271,19 +243,19 @@ func (d *dataLoader) parseTribeLine(line []string) (*models.Tribe, error) { } func (d *dataLoader) LoadTribes() ([]*models.Tribe, error) { - url := d.baseURL + EndpointTribe - lines, err := d.getCSVData(url, true) + formattedURL := d.baseURL + EndpointTribe + lines, err := d.getCSVData(formattedURL, true) if err != nil { lines, err = d.getCSVData(d.baseURL+EndpointTribeNotGzipped, false) if err != nil { - return nil, errors.Wrapf(err, "cannot to get data, url %s", url) + return nil, errors.Wrapf(err, "cannot to get data, url %s", formattedURL) } } - tribes := []*models.Tribe{} + var tribes []*models.Tribe for _, line := range lines { tribe, err := d.parseTribeLine(line) if err != nil { - return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ",")) + return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } tribes = append(tribes, tribe) } @@ -292,7 +264,7 @@ func (d *dataLoader) LoadTribes() ([]*models.Tribe, error) { func (d *dataLoader) parseVillageLine(line []string) (*models.Village, error) { if len(line) != 7 { - return nil, fmt.Errorf("Invalid line format (should be id,name,x,y,playerID,points,bonus)") + return nil, errors.New("invalid line format (should be id,name,x,y,playerID,points,bonus)") } var err error village := &models.Village{} @@ -328,19 +300,19 @@ func (d *dataLoader) parseVillageLine(line []string) (*models.Village, error) { } func (d *dataLoader) LoadVillages() ([]*models.Village, error) { - url := d.baseURL + EndpointVillage - lines, err := d.getCSVData(url, true) + formattedURL := d.baseURL + EndpointVillage + lines, err := d.getCSVData(formattedURL, true) if err != nil { lines, err = d.getCSVData(d.baseURL+EndpointVillageNotGzipped, false) if err != nil { - return nil, errors.Wrapf(err, "cannot get data, url %s", url) + return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) } } - villages := []*models.Village{} + var villages []*models.Village for _, line := range lines { village, err := d.parseVillageLine(line) if err != nil { - return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ",")) + return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ",")) } villages = append(villages, village) } @@ -349,7 +321,7 @@ func (d *dataLoader) LoadVillages() ([]*models.Village, error) { func (d *dataLoader) parseEnnoblementLine(line []string) (*models.Ennoblement, error) { if len(line) != 4 { - return nil, fmt.Errorf("Invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)") + return nil, errors.New("invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)") } var err error ennoblement := &models.Ennoblement{} @@ -383,25 +355,25 @@ func (d *dataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.En cfg = &LoadEnnoblementsConfig{} } yesterdaysDate := time.Now().Add(-23 * time.Hour) - url := d.baseURL + EndpointConquer + formattedURL := d.baseURL + EndpointConquer compressed := true if cfg.EnnobledAtGT.After(yesterdaysDate) || cfg.EnnobledAtGT.Equal(yesterdaysDate) { - url = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix()) + formattedURL = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix()) compressed = false } - lines, err := d.getCSVData(url, compressed) + lines, err := d.getCSVData(formattedURL, compressed) if err != nil && compressed { lines, err = d.getCSVData(d.baseURL+EndpointConquerNotGzipped, false) } if err != nil { - return nil, errors.Wrapf(err, "cannot get data, url %s", url) + return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) } - ennoblements := []*models.Ennoblement{} + var ennoblements []*models.Ennoblement for _, line := range lines { ennoblement, err := d.parseEnnoblementLine(line) if err != nil { - return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ",")) + return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ",")) } if ennoblement.EnnobledAt.After(cfg.EnnobledAtGT) { ennoblements = append(ennoblements, ennoblement) @@ -411,9 +383,9 @@ func (d *dataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.En } func (d *dataLoader) GetConfig() (*models.ServerConfig, error) { - url := d.baseURL + EndpointConfig + formattedURL := d.baseURL + EndpointConfig cfg := &models.ServerConfig{} - err := d.getXML(url, cfg) + err := d.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getConfig") } @@ -421,9 +393,9 @@ func (d *dataLoader) GetConfig() (*models.ServerConfig, error) { } func (d *dataLoader) GetBuildingConfig() (*models.BuildingConfig, error) { - url := d.baseURL + EndpointBuildingConfig + formattedURL := d.baseURL + EndpointBuildingConfig cfg := &models.BuildingConfig{} - err := d.getXML(url, cfg) + err := d.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getBuildingConfig") } @@ -431,9 +403,9 @@ func (d *dataLoader) GetBuildingConfig() (*models.BuildingConfig, error) { } func (d *dataLoader) GetUnitConfig() (*models.UnitConfig, error) { - url := d.baseURL + EndpointUnitConfig + formattedURL := d.baseURL + EndpointUnitConfig cfg := &models.UnitConfig{} - err := d.getXML(url, cfg) + err := d.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getUnitConfig") } diff --git a/tw/dataloader/endpoints.go b/tw/dataloader/endpoints.go new file mode 100644 index 0000000..d623af9 --- /dev/null +++ b/tw/dataloader/endpoints.go @@ -0,0 +1,30 @@ +package dataloader + +const ( + EndpointConfig = "/interface.php?func=get_config" + EndpointUnitConfig = "/interface.php?func=get_unit_info" + EndpointBuildingConfig = "/interface.php?func=get_building_info" + EndpointPlayer = "/map/player.txt.gz" + EndpointPlayerNotGzipped = "/map/player.txt" + EndpointTribe = "/map/ally.txt.gz" + EndpointTribeNotGzipped = "/map/ally.txt" + EndpointVillage = "/map/village.txt.gz" + EndpointVillageNotGzipped = "/map/village.txt" + EndpointKillAtt = "/map/kill_att.txt.gz" + EndpointKillAttNotGzipped = "/map/kill_att.txt" + EndpointKillDef = "/map/kill_def.txt.gz" + EndpointKillDefNotGzipped = "/map/kill_def.txt" + EndpointKillSup = "/map/kill_sup.txt.gz" + EndpointKillSupNotGzipped = "/map/kill_sup.txt" + EndpointKillAll = "/map/kill_all.txt.gz" + EndpointKillAllNotGzipped = "/map/kill_all.txt" + EndpointKillAttTribe = "/map/kill_att_tribe.txt.gz" + EndpointKillAttTribeNotGzipped = "/map/kill_att_tribe.txt" + EndpointKillDefTribe = "/map/kill_def_tribe.txt.gz" + EndpointKillDefTribeNotGzipped = "/map/kill_def_tribe.txt" + EndpointKillAllTribe = "/map/kill_all_tribe.txt.gz" + EndpointKillAllTribeNotGzipped = "/map/kill_all_tribe.txt" + EndpointConquer = "/map/conquer.txt.gz" + EndpointConquerNotGzipped = "/map/conquer.txt" + EndpointGetConquer = "/interface.php?func=get_conquer&since=%d" +) diff --git a/tw/version_code_from_server_key.go b/tw/twutil/version_code_from_server_key.go similarity index 92% rename from tw/version_code_from_server_key.go rename to tw/twutil/version_code_from_server_key.go index ec0d73a..82da302 100644 --- a/tw/version_code_from_server_key.go +++ b/tw/twutil/version_code_from_server_key.go @@ -1,4 +1,4 @@ -package tw +package twutil import ( "github.com/tribalwarshelp/shared/models" diff --git a/tw/urls.go b/tw/urlbuilder/build_url.go similarity index 52% rename from tw/urls.go rename to tw/urlbuilder/build_url.go index 3304245..6ef1394 100644 --- a/tw/urls.go +++ b/tw/urlbuilder/build_url.go @@ -1,4 +1,4 @@ -package tw +package urlbuilder import "fmt" @@ -8,14 +8,18 @@ const ( EndpointVillageProfile = "/game.php?screen=info_village&id=%d" ) -func BuildVillageURL(server, host string, id int) string { - return fmt.Sprintf("https://%s.%s"+EndpointVillageProfile, server, host, id) -} - -func BuildTribeURL(server, host string, id int) string { - return fmt.Sprintf("https://%s.%s"+EndpointTribeProfile, server, host, id) +func BuildServerURL(server, host string) string { + return fmt.Sprintf("https://%s.%s", server, host) } func BuildPlayerURL(server, host string, id int) string { - return fmt.Sprintf("https://%s.%s"+EndpointPlayerProfile, server, host, id) + return BuildServerURL(server, host) + fmt.Sprintf(EndpointPlayerProfile, id) +} + +func BuildVillageURL(server, host string, id int) string { + return BuildServerURL(server, host) + fmt.Sprintf(EndpointVillageProfile, id) +} + +func BuildTribeURL(server, host string, id int) string { + return BuildServerURL(server, host) + fmt.Sprintf(EndpointTribeProfile, id) } diff --git a/utils/track_execution_time.go b/utils/track_execution_time.go deleted file mode 100644 index 99a05af..0000000 --- a/utils/track_execution_time.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import ( - "time" - - "github.com/sirupsen/logrus" -) - -func TrackExecutionTime(log *logrus.Entry, fn func(), fnName string) func() { - return func() { - now := time.Now() - log := log.WithField("fnName", fnName) - log.Infof("%s: called", fnName) - - fn() - - log. - WithField("executionTime", time.Since(now).String()). - Infof("%s: finished executing", fnName) - } -}