refactor, add two new packages tw/urlbuilder and tw/twutil
tw/dataloader: - Move endpoints to the new file - use errors.New instead of fmt.Errorf - new error messages
This commit is contained in:
parent
b8f533bc97
commit
e70c73a3fa
|
@ -14,36 +14,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tribalwarshelp/shared/models"
|
||||
)
|
||||
|
||||
const (
|
||||
EndpointConfig = "/interface.php?func=get_config"
|
||||
EndpointUnitConfig = "/interface.php?func=get_unit_info"
|
||||
EndpointBuildingConfig = "/interface.php?func=get_building_info"
|
||||
EndpointPlayer = "/map/player.txt.gz"
|
||||
EndpointPlayerNotGzipped = "/map/player.txt"
|
||||
EndpointTribe = "/map/ally.txt.gz"
|
||||
EndpointTribeNotGzipped = "/map/ally.txt"
|
||||
EndpointVillage = "/map/village.txt.gz"
|
||||
EndpointVillageNotGzipped = "/map/village.txt"
|
||||
EndpointKillAtt = "/map/kill_att.txt.gz"
|
||||
EndpointKillAttNotGzipped = "/map/kill_att.txt"
|
||||
EndpointKillDef = "/map/kill_def.txt.gz"
|
||||
EndpointKillDefNotGzipped = "/map/kill_def.txt"
|
||||
EndpointKillSup = "/map/kill_sup.txt.gz"
|
||||
EndpointKillSupNotGzipped = "/map/kill_sup.txt"
|
||||
EndpointKillAll = "/map/kill_all.txt.gz"
|
||||
EndpointKillAllNotGzipped = "/map/kill_all.txt"
|
||||
EndpointKillAttTribe = "/map/kill_att_tribe.txt.gz"
|
||||
EndpointKillAttTribeNotGzipped = "/map/kill_att_tribe.txt"
|
||||
EndpointKillDefTribe = "/map/kill_def_tribe.txt.gz"
|
||||
EndpointKillDefTribeNotGzipped = "/map/kill_def_tribe.txt"
|
||||
EndpointKillAllTribe = "/map/kill_all_tribe.txt.gz"
|
||||
EndpointKillAllTribeNotGzipped = "/map/kill_all_tribe.txt"
|
||||
EndpointConquer = "/map/conquer.txt.gz"
|
||||
EndpointConquerNotGzipped = "/map/conquer.txt"
|
||||
EndpointGetConquer = "/interface.php?func=get_conquer&since=%d"
|
||||
"github.com/tribalwarshelp/shared/models"
|
||||
)
|
||||
|
||||
type DataLoader interface {
|
||||
|
@ -90,7 +62,7 @@ type parsedODLine struct {
|
|||
|
||||
func (d *dataLoader) parseODLine(line []string) (*parsedODLine, error) {
|
||||
if len(line) != 3 {
|
||||
return nil, fmt.Errorf("Invalid line format (should be rank,id,score)")
|
||||
return nil, errors.New("invalid line format (should be rank,id,score)")
|
||||
}
|
||||
p := &parsedODLine{}
|
||||
var err error
|
||||
|
@ -111,51 +83,51 @@ func (d *dataLoader) parseODLine(line []string) (*parsedODLine, error) {
|
|||
|
||||
func (d *dataLoader) LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, error) {
|
||||
m := make(map[int]*models.OpponentsDefeated)
|
||||
urls := []string{
|
||||
formattedURLs := []string{
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAll),
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAtt),
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDef),
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillSup),
|
||||
}
|
||||
if tribe {
|
||||
urls = []string{
|
||||
formattedURLs = []string{
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAllTribe),
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAttTribe),
|
||||
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDefTribe),
|
||||
"",
|
||||
}
|
||||
}
|
||||
for _, url := range urls {
|
||||
if url == "" {
|
||||
for _, formattedURL := range formattedURLs {
|
||||
if formattedURL == "" {
|
||||
continue
|
||||
}
|
||||
lines, err := d.getCSVData(url, true)
|
||||
lines, err := d.getCSVData(formattedURL, true)
|
||||
if err != nil {
|
||||
//fallback to not gzipped file
|
||||
lines, err = d.getCSVData(strings.ReplaceAll(url, ".gz", ""), false)
|
||||
lines, err = d.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot get data, url %s", url)
|
||||
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
|
||||
}
|
||||
}
|
||||
for _, line := range lines {
|
||||
parsed, err := d.parseODLine(line)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ","))
|
||||
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
|
||||
}
|
||||
if _, ok := m[parsed.ID]; !ok {
|
||||
m[parsed.ID] = &models.OpponentsDefeated{}
|
||||
}
|
||||
switch url {
|
||||
case urls[0]:
|
||||
switch formattedURL {
|
||||
case formattedURLs[0]:
|
||||
m[parsed.ID].RankTotal = parsed.Rank
|
||||
m[parsed.ID].ScoreTotal = parsed.Score
|
||||
case urls[1]:
|
||||
case formattedURLs[1]:
|
||||
m[parsed.ID].RankAtt = parsed.Rank
|
||||
m[parsed.ID].ScoreAtt = parsed.Score
|
||||
case urls[2]:
|
||||
case formattedURLs[2]:
|
||||
m[parsed.ID].RankDef = parsed.Rank
|
||||
m[parsed.ID].ScoreDef = parsed.Score
|
||||
case urls[3]:
|
||||
case formattedURLs[3]:
|
||||
m[parsed.ID].RankSup = parsed.Rank
|
||||
m[parsed.ID].ScoreSup = parsed.Score
|
||||
}
|
||||
|
@ -166,7 +138,7 @@ func (d *dataLoader) LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, erro
|
|||
|
||||
func (d *dataLoader) parsePlayerLine(line []string) (*models.Player, error) {
|
||||
if len(line) != 6 {
|
||||
return nil, fmt.Errorf("Invalid line format (should be id,name,tribeid,villages,points,rank)")
|
||||
return nil, errors.New("Invalid line format (should be id,name,tribeid,villages,points,rank)")
|
||||
}
|
||||
|
||||
var err error
|
||||
|
@ -203,20 +175,20 @@ func (d *dataLoader) parsePlayerLine(line []string) (*models.Player, error) {
|
|||
}
|
||||
|
||||
func (d *dataLoader) LoadPlayers() ([]*models.Player, error) {
|
||||
url := d.baseURL + EndpointPlayer
|
||||
lines, err := d.getCSVData(url, true)
|
||||
formattedURL := d.baseURL + EndpointPlayer
|
||||
lines, err := d.getCSVData(formattedURL, true)
|
||||
if err != nil {
|
||||
lines, err = d.getCSVData(d.baseURL+EndpointPlayerNotGzipped, false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot get data, url %s", url)
|
||||
return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL)
|
||||
}
|
||||
}
|
||||
|
||||
players := []*models.Player{}
|
||||
var players []*models.Player
|
||||
for _, line := range lines {
|
||||
player, err := d.parsePlayerLine(line)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ","))
|
||||
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
|
||||
}
|
||||
players = append(players, player)
|
||||
}
|
||||
|
@ -226,7 +198,7 @@ func (d *dataLoader) LoadPlayers() ([]*models.Player, error) {
|
|||
|
||||
func (d *dataLoader) parseTribeLine(line []string) (*models.Tribe, error) {
|
||||
if len(line) != 8 {
|
||||
return nil, fmt.Errorf("Invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)")
|
||||
return nil, errors.New("invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)")
|
||||
}
|
||||
|
||||
var err error
|
||||
|
@ -271,19 +243,19 @@ func (d *dataLoader) parseTribeLine(line []string) (*models.Tribe, error) {
|
|||
}
|
||||
|
||||
func (d *dataLoader) LoadTribes() ([]*models.Tribe, error) {
|
||||
url := d.baseURL + EndpointTribe
|
||||
lines, err := d.getCSVData(url, true)
|
||||
formattedURL := d.baseURL + EndpointTribe
|
||||
lines, err := d.getCSVData(formattedURL, true)
|
||||
if err != nil {
|
||||
lines, err = d.getCSVData(d.baseURL+EndpointTribeNotGzipped, false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot to get data, url %s", url)
|
||||
return nil, errors.Wrapf(err, "cannot to get data, url %s", formattedURL)
|
||||
}
|
||||
}
|
||||
tribes := []*models.Tribe{}
|
||||
var tribes []*models.Tribe
|
||||
for _, line := range lines {
|
||||
tribe, err := d.parseTribeLine(line)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ","))
|
||||
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
|
||||
}
|
||||
tribes = append(tribes, tribe)
|
||||
}
|
||||
|
@ -292,7 +264,7 @@ func (d *dataLoader) LoadTribes() ([]*models.Tribe, error) {
|
|||
|
||||
func (d *dataLoader) parseVillageLine(line []string) (*models.Village, error) {
|
||||
if len(line) != 7 {
|
||||
return nil, fmt.Errorf("Invalid line format (should be id,name,x,y,playerID,points,bonus)")
|
||||
return nil, errors.New("invalid line format (should be id,name,x,y,playerID,points,bonus)")
|
||||
}
|
||||
var err error
|
||||
village := &models.Village{}
|
||||
|
@ -328,19 +300,19 @@ func (d *dataLoader) parseVillageLine(line []string) (*models.Village, error) {
|
|||
}
|
||||
|
||||
func (d *dataLoader) LoadVillages() ([]*models.Village, error) {
|
||||
url := d.baseURL + EndpointVillage
|
||||
lines, err := d.getCSVData(url, true)
|
||||
formattedURL := d.baseURL + EndpointVillage
|
||||
lines, err := d.getCSVData(formattedURL, true)
|
||||
if err != nil {
|
||||
lines, err = d.getCSVData(d.baseURL+EndpointVillageNotGzipped, false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot get data, url %s", url)
|
||||
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
|
||||
}
|
||||
}
|
||||
villages := []*models.Village{}
|
||||
var villages []*models.Village
|
||||
for _, line := range lines {
|
||||
village, err := d.parseVillageLine(line)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ","))
|
||||
return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ","))
|
||||
}
|
||||
villages = append(villages, village)
|
||||
}
|
||||
|
@ -349,7 +321,7 @@ func (d *dataLoader) LoadVillages() ([]*models.Village, error) {
|
|||
|
||||
func (d *dataLoader) parseEnnoblementLine(line []string) (*models.Ennoblement, error) {
|
||||
if len(line) != 4 {
|
||||
return nil, fmt.Errorf("Invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)")
|
||||
return nil, errors.New("invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)")
|
||||
}
|
||||
var err error
|
||||
ennoblement := &models.Ennoblement{}
|
||||
|
@ -383,25 +355,25 @@ func (d *dataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.En
|
|||
cfg = &LoadEnnoblementsConfig{}
|
||||
}
|
||||
yesterdaysDate := time.Now().Add(-23 * time.Hour)
|
||||
url := d.baseURL + EndpointConquer
|
||||
formattedURL := d.baseURL + EndpointConquer
|
||||
compressed := true
|
||||
if cfg.EnnobledAtGT.After(yesterdaysDate) || cfg.EnnobledAtGT.Equal(yesterdaysDate) {
|
||||
url = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix())
|
||||
formattedURL = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix())
|
||||
compressed = false
|
||||
}
|
||||
lines, err := d.getCSVData(url, compressed)
|
||||
lines, err := d.getCSVData(formattedURL, compressed)
|
||||
if err != nil && compressed {
|
||||
lines, err = d.getCSVData(d.baseURL+EndpointConquerNotGzipped, false)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot get data, url %s", url)
|
||||
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
|
||||
}
|
||||
|
||||
ennoblements := []*models.Ennoblement{}
|
||||
var ennoblements []*models.Ennoblement
|
||||
for _, line := range lines {
|
||||
ennoblement, err := d.parseEnnoblementLine(line)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "cannot parse line, url %s, line %s", url, strings.Join(line, ","))
|
||||
return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ","))
|
||||
}
|
||||
if ennoblement.EnnobledAt.After(cfg.EnnobledAtGT) {
|
||||
ennoblements = append(ennoblements, ennoblement)
|
||||
|
@ -411,9 +383,9 @@ func (d *dataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.En
|
|||
}
|
||||
|
||||
func (d *dataLoader) GetConfig() (*models.ServerConfig, error) {
|
||||
url := d.baseURL + EndpointConfig
|
||||
formattedURL := d.baseURL + EndpointConfig
|
||||
cfg := &models.ServerConfig{}
|
||||
err := d.getXML(url, cfg)
|
||||
err := d.getXML(formattedURL, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getConfig")
|
||||
}
|
||||
|
@ -421,9 +393,9 @@ func (d *dataLoader) GetConfig() (*models.ServerConfig, error) {
|
|||
}
|
||||
|
||||
func (d *dataLoader) GetBuildingConfig() (*models.BuildingConfig, error) {
|
||||
url := d.baseURL + EndpointBuildingConfig
|
||||
formattedURL := d.baseURL + EndpointBuildingConfig
|
||||
cfg := &models.BuildingConfig{}
|
||||
err := d.getXML(url, cfg)
|
||||
err := d.getXML(formattedURL, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getBuildingConfig")
|
||||
}
|
||||
|
@ -431,9 +403,9 @@ func (d *dataLoader) GetBuildingConfig() (*models.BuildingConfig, error) {
|
|||
}
|
||||
|
||||
func (d *dataLoader) GetUnitConfig() (*models.UnitConfig, error) {
|
||||
url := d.baseURL + EndpointUnitConfig
|
||||
formattedURL := d.baseURL + EndpointUnitConfig
|
||||
cfg := &models.UnitConfig{}
|
||||
err := d.getXML(url, cfg)
|
||||
err := d.getXML(formattedURL, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getUnitConfig")
|
||||
}
|
||||
|
|
30
tw/dataloader/endpoints.go
Normal file
30
tw/dataloader/endpoints.go
Normal file
|
@ -0,0 +1,30 @@
|
|||
package dataloader
|
||||
|
||||
const (
|
||||
EndpointConfig = "/interface.php?func=get_config"
|
||||
EndpointUnitConfig = "/interface.php?func=get_unit_info"
|
||||
EndpointBuildingConfig = "/interface.php?func=get_building_info"
|
||||
EndpointPlayer = "/map/player.txt.gz"
|
||||
EndpointPlayerNotGzipped = "/map/player.txt"
|
||||
EndpointTribe = "/map/ally.txt.gz"
|
||||
EndpointTribeNotGzipped = "/map/ally.txt"
|
||||
EndpointVillage = "/map/village.txt.gz"
|
||||
EndpointVillageNotGzipped = "/map/village.txt"
|
||||
EndpointKillAtt = "/map/kill_att.txt.gz"
|
||||
EndpointKillAttNotGzipped = "/map/kill_att.txt"
|
||||
EndpointKillDef = "/map/kill_def.txt.gz"
|
||||
EndpointKillDefNotGzipped = "/map/kill_def.txt"
|
||||
EndpointKillSup = "/map/kill_sup.txt.gz"
|
||||
EndpointKillSupNotGzipped = "/map/kill_sup.txt"
|
||||
EndpointKillAll = "/map/kill_all.txt.gz"
|
||||
EndpointKillAllNotGzipped = "/map/kill_all.txt"
|
||||
EndpointKillAttTribe = "/map/kill_att_tribe.txt.gz"
|
||||
EndpointKillAttTribeNotGzipped = "/map/kill_att_tribe.txt"
|
||||
EndpointKillDefTribe = "/map/kill_def_tribe.txt.gz"
|
||||
EndpointKillDefTribeNotGzipped = "/map/kill_def_tribe.txt"
|
||||
EndpointKillAllTribe = "/map/kill_all_tribe.txt.gz"
|
||||
EndpointKillAllTribeNotGzipped = "/map/kill_all_tribe.txt"
|
||||
EndpointConquer = "/map/conquer.txt.gz"
|
||||
EndpointConquerNotGzipped = "/map/conquer.txt"
|
||||
EndpointGetConquer = "/interface.php?func=get_conquer&since=%d"
|
||||
)
|
|
@ -1,4 +1,4 @@
|
|||
package tw
|
||||
package twutil
|
||||
|
||||
import (
|
||||
"github.com/tribalwarshelp/shared/models"
|
|
@ -1,4 +1,4 @@
|
|||
package tw
|
||||
package urlbuilder
|
||||
|
||||
import "fmt"
|
||||
|
||||
|
@ -8,14 +8,18 @@ const (
|
|||
EndpointVillageProfile = "/game.php?screen=info_village&id=%d"
|
||||
)
|
||||
|
||||
func BuildVillageURL(server, host string, id int) string {
|
||||
return fmt.Sprintf("https://%s.%s"+EndpointVillageProfile, server, host, id)
|
||||
}
|
||||
|
||||
func BuildTribeURL(server, host string, id int) string {
|
||||
return fmt.Sprintf("https://%s.%s"+EndpointTribeProfile, server, host, id)
|
||||
func BuildServerURL(server, host string) string {
|
||||
return fmt.Sprintf("https://%s.%s", server, host)
|
||||
}
|
||||
|
||||
func BuildPlayerURL(server, host string, id int) string {
|
||||
return fmt.Sprintf("https://%s.%s"+EndpointPlayerProfile, server, host, id)
|
||||
return BuildServerURL(server, host) + fmt.Sprintf(EndpointPlayerProfile, id)
|
||||
}
|
||||
|
||||
func BuildVillageURL(server, host string, id int) string {
|
||||
return BuildServerURL(server, host) + fmt.Sprintf(EndpointVillageProfile, id)
|
||||
}
|
||||
|
||||
func BuildTribeURL(server, host string, id int) string {
|
||||
return BuildServerURL(server, host) + fmt.Sprintf(EndpointTribeProfile, id)
|
||||
}
|
|
@ -1,21 +0,0 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TrackExecutionTime(log *logrus.Entry, fn func(), fnName string) func() {
|
||||
return func() {
|
||||
now := time.Now()
|
||||
log := log.WithField("fnName", fnName)
|
||||
log.Infof("%s: called", fnName)
|
||||
|
||||
fn()
|
||||
|
||||
log.
|
||||
WithField("executionTime", time.Since(now).String()).
|
||||
Infof("%s: finished executing", fnName)
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user