add version_data_loader_test.go

This commit is contained in:
Dawid Wysokiński 2021-07-13 07:30:45 +02:00
parent 6456a82235
commit de68ea117b
5 changed files with 133 additions and 54 deletions

3
go.mod
View File

@ -7,7 +7,10 @@ require (
github.com/Kichiyaki/gopgutil/v10 v10.0.0-20210521204542-cc672e361b3d
github.com/go-pg/pg/v10 v10.10.2
github.com/pkg/errors v0.9.1
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.1 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

5
go.sum
View File

@ -68,6 +68,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
@ -177,6 +180,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=

View File

@ -49,7 +49,7 @@ type parsedODLine struct {
Score int
}
func (d *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) {
func (dl *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) {
if len(line) != 3 {
return nil, errors.New("invalid line format (should be rank,id,score)")
}
@ -70,19 +70,19 @@ func (d *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) {
return p, nil
}
func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeated, error) {
func (dl *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeated, error) {
m := make(map[int]*twmodel.OpponentsDefeated)
formattedURLs := []string{
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAll),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAtt),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDef),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillSup),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAll),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAtt),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillDef),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillSup),
}
if tribe {
formattedURLs = []string{
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAllTribe),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAttTribe),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDefTribe),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAllTribe),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAttTribe),
fmt.Sprintf("%s%s", dl.baseURL, EndpointKillDefTribe),
"",
}
}
@ -90,16 +90,16 @@ func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeate
if formattedURL == "" {
continue
}
lines, err := d.getCSVData(formattedURL, true)
lines, err := dl.getCSVData(formattedURL, true)
if err != nil {
//fallback to not gzipped file
lines, err = d.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false)
lines, err = dl.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false)
if err != nil {
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
}
}
for _, line := range lines {
parsed, err := d.parseODLine(line)
parsed, err := dl.parseODLine(line)
if err != nil {
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
}
@ -125,7 +125,7 @@ func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeate
return m, nil
}
func (d *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, error) {
func (dl *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, error) {
if len(line) != 6 {
return nil, errors.New("invalid line format (should be id,name,tribeid,villages,points,rank)")
}
@ -163,11 +163,11 @@ func (d *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, erro
return player, nil
}
func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) {
formattedURL := d.baseURL + EndpointPlayer
lines, err := d.getCSVData(formattedURL, true)
func (dl *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) {
formattedURL := dl.baseURL + EndpointPlayer
lines, err := dl.getCSVData(formattedURL, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointPlayerNotGzipped, false)
lines, err = dl.getCSVData(dl.baseURL+EndpointPlayerNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL)
}
@ -175,7 +175,7 @@ func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) {
var players []*twmodel.Player
for _, line := range lines {
player, err := d.parsePlayerLine(line)
player, err := dl.parsePlayerLine(line)
if err != nil {
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
}
@ -185,7 +185,7 @@ func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) {
return players, nil
}
func (d *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error) {
func (dl *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error) {
if len(line) != 8 {
return nil, errors.New("invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)")
}
@ -231,18 +231,18 @@ func (d *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error)
return tribe, nil
}
func (d *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) {
formattedURL := d.baseURL + EndpointTribe
lines, err := d.getCSVData(formattedURL, true)
func (dl *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) {
formattedURL := dl.baseURL + EndpointTribe
lines, err := dl.getCSVData(formattedURL, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointTribeNotGzipped, false)
lines, err = dl.getCSVData(dl.baseURL+EndpointTribeNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL)
}
}
var tribes []*twmodel.Tribe
for _, line := range lines {
tribe, err := d.parseTribeLine(line)
tribe, err := dl.parseTribeLine(line)
if err != nil {
return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ","))
}
@ -251,7 +251,7 @@ func (d *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) {
return tribes, nil
}
func (d *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, error) {
func (dl *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, error) {
if len(line) != 7 {
return nil, errors.New("invalid line format (should be id,name,x,y,playerID,points,bonus)")
}
@ -288,18 +288,18 @@ func (d *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, er
return village, nil
}
func (d *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) {
formattedURL := d.baseURL + EndpointVillage
lines, err := d.getCSVData(formattedURL, true)
func (dl *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) {
formattedURL := dl.baseURL + EndpointVillage
lines, err := dl.getCSVData(formattedURL, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointVillageNotGzipped, false)
lines, err = dl.getCSVData(dl.baseURL+EndpointVillageNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
}
}
var villages []*twmodel.Village
for _, line := range lines {
village, err := d.parseVillageLine(line)
village, err := dl.parseVillageLine(line)
if err != nil {
return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ","))
}
@ -308,7 +308,7 @@ func (d *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) {
return villages, nil
}
func (d *ServerDataLoader) parseEnnoblementLine(line []string) (*twmodel.Ennoblement, error) {
func (dl *ServerDataLoader) parseEnnoblementLine(line []string) (*twmodel.Ennoblement, error) {
if len(line) != 4 {
return nil, errors.New("invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)")
}
@ -339,20 +339,20 @@ type LoadEnnoblementsConfig struct {
EnnobledAtGT time.Time
}
func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twmodel.Ennoblement, error) {
func (dl *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twmodel.Ennoblement, error) {
if cfg == nil {
cfg = &LoadEnnoblementsConfig{}
}
yesterdaysDate := time.Now().Add(-23 * time.Hour)
formattedURL := d.baseURL + EndpointConquer
formattedURL := dl.baseURL + EndpointConquer
compressed := true
if cfg.EnnobledAtGT.After(yesterdaysDate) || cfg.EnnobledAtGT.Equal(yesterdaysDate) {
formattedURL = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix())
formattedURL = dl.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix())
compressed = false
}
lines, err := d.getCSVData(formattedURL, compressed)
lines, err := dl.getCSVData(formattedURL, compressed)
if err != nil && compressed {
lines, err = d.getCSVData(d.baseURL+EndpointConquerNotGzipped, false)
lines, err = dl.getCSVData(dl.baseURL+EndpointConquerNotGzipped, false)
}
if err != nil {
return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL)
@ -360,7 +360,7 @@ func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twm
var ennoblements []*twmodel.Ennoblement
for _, line := range lines {
ennoblement, err := d.parseEnnoblementLine(line)
ennoblement, err := dl.parseEnnoblementLine(line)
if err != nil {
return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ","))
}
@ -371,38 +371,38 @@ func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twm
return ennoblements, nil
}
func (d *ServerDataLoader) GetConfig() (*twmodel.ServerConfig, error) {
formattedURL := d.baseURL + EndpointConfig
func (dl *ServerDataLoader) GetConfig() (*twmodel.ServerConfig, error) {
formattedURL := dl.baseURL + EndpointConfig
cfg := &twmodel.ServerConfig{}
err := d.getXML(formattedURL, cfg)
err := dl.getXML(formattedURL, cfg)
if err != nil {
return nil, errors.Wrap(err, "getConfig")
}
return cfg, nil
}
func (d *ServerDataLoader) GetBuildingConfig() (*twmodel.BuildingConfig, error) {
formattedURL := d.baseURL + EndpointBuildingConfig
func (dl *ServerDataLoader) GetBuildingConfig() (*twmodel.BuildingConfig, error) {
formattedURL := dl.baseURL + EndpointBuildingConfig
cfg := &twmodel.BuildingConfig{}
err := d.getXML(formattedURL, cfg)
err := dl.getXML(formattedURL, cfg)
if err != nil {
return nil, errors.Wrap(err, "getBuildingConfig")
}
return cfg, nil
}
func (d *ServerDataLoader) GetUnitConfig() (*twmodel.UnitConfig, error) {
formattedURL := d.baseURL + EndpointUnitConfig
func (dl *ServerDataLoader) GetUnitConfig() (*twmodel.UnitConfig, error) {
formattedURL := dl.baseURL + EndpointUnitConfig
cfg := &twmodel.UnitConfig{}
err := d.getXML(formattedURL, cfg)
err := dl.getXML(formattedURL, cfg)
if err != nil {
return nil, errors.Wrap(err, "getUnitConfig")
}
return cfg, nil
}
func (d *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string, error) {
resp, err := d.client.Get(url)
func (dl *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string, error) {
resp, err := dl.client.Get(url)
if err != nil {
return nil, err
}
@ -413,8 +413,8 @@ func (d *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string,
return uncompressAndReadCsvLines(resp.Body)
}
func (d *ServerDataLoader) getXML(url string, decode interface{}) error {
resp, err := d.client.Get(url)
func (dl *ServerDataLoader) getXML(url string, decode interface{}) error {
resp, err := dl.client.Get(url)
if err != nil {
return err
}

View File

@ -40,8 +40,8 @@ func NewVersionDataLoader(cfg *VersionDataLoaderConfig) *VersionDataLoader {
}
}
func (d *VersionDataLoader) LoadServers() ([]*Server, error) {
resp, err := d.client.Get(fmt.Sprintf("https://%s%s", d.host, EndpointGetServers))
func (dl *VersionDataLoader) LoadServers() ([]*Server, error) {
resp, err := dl.client.Get(fmt.Sprintf("https://%s%s", dl.host, EndpointGetServers))
if err != nil {
return nil, errors.Wrap(err, "couldn't load servers")
}
@ -52,8 +52,12 @@ func (d *VersionDataLoader) LoadServers() ([]*Server, error) {
return nil, errors.Wrap(err, "couldn't read the response body")
}
body, err := phpserialize.Decode(string(bodyBytes))
if err != nil {
return nil, errors.Wrap(err, "couldn't decode the response body into the go value")
if err != nil || body == nil {
fmtedErr := errors.New("couldn't decode the response body into a go value")
if err != nil {
fmtedErr = errors.Wrap(err, fmtedErr.Error())
}
return nil, fmtedErr
}
var servers []*Server

View File

@ -0,0 +1,67 @@
package twdataloader
import (
"fmt"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
)
func prepareTestServer(resp string) *httptest.Server {
return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case EndpointGetServers:
_, err := fmt.Fprintln(w, resp)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
return
default:
w.WriteHeader(http.StatusNotFound)
}
}))
}
func TestLoadServers(t *testing.T) {
t.Run("invalid payload", func(t *testing.T) {
resp := `:"https://pl165.plemiona.pl";s:5:"pl166";s:25:"https://pl166.plemiona.pl";s:5:"pl167";s:25:"https://pl167.plemiona.pl";}`
ts := prepareTestServer(resp)
defer ts.Close()
dl := NewVersionDataLoader(&VersionDataLoaderConfig{
Host: strings.ReplaceAll(ts.URL, "https://", ""),
Client: ts.Client(),
})
_, err := dl.LoadServers()
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "couldn't decode the response body into a go value")
})
t.Run("success", func(t *testing.T) {
resp := `a:19:{s:5:"pl150";s:25:"https://pl150.plemiona.pl";s:5:"pl151";s:25:"https://pl151.plemiona.pl";s:5:"pl152";s:25:"https://pl152.plemiona.pl";s:5:"pl153";s:25:"https://pl153.plemiona.pl";s:5:"pl154";s:25:"https://pl154.plemiona.pl";s:5:"pl155";s:25:"https://pl155.plemiona.pl";s:5:"pl156";s:25:"https://pl156.plemiona.pl";s:5:"pl157";s:25:"https://pl157.plemiona.pl";s:5:"pl158";s:25:"https://pl158.plemiona.pl";s:5:"pl159";s:25:"https://pl159.plemiona.pl";s:5:"pl160";s:25:"https://pl160.plemiona.pl";s:5:"pl161";s:25:"https://pl161.plemiona.pl";s:5:"pl162";s:25:"https://pl162.plemiona.pl";s:5:"pl163";s:25:"https://pl163.plemiona.pl";s:5:"pl164";s:25:"https://pl164.plemiona.pl";s:4:"plp7";s:24:"https://plp7.plemiona.pl";s:5:"pl165";s:25:"https://pl165.plemiona.pl";s:5:"pl166";s:25:"https://pl166.plemiona.pl";s:5:"pl167";s:25:"https://pl167.plemiona.pl";}`
ts := prepareTestServer(resp)
defer ts.Close()
expectedLength, err := strconv.Atoi(strings.Split(resp, ":")[1])
assert.Nil(t, err)
dl := NewVersionDataLoader(&VersionDataLoaderConfig{
Host: strings.ReplaceAll(ts.URL, "https://", ""),
Client: ts.Client(),
})
servers, err := dl.LoadServers()
assert.Nil(t, err)
cnt := 0
for _, server := range servers {
if strings.Contains(resp, server.URL) && strings.Contains(resp, server.Key) {
cnt++
}
}
assert.Equal(t, expectedLength, cnt)
})
}