From de68ea117bbc5b7054288f6d8bebfe899f5b24c4 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Tue, 13 Jul 2021 07:30:45 +0200 Subject: [PATCH] add version_data_loader_test.go --- go.mod | 3 + go.sum | 5 + tw/twdataloader/server_data_loader.go | 100 ++++++++++---------- tw/twdataloader/version_data_loader.go | 12 ++- tw/twdataloader/version_data_loader_test.go | 67 +++++++++++++ 5 files changed, 133 insertions(+), 54 deletions(-) create mode 100644 tw/twdataloader/version_data_loader_test.go diff --git a/go.mod b/go.mod index 9ba0f64..75972b1 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,10 @@ require ( github.com/Kichiyaki/gopgutil/v10 v10.0.0-20210521204542-cc672e361b3d github.com/go-pg/pg/v10 v10.10.2 github.com/pkg/errors v0.9.1 + github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/testify v1.7.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.1 // indirect go.opentelemetry.io/otel v0.20.0 // indirect golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index c263c6d..937728c 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= @@ -177,6 +180,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= diff --git a/tw/twdataloader/server_data_loader.go b/tw/twdataloader/server_data_loader.go index 38ebee1..d58b1c2 100644 --- a/tw/twdataloader/server_data_loader.go +++ b/tw/twdataloader/server_data_loader.go @@ -49,7 +49,7 @@ type parsedODLine struct { Score int } -func (d *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) { +func (dl *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) { if len(line) != 3 { return nil, errors.New("invalid line format (should be rank,id,score)") } @@ -70,19 +70,19 @@ func (d *ServerDataLoader) parseODLine(line []string) (*parsedODLine, error) { return p, nil } -func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeated, error) { +func (dl *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeated, error) { m := make(map[int]*twmodel.OpponentsDefeated) formattedURLs := []string{ - fmt.Sprintf("%s%s", d.baseURL, EndpointKillAll), - fmt.Sprintf("%s%s", d.baseURL, EndpointKillAtt), - fmt.Sprintf("%s%s", d.baseURL, EndpointKillDef), - fmt.Sprintf("%s%s", d.baseURL, EndpointKillSup), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAll), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAtt), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillDef), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillSup), } if tribe { formattedURLs = []string{ - fmt.Sprintf("%s%s", d.baseURL, EndpointKillAllTribe), - fmt.Sprintf("%s%s", d.baseURL, EndpointKillAttTribe), - fmt.Sprintf("%s%s", d.baseURL, EndpointKillDefTribe), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAllTribe), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillAttTribe), + fmt.Sprintf("%s%s", dl.baseURL, EndpointKillDefTribe), "", } } @@ -90,16 +90,16 @@ func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeate if formattedURL == "" { continue } - lines, err := d.getCSVData(formattedURL, true) + lines, err := dl.getCSVData(formattedURL, true) if err != nil { //fallback to not gzipped file - lines, err = d.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false) + lines, err = dl.getCSVData(strings.ReplaceAll(formattedURL, ".gz", ""), false) if err != nil { return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) } } for _, line := range lines { - parsed, err := d.parseODLine(line) + parsed, err := dl.parseODLine(line) if err != nil { return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } @@ -125,7 +125,7 @@ func (d *ServerDataLoader) LoadOD(tribe bool) (map[int]*twmodel.OpponentsDefeate return m, nil } -func (d *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, error) { +func (dl *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, error) { if len(line) != 6 { return nil, errors.New("invalid line format (should be id,name,tribeid,villages,points,rank)") } @@ -163,11 +163,11 @@ func (d *ServerDataLoader) parsePlayerLine(line []string) (*twmodel.Player, erro return player, nil } -func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) { - formattedURL := d.baseURL + EndpointPlayer - lines, err := d.getCSVData(formattedURL, true) +func (dl *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) { + formattedURL := dl.baseURL + EndpointPlayer + lines, err := dl.getCSVData(formattedURL, true) if err != nil { - lines, err = d.getCSVData(d.baseURL+EndpointPlayerNotGzipped, false) + lines, err = dl.getCSVData(dl.baseURL+EndpointPlayerNotGzipped, false) if err != nil { return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL) } @@ -175,7 +175,7 @@ func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) { var players []*twmodel.Player for _, line := range lines { - player, err := d.parsePlayerLine(line) + player, err := dl.parsePlayerLine(line) if err != nil { return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } @@ -185,7 +185,7 @@ func (d *ServerDataLoader) LoadPlayers() ([]*twmodel.Player, error) { return players, nil } -func (d *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error) { +func (dl *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error) { if len(line) != 8 { return nil, errors.New("invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)") } @@ -231,18 +231,18 @@ func (d *ServerDataLoader) parseTribeLine(line []string) (*twmodel.Tribe, error) return tribe, nil } -func (d *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) { - formattedURL := d.baseURL + EndpointTribe - lines, err := d.getCSVData(formattedURL, true) +func (dl *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) { + formattedURL := dl.baseURL + EndpointTribe + lines, err := dl.getCSVData(formattedURL, true) if err != nil { - lines, err = d.getCSVData(d.baseURL+EndpointTribeNotGzipped, false) + lines, err = dl.getCSVData(dl.baseURL+EndpointTribeNotGzipped, false) if err != nil { return nil, errors.Wrapf(err, "couldn't load data, url %s", formattedURL) } } var tribes []*twmodel.Tribe for _, line := range lines { - tribe, err := d.parseTribeLine(line) + tribe, err := dl.parseTribeLine(line) if err != nil { return nil, errors.Wrapf(err, "couldn't parse the line, url %s, line %s", formattedURL, strings.Join(line, ",")) } @@ -251,7 +251,7 @@ func (d *ServerDataLoader) LoadTribes() ([]*twmodel.Tribe, error) { return tribes, nil } -func (d *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, error) { +func (dl *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, error) { if len(line) != 7 { return nil, errors.New("invalid line format (should be id,name,x,y,playerID,points,bonus)") } @@ -288,18 +288,18 @@ func (d *ServerDataLoader) parseVillageLine(line []string) (*twmodel.Village, er return village, nil } -func (d *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) { - formattedURL := d.baseURL + EndpointVillage - lines, err := d.getCSVData(formattedURL, true) +func (dl *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) { + formattedURL := dl.baseURL + EndpointVillage + lines, err := dl.getCSVData(formattedURL, true) if err != nil { - lines, err = d.getCSVData(d.baseURL+EndpointVillageNotGzipped, false) + lines, err = dl.getCSVData(dl.baseURL+EndpointVillageNotGzipped, false) if err != nil { return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) } } var villages []*twmodel.Village for _, line := range lines { - village, err := d.parseVillageLine(line) + village, err := dl.parseVillageLine(line) if err != nil { return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ",")) } @@ -308,7 +308,7 @@ func (d *ServerDataLoader) LoadVillages() ([]*twmodel.Village, error) { return villages, nil } -func (d *ServerDataLoader) parseEnnoblementLine(line []string) (*twmodel.Ennoblement, error) { +func (dl *ServerDataLoader) parseEnnoblementLine(line []string) (*twmodel.Ennoblement, error) { if len(line) != 4 { return nil, errors.New("invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)") } @@ -339,20 +339,20 @@ type LoadEnnoblementsConfig struct { EnnobledAtGT time.Time } -func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twmodel.Ennoblement, error) { +func (dl *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twmodel.Ennoblement, error) { if cfg == nil { cfg = &LoadEnnoblementsConfig{} } yesterdaysDate := time.Now().Add(-23 * time.Hour) - formattedURL := d.baseURL + EndpointConquer + formattedURL := dl.baseURL + EndpointConquer compressed := true if cfg.EnnobledAtGT.After(yesterdaysDate) || cfg.EnnobledAtGT.Equal(yesterdaysDate) { - formattedURL = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix()) + formattedURL = dl.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGT.Unix()) compressed = false } - lines, err := d.getCSVData(formattedURL, compressed) + lines, err := dl.getCSVData(formattedURL, compressed) if err != nil && compressed { - lines, err = d.getCSVData(d.baseURL+EndpointConquerNotGzipped, false) + lines, err = dl.getCSVData(dl.baseURL+EndpointConquerNotGzipped, false) } if err != nil { return nil, errors.Wrapf(err, "couldn't load data, formattedURL %s", formattedURL) @@ -360,7 +360,7 @@ func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twm var ennoblements []*twmodel.Ennoblement for _, line := range lines { - ennoblement, err := d.parseEnnoblementLine(line) + ennoblement, err := dl.parseEnnoblementLine(line) if err != nil { return nil, errors.Wrapf(err, "couldn't parse the line, formattedURL %s, line %s", formattedURL, strings.Join(line, ",")) } @@ -371,38 +371,38 @@ func (d *ServerDataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*twm return ennoblements, nil } -func (d *ServerDataLoader) GetConfig() (*twmodel.ServerConfig, error) { - formattedURL := d.baseURL + EndpointConfig +func (dl *ServerDataLoader) GetConfig() (*twmodel.ServerConfig, error) { + formattedURL := dl.baseURL + EndpointConfig cfg := &twmodel.ServerConfig{} - err := d.getXML(formattedURL, cfg) + err := dl.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getConfig") } return cfg, nil } -func (d *ServerDataLoader) GetBuildingConfig() (*twmodel.BuildingConfig, error) { - formattedURL := d.baseURL + EndpointBuildingConfig +func (dl *ServerDataLoader) GetBuildingConfig() (*twmodel.BuildingConfig, error) { + formattedURL := dl.baseURL + EndpointBuildingConfig cfg := &twmodel.BuildingConfig{} - err := d.getXML(formattedURL, cfg) + err := dl.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getBuildingConfig") } return cfg, nil } -func (d *ServerDataLoader) GetUnitConfig() (*twmodel.UnitConfig, error) { - formattedURL := d.baseURL + EndpointUnitConfig +func (dl *ServerDataLoader) GetUnitConfig() (*twmodel.UnitConfig, error) { + formattedURL := dl.baseURL + EndpointUnitConfig cfg := &twmodel.UnitConfig{} - err := d.getXML(formattedURL, cfg) + err := dl.getXML(formattedURL, cfg) if err != nil { return nil, errors.Wrap(err, "getUnitConfig") } return cfg, nil } -func (d *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string, error) { - resp, err := d.client.Get(url) +func (dl *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string, error) { + resp, err := dl.client.Get(url) if err != nil { return nil, err } @@ -413,8 +413,8 @@ func (d *ServerDataLoader) getCSVData(url string, compressed bool) ([][]string, return uncompressAndReadCsvLines(resp.Body) } -func (d *ServerDataLoader) getXML(url string, decode interface{}) error { - resp, err := d.client.Get(url) +func (dl *ServerDataLoader) getXML(url string, decode interface{}) error { + resp, err := dl.client.Get(url) if err != nil { return err } diff --git a/tw/twdataloader/version_data_loader.go b/tw/twdataloader/version_data_loader.go index 2cbafc4..db26962 100644 --- a/tw/twdataloader/version_data_loader.go +++ b/tw/twdataloader/version_data_loader.go @@ -40,8 +40,8 @@ func NewVersionDataLoader(cfg *VersionDataLoaderConfig) *VersionDataLoader { } } -func (d *VersionDataLoader) LoadServers() ([]*Server, error) { - resp, err := d.client.Get(fmt.Sprintf("https://%s%s", d.host, EndpointGetServers)) +func (dl *VersionDataLoader) LoadServers() ([]*Server, error) { + resp, err := dl.client.Get(fmt.Sprintf("https://%s%s", dl.host, EndpointGetServers)) if err != nil { return nil, errors.Wrap(err, "couldn't load servers") } @@ -52,8 +52,12 @@ func (d *VersionDataLoader) LoadServers() ([]*Server, error) { return nil, errors.Wrap(err, "couldn't read the response body") } body, err := phpserialize.Decode(string(bodyBytes)) - if err != nil { - return nil, errors.Wrap(err, "couldn't decode the response body into the go value") + if err != nil || body == nil { + fmtedErr := errors.New("couldn't decode the response body into a go value") + if err != nil { + fmtedErr = errors.Wrap(err, fmtedErr.Error()) + } + return nil, fmtedErr } var servers []*Server diff --git a/tw/twdataloader/version_data_loader_test.go b/tw/twdataloader/version_data_loader_test.go new file mode 100644 index 0000000..f08abed --- /dev/null +++ b/tw/twdataloader/version_data_loader_test.go @@ -0,0 +1,67 @@ +package twdataloader + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" +) + +func prepareTestServer(resp string) *httptest.Server { + return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case EndpointGetServers: + _, err := fmt.Fprintln(w, resp) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + return + default: + w.WriteHeader(http.StatusNotFound) + } + })) +} + +func TestLoadServers(t *testing.T) { + t.Run("invalid payload", func(t *testing.T) { + resp := `:"https://pl165.plemiona.pl";s:5:"pl166";s:25:"https://pl166.plemiona.pl";s:5:"pl167";s:25:"https://pl167.plemiona.pl";}` + ts := prepareTestServer(resp) + defer ts.Close() + + dl := NewVersionDataLoader(&VersionDataLoaderConfig{ + Host: strings.ReplaceAll(ts.URL, "https://", ""), + Client: ts.Client(), + }) + + _, err := dl.LoadServers() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "couldn't decode the response body into a go value") + }) + + t.Run("success", func(t *testing.T) { + resp := `a:19:{s:5:"pl150";s:25:"https://pl150.plemiona.pl";s:5:"pl151";s:25:"https://pl151.plemiona.pl";s:5:"pl152";s:25:"https://pl152.plemiona.pl";s:5:"pl153";s:25:"https://pl153.plemiona.pl";s:5:"pl154";s:25:"https://pl154.plemiona.pl";s:5:"pl155";s:25:"https://pl155.plemiona.pl";s:5:"pl156";s:25:"https://pl156.plemiona.pl";s:5:"pl157";s:25:"https://pl157.plemiona.pl";s:5:"pl158";s:25:"https://pl158.plemiona.pl";s:5:"pl159";s:25:"https://pl159.plemiona.pl";s:5:"pl160";s:25:"https://pl160.plemiona.pl";s:5:"pl161";s:25:"https://pl161.plemiona.pl";s:5:"pl162";s:25:"https://pl162.plemiona.pl";s:5:"pl163";s:25:"https://pl163.plemiona.pl";s:5:"pl164";s:25:"https://pl164.plemiona.pl";s:4:"plp7";s:24:"https://plp7.plemiona.pl";s:5:"pl165";s:25:"https://pl165.plemiona.pl";s:5:"pl166";s:25:"https://pl166.plemiona.pl";s:5:"pl167";s:25:"https://pl167.plemiona.pl";}` + ts := prepareTestServer(resp) + defer ts.Close() + + expectedLength, err := strconv.Atoi(strings.Split(resp, ":")[1]) + assert.Nil(t, err) + + dl := NewVersionDataLoader(&VersionDataLoaderConfig{ + Host: strings.ReplaceAll(ts.URL, "https://", ""), + Client: ts.Client(), + }) + + servers, err := dl.LoadServers() + assert.Nil(t, err) + cnt := 0 + for _, server := range servers { + if strings.Contains(resp, server.URL) && strings.Contains(resp, server.Key) { + cnt++ + } + } + assert.Equal(t, expectedLength, cnt) + }) +}