diff --git a/go.mod b/go.mod index 016824f..4c21dc1 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.16 require ( github.com/Kichiyaki/appmode v0.0.0-20210502105643-0a26207c548d github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210502060056-ad595ba7b858 - github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 github.com/Kichiyaki/goutil v0.0.0-20210502135617-7868852b90cc github.com/go-pg/pg/v10 v10.9.1 github.com/go-redis/redis/v8 v8.8.2 @@ -15,7 +14,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 - github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94 + github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd github.com/vmihailenco/taskq/v3 v3.2.3 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect ) diff --git a/go.sum b/go.sum index 5049a20..4f158da 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= -github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94 h1:tQysqj1zlVBLFiRqVYRlRd4hPg5EBKGwYav7+gjxQMo= -github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94/go.mod h1:NDNw8pZ/lbNaORVwGWl22+Czf4pwtwXELR2ExeUn0q0= +github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd h1:iaH5z+mjx2l7603hCFo72MSZXQ5DxCUIi3Fbg6At1dk= +github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd/go.mod h1:NDNw8pZ/lbNaORVwGWl22+Czf4pwtwXELR2ExeUn0q0= github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= github.com/vmihailenco/msgpack/v5 v5.0.0/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= diff --git a/internal/cron/tasks/helpers.go b/internal/cron/tasks/helpers.go index 448d4f4..eb51d18 100644 --- a/internal/cron/tasks/helpers.go +++ b/internal/cron/tasks/helpers.go @@ -39,7 +39,7 @@ func newHTTPClient() *http.Client { } func newServerDataLoader(url string) twdataloader.ServerDataLoader { - return twdataloader.NewServerDataLoader(&twdataloader.Config{ + return twdataloader.NewServerDataLoader(&twdataloader.ServerDataLoaderConfig{ BaseURL: url, Client: newHTTPClient(), }) diff --git a/internal/cron/tasks/task_delete_non_existent_villages.go b/internal/cron/tasks/task_delete_non_existent_villages.go index 5d88ee7..9546c98 100644 --- a/internal/cron/tasks/task_delete_non_existent_villages.go +++ b/internal/cron/tasks/task_delete_non_existent_villages.go @@ -2,9 +2,9 @@ package tasks import ( "context" - "fmt" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/tw/twmodel" + "github.com/tribalwarshelp/shared/tw/twurlbuilder" "github.com/tribalwarshelp/cron/internal/cron/queue" ) @@ -36,7 +36,7 @@ func (t *taskDeleteNonExistentVillages) execute() error { Get(TaskNameServerDeleteNonExistentVillages). WithArgs( context.Background(), - fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), + twurlbuilder.BuildServerURL(server.Key, server.Version.Host), s, ), ) diff --git a/internal/cron/tasks/task_load_servers_and_update_data.go b/internal/cron/tasks/task_load_servers_and_update_data.go index ff7f5fd..e5e8afb 100644 --- a/internal/cron/tasks/task_load_servers_and_update_data.go +++ b/internal/cron/tasks/task_load_servers_and_update_data.go @@ -2,27 +2,25 @@ package tasks import ( "context" - "fmt" - phpserialize "github.com/Kichiyaki/go-php-serialize" "github.com/go-pg/pg/v10" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/tribalwarshelp/shared/tw/twdataloader" "github.com/tribalwarshelp/shared/tw/twmodel" - "io/ioutil" - "net/http" "github.com/tribalwarshelp/cron/internal/cron/queue" "github.com/tribalwarshelp/cron/internal/postgres" ) -const ( - endpointGetServers = "/backend/get_servers.php" -) - type taskLoadServersAndUpdateData struct { *task } +type serverWithURL struct { + *twmodel.Server `pg:",inherit"` + url string +} + func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error { if err := t.validatePayload(version); err != nil { log.Debug(err) @@ -30,20 +28,25 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error { } entry := log.WithField("host", version.Host) entry.Infof("taskLoadServersAndUpdateData.execute: %s: Loading servers", version.Host) - data, err := t.getServers(version) + loadedServers, err := twdataloader. + NewVersionDataLoader(&twdataloader.VersionDataLoaderConfig{ + Host: version.Host, + Client: newHTTPClient(), + }). + LoadServers() if err != nil { log.Errorln(err) return err } var serverKeys []string - var servers []*twmodel.Server - for serverKey := range data { - if version.SpecialServers.Contains(serverKey) { + var servers []*serverWithURL + for _, loadedServer := range loadedServers { + if version.SpecialServers.Contains(loadedServer.Key) { continue } server := &twmodel.Server{ - Key: serverKey, + Key: loadedServer.Key, Status: twmodel.ServerStatusOpen, VersionCode: version.Code, Version: version, @@ -52,8 +55,11 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error { logrus.Warn(errors.Wrapf(err, "taskLoadServersAndUpdateData.execute: %s: couldn't create the schema", server.Key)) continue } - servers = append(servers, server) - serverKeys = append(serverKeys, serverKey) + servers = append(servers, &serverWithURL{ + Server: server, + url: loadedServer.URL, + }) + serverKeys = append(serverKeys, server.Key) } if len(servers) > 0 { @@ -79,8 +85,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error { } for _, server := range servers { - s := server - t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), data[s.Key], s)) + t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), server.url, server.Server)) } entry.Infof("%s: Servers have been loaded", version.Host) @@ -93,30 +98,3 @@ func (t *taskLoadServersAndUpdateData) validatePayload(version *twmodel.Version) } return nil } - -func (t *taskLoadServersAndUpdateData) getServers(version *twmodel.Version) (map[string]string, error) { - resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers)) - if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't load servers", version.Host) - } - defer resp.Body.Close() - - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the response body", version.Host) - } - body, err := phpserialize.Decode(string(bodyBytes)) - if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the response body into the go value", version.Host) - } - - result := make(map[string]string) - for serverKey, url := range body.(map[interface{}]interface{}) { - serverKeyStr := serverKey.(string) - urlStr := url.(string) - if serverKeyStr != "" && urlStr != "" { - result[serverKeyStr] = urlStr - } - } - return result, nil -} diff --git a/internal/cron/tasks/task_update_ennoblements.go b/internal/cron/tasks/task_update_ennoblements.go index 428a18c..f685e56 100644 --- a/internal/cron/tasks/task_update_ennoblements.go +++ b/internal/cron/tasks/task_update_ennoblements.go @@ -2,9 +2,9 @@ package tasks import ( "context" - "fmt" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/tw/twmodel" + "github.com/tribalwarshelp/shared/tw/twurlbuilder" "github.com/tribalwarshelp/cron/internal/cron/queue" ) @@ -31,7 +31,7 @@ func (t *taskUpdateEnnoblements) execute() error { err := t.queue.Add( queue.EnnoblementsQueue, Get(TaskUpdateServerEnnoblements). - WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s), + WithArgs(context.Background(), twurlbuilder.BuildServerURL(server.Key, server.Version.Host), s), ) if err != nil { log.Warn(