twhelp
/
dataupdater
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

taskLoadServersAndUpdateData: use twdataloader.VersionDataLoader and remove method "getServers"

This commit is contained in:
Dawid Wysokiński 2021-05-03 17:34:36 +02:00
parent 6d4a563f2a
commit 15c7368761
6 changed files with 30 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -5,7 +5,6 @@ go 1.16
require (
github.com/Kichiyaki/appmode v0.0.0-20210502105643-0a26207c548d
github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210502060056-ad595ba7b858
github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83
github.com/Kichiyaki/goutil v0.0.0-20210502135617-7868852b90cc
github.com/go-pg/pg/v10 v10.9.1
github.com/go-redis/redis/v8 v8.8.2
@ -15,7 +14,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.8.1
github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94
github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd
github.com/vmihailenco/taskq/v3 v3.2.3
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect
)

4
go.sum
View File

@ -128,8 +128,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94 h1:tQysqj1zlVBLFiRqVYRlRd4hPg5EBKGwYav7+gjxQMo=
github.com/tribalwarshelp/shared v0.0.0-20210502140609-e137bd656c94/go.mod h1:NDNw8pZ/lbNaORVwGWl22+Czf4pwtwXELR2ExeUn0q0=
github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd h1:iaH5z+mjx2l7603hCFo72MSZXQ5DxCUIi3Fbg6At1dk=
github.com/tribalwarshelp/shared v0.0.0-20210503145038-742945be9fdd/go.mod h1:NDNw8pZ/lbNaORVwGWl22+Czf4pwtwXELR2ExeUn0q0=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
github.com/vmihailenco/msgpack/v5 v5.0.0/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo=

2
internal/cron/tasks/helpers.go
View File

@ -39,7 +39,7 @@ func newHTTPClient() *http.Client {
}
func newServerDataLoader(url string) twdataloader.ServerDataLoader {
return twdataloader.NewServerDataLoader(&twdataloader.Config{
return twdataloader.NewServerDataLoader(&twdataloader.ServerDataLoaderConfig{
BaseURL: url,
Client: newHTTPClient(),
})

4
internal/cron/tasks/task_delete_non_existent_villages.go
View File

@ -2,9 +2,9 @@ package tasks
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/shared/tw/twurlbuilder"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -36,7 +36,7 @@ func (t *taskDeleteNonExistentVillages) execute() error {
Get(TaskNameServerDeleteNonExistentVillages).
WithArgs(
context.Background(),
fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host),
twurlbuilder.BuildServerURL(server.Key, server.Version.Host),
s,
),
)

66
internal/cron/tasks/task_load_servers_and_update_data.go
View File

@ -2,27 +2,25 @@ package tasks
import (
"context"
"fmt"
phpserialize "github.com/Kichiyaki/go-php-serialize"
"github.com/go-pg/pg/v10"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/tribalwarshelp/shared/tw/twdataloader"
"github.com/tribalwarshelp/shared/tw/twmodel"
"io/ioutil"
"net/http"
"github.com/tribalwarshelp/cron/internal/cron/queue"
"github.com/tribalwarshelp/cron/internal/postgres"
)
const (
endpointGetServers = "/backend/get_servers.php"
)
type taskLoadServersAndUpdateData struct {
*task
}
type serverWithURL struct {
*twmodel.Server `pg:",inherit"`
url string
}
func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
if err := t.validatePayload(version); err != nil {
log.Debug(err)
@ -30,20 +28,25 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
}
entry := log.WithField("host", version.Host)
entry.Infof("taskLoadServersAndUpdateData.execute: %s: Loading servers", version.Host)
data, err := t.getServers(version)
loadedServers, err := twdataloader.
NewVersionDataLoader(&twdataloader.VersionDataLoaderConfig{
Host: version.Host,
Client: newHTTPClient(),
}).
LoadServers()
if err != nil {
log.Errorln(err)
return err
}
var serverKeys []string
var servers []*twmodel.Server
for serverKey := range data {
if version.SpecialServers.Contains(serverKey) {
var servers []*serverWithURL
for _, loadedServer := range loadedServers {
if version.SpecialServers.Contains(loadedServer.Key) {
continue
}
server := &twmodel.Server{
Key: serverKey,
Key: loadedServer.Key,
Status: twmodel.ServerStatusOpen,
VersionCode: version.Code,
Version: version,
@ -52,8 +55,11 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
logrus.Warn(errors.Wrapf(err, "taskLoadServersAndUpdateData.execute: %s: couldn't create the schema", server.Key))
continue
}
servers = append(servers, server)
serverKeys = append(serverKeys, serverKey)
servers = append(servers, &serverWithURL{
Server: server,
url: loadedServer.URL,
})
serverKeys = append(serverKeys, server.Key)
}
if len(servers) > 0 {
@ -79,8 +85,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *twmodel.Version) error {
}
for _, server := range servers {
s := server
t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), data[s.Key], s))
t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), server.url, server.Server))
}
entry.Infof("%s: Servers have been loaded", version.Host)
@ -93,30 +98,3 @@ func (t *taskLoadServersAndUpdateData) validatePayload(version *twmodel.Version)
}
return nil
}
func (t *taskLoadServersAndUpdateData) getServers(version *twmodel.Version) (map[string]string, error) {
resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers))
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't load servers", version.Host)
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the response body", version.Host)
}
body, err := phpserialize.Decode(string(bodyBytes))
if err != nil {
return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the response body into the go value", version.Host)
}
result := make(map[string]string)
for serverKey, url := range body.(map[interface{}]interface{}) {
serverKeyStr := serverKey.(string)
urlStr := url.(string)
if serverKeyStr != "" && urlStr != "" {
result[serverKeyStr] = urlStr
}
}
return result, nil
}

4
internal/cron/tasks/task_update_ennoblements.go
View File

@ -2,9 +2,9 @@ package tasks
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/tw/twmodel"
"github.com/tribalwarshelp/shared/tw/twurlbuilder"
"github.com/tribalwarshelp/cron/internal/cron/queue"
)
@ -31,7 +31,7 @@ func (t *taskUpdateEnnoblements) execute() error {
err := t.queue.Add(
queue.EnnoblementsQueue,
Get(TaskUpdateServerEnnoblements).
WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s),
WithArgs(context.Background(), twurlbuilder.BuildServerURL(server.Key, server.Version.Host), s),
)
if err != nil {
log.Warn(