twhelp
/
dataupdater
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cron should not store special servers in db, bump github.com/tribalwarshelp/shared

This commit is contained in:
Dawid Wysokiński 2020-06-22 10:52:12 +02:00 committed by Kichiyaki
parent 542b6301b0
commit e9ecec51e8
4 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cron/handler.go
View File

@ -113,8 +113,9 @@ func (h *handler) init() error {
defer tx.Close()
models := []interface{}{
(*models.LangVersion)(nil),
(*models.SpecialServer)(nil),
(*models.Server)(nil),
(*models.LangVersion)(nil),
}
for _, model := range models {
@ -173,33 +174,36 @@ func (h *handler) createSchema(key string) error {
}
func (h *handler) getServers() ([]*models.Server, map[string]string, error) {
versions := []*models.LangVersion{}
if err := h.db.Model(&versions).Select(); err != nil {
langVersions := []*models.LangVersion{}
if err := h.db.Model(&langVersions).Relation("SpecialServers").Select(); err != nil {
return nil, nil, errors.Wrap(err, "getServers")
}
serverKeys := []string{}
servers := []*models.Server{}
urls := make(map[string]string)
for _, version := range versions {
resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers))
for _, langVersion := range langVersions {
resp, err := http.Get(fmt.Sprintf("https://%s%s", langVersion.Host, endpointGetServers))
if err != nil {
log.Print(errors.Wrapf(err, "Cannot fetch servers from %s", version.Host))
log.Print(errors.Wrapf(err, "Cannot fetch servers from %s", langVersion.Host))
continue
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Print(errors.Wrapf(err, "Cannot read response body from %s", version.Host))
log.Print(errors.Wrapf(err, "Cannot read response body from %s", langVersion.Host))
continue
}
body, err := phpserialize.Decode(string(bodyBytes))
if err != nil {
log.Print(errors.Wrapf(err, "Cannot serialize body from %s into go value", version.Host))
log.Print(errors.Wrapf(err, "Cannot serialize body from %s into go value", langVersion.Host))
continue
}
for serverKey, url := range body.(map[interface{}]interface{}) {
serverKeyStr := serverKey.(string)
if langVersion.SpecialServers.Contains(serverKeyStr) {
continue
}
if err := h.createSchema(serverKeyStr); err != nil {
log.Print(errors.Wrapf(err, "Cannot create schema for %s", serverKey))
continue
@ -209,8 +213,8 @@ func (h *handler) getServers() ([]*models.Server, map[string]string, error) {
servers = append(servers, &models.Server{
Key: serverKeyStr,
Status: models.ServerStatusOpen,
LangVersionTag: version.Tag,
LangVersion: version,
LangVersionTag: langVersion.Tag,
LangVersion: langVersion,
})
}
}

4
cron/helpers.go
View File

@ -16,7 +16,7 @@ var client = &http.Client{
Timeout: 20 * time.Second,
}
func uncompressAndGetCsvLines(r io.Reader) ([][]string, error) {
func uncompressAndReadCsvLines(r io.Reader) ([][]string, error) {
uncompressedStream, err := gzip.NewReader(r)
if err != nil {
return [][]string{}, err
@ -34,7 +34,7 @@ func getCSVData(url string, compressed bool) ([][]string, error) {
if !compressed {
return csv.NewReader(resp.Body).ReadAll()
}
return uncompressAndGetCsvLines(resp.Body)
return uncompressAndReadCsvLines(resp.Body)
}
func getXML(url string, decode interface{}) error {

2
go.mod
View File

@ -8,6 +8,6 @@ require (
github.com/joho/godotenv v1.3.0
github.com/pkg/errors v0.9.1
github.com/robfig/cron/v3 v3.0.1
github.com/tribalwarshelp/shared v0.0.0-20200621115742-5ae0d5ec33f2
github.com/tribalwarshelp/shared v0.0.0-20200622084436-3a768c8bf574
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b // indirect
)

4
go.sum
View File

@ -114,6 +114,10 @@ github.com/tribalwarshelp/shared v0.0.0-20200621083628-e0fdaa9bdff6 h1:HjOXWkYjC
github.com/tribalwarshelp/shared v0.0.0-20200621083628-e0fdaa9bdff6/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
github.com/tribalwarshelp/shared v0.0.0-20200621115742-5ae0d5ec33f2 h1:xHR69AkeSO34WkjYlTSzD6v2X37hY68e53o6GEoIyFs=
github.com/tribalwarshelp/shared v0.0.0-20200621115742-5ae0d5ec33f2/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
github.com/tribalwarshelp/shared v0.0.0-20200622075646-33cddd72193a h1:f7Ldd7UJf0NF3Vcbc5UfIIRqqkI4zgXxbKEwPY2LmYM=
github.com/tribalwarshelp/shared v0.0.0-20200622075646-33cddd72193a/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
github.com/tribalwarshelp/shared v0.0.0-20200622084436-3a768c8bf574 h1:y2EoH6zRK9Uc0AeswnJRYUUIQYcSLZB5VDFuxPCKxNM=
github.com/tribalwarshelp/shared v0.0.0-20200622084436-3a768c8bf574/go.mod h1:tf+2yTHasV6jAF3V2deZ9slNoCyBzC0fMdTjI7clf6Y=
github.com/vmihailenco/bufpool v0.1.5/go.mod h1:fL9i/PRTuS7AELqAHwSU1Zf1c70xhkhGe/cD5ud9pJk=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=