twhelp
/
shared
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat: add tw package

This commit is contained in:
Dawid Wysokiński 2020-10-31 11:57:53 +01:00
parent eeb4645da4
commit 5d4d8b3077
6 changed files with 512 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -4,6 +4,7 @@ go 1.14
require (
github.com/kr/pretty v0.1.0 // indirect
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.7.0
github.com/stretchr/testify v1.6.1 // indirect
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect

2
go.sum
View File

@ -6,6 +6,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=

436
tw/dataloader/dataloader.go Normal file
View File

@ -0,0 +1,436 @@
package dataloader
import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"github.com/tribalwarshelp/shared/models"
)
const (
EndpointConfig = "/interface.php?func=get_config"
EndpointUnitConfig = "/interface.php?func=get_unit_info"
EndpointBuildingConfig = "/interface.php?func=get_building_info"
EndpointPlayer = "/map/player.txt.gz"
EndpointPlayerNotGzipped = "/map/player.txt"
EndpointTribe = "/map/ally.txt.gz"
EndpointTribeNotGzipped = "/map/ally.txt"
EndpointVillage = "/map/village.txt.gz"
EndpointVillageNotGzipped = "/map/village.txt"
EndpointKillAtt = "/map/kill_att.txt.gz"
EndpointKillAttNotGzipped = "/map/kill_att.txt"
EndpointKillDef = "/map/kill_def.txt.gz"
EndpointKillDefNotGzipped = "/map/kill_def.txt"
EndpointKillSup = "/map/kill_sup.txt.gz"
EndpointKillSupNotGzipped = "/map/kill_sup.txt"
EndpointKillAll = "/map/kill_all.txt.gz"
EndpointKillAllNotGzipped = "/map/kill_all.txt"
EndpointKillAttTribe = "/map/kill_att_tribe.txt.gz"
EndpointKillAttTribeNotGzipped = "/map/kill_att_tribe.txt"
EndpointKillDefTribe = "/map/kill_def_tribe.txt.gz"
EndpointKillDefTribeNotGzipped = "/map/kill_def_tribe.txt"
EndpointKillAllTribe = "/map/kill_all_tribe.txt.gz"
EndpointKillAllTribeNotGzipped = "/map/kill_all_tribe.txt"
EndpointConquer = "/map/conquer.txt.gz"
EndpointConquerNotGzipped = "/map/conquer.txt"
EndpointGetConquer = "/interface.php?func=get_conquer&since=%d"
)
type DataLoader interface {
LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, error)
LoadPlayers() ([]*models.Player, error)
LoadTribes() ([]*models.Tribe, error)
LoadVillages() ([]*models.Village, error)
LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.Ennoblement, error)
GetConfig() (*models.ServerConfig, error)
GetBuildingConfig() (*models.BuildingConfig, error)
GetUnitConfig() (*models.UnitConfig, error)
}
type Config struct {
BaseURL string
Client *http.Client
}
type dataLoader struct {
baseURL string
client *http.Client
}
func New(cfg *Config) DataLoader {
if cfg == nil {
cfg = &Config{}
}
if cfg.Client == nil {
cfg.Client = &http.Client{
Timeout: 5 * time.Second,
}
}
return &dataLoader{
cfg.BaseURL,
cfg.Client,
}
}
type parsedODLine struct {
ID int
Rank int
Score int
}
func (d *dataLoader) parseODLine(line []string) (*parsedODLine, error) {
if len(line) != 3 {
return nil, fmt.Errorf("Invalid line format (should be rank,id,score)")
}
p := &parsedODLine{}
var err error
p.Rank, err = strconv.Atoi(line[0])
if err != nil {
return nil, errors.Wrap(err, "parsedODLine.Rank")
}
p.ID, err = strconv.Atoi(line[1])
if err != nil {
return nil, errors.Wrap(err, "parsedODLine.ID")
}
p.Score, err = strconv.Atoi(line[2])
if err != nil {
return nil, errors.Wrap(err, "parsedODLine.Score")
}
return p, nil
}
func (d *dataLoader) LoadOD(tribe bool) (map[int]*models.OpponentsDefeated, error) {
m := make(map[int]*models.OpponentsDefeated)
urls := []string{
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAll),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAtt),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDef),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillSup),
}
if tribe {
urls = []string{
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAllTribe),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillAttTribe),
fmt.Sprintf("%s%s", d.baseURL, EndpointKillDefTribe),
"",
}
}
for _, url := range urls {
if url == "" {
continue
}
lines, err := d.getCSVData(url, true)
if err != nil {
//fallback to not gzipped file
lines, err = d.getCSVData(strings.ReplaceAll(url, ".gz", ""), false)
if err != nil {
return nil, errors.Wrapf(err, "unable to get data, url %s", url)
}
}
for _, line := range lines {
parsed, err := d.parseODLine(line)
if err != nil {
return nil, errors.Wrapf(err, "unable to parse line, url %s, line %s", url, strings.Join(line, ","))
}
if _, ok := m[parsed.ID]; !ok {
m[parsed.ID] = &models.OpponentsDefeated{}
}
switch url {
case urls[0]:
m[parsed.ID].RankTotal = parsed.Rank
m[parsed.ID].ScoreTotal = parsed.Score
case urls[1]:
m[parsed.ID].RankAtt = parsed.Rank
m[parsed.ID].ScoreAtt = parsed.Score
case urls[2]:
m[parsed.ID].RankDef = parsed.Rank
m[parsed.ID].ScoreDef = parsed.Score
case urls[3]:
m[parsed.ID].RankSup = parsed.Rank
m[parsed.ID].ScoreSup = parsed.Score
}
}
}
return m, nil
}
func (d *dataLoader) parsePlayerLine(line []string) (*models.Player, error) {
if len(line) != 6 {
return nil, fmt.Errorf("Invalid line format (should be id,name,tribeid,villages,points,rank)")
}
var err error
ex := true
player := &models.Player{
Exists: &ex,
}
player.ID, err = strconv.Atoi(line[0])
if err != nil {
return nil, errors.Wrap(err, "player.ID")
}
player.Name, err = url.QueryUnescape(line[1])
if err != nil {
return nil, errors.Wrap(err, "player.Name")
}
player.TribeID, err = strconv.Atoi(line[2])
if err != nil {
return nil, errors.Wrap(err, "player.TribeID")
}
player.TotalVillages, err = strconv.Atoi(line[3])
if err != nil {
return nil, errors.Wrap(err, "player.TotalVillages")
}
player.Points, err = strconv.Atoi(line[4])
if err != nil {
return nil, errors.Wrap(err, "player.Points")
}
player.Rank, err = strconv.Atoi(line[5])
if err != nil {
return nil, errors.Wrap(err, "player.Rank")
}
return player, nil
}
func (d *dataLoader) LoadPlayers() ([]*models.Player, error) {
url := d.baseURL + EndpointPlayer
lines, err := d.getCSVData(url, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointPlayerNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "unable to get data, url %s", url)
}
}
players := []*models.Player{}
for _, line := range lines {
player, err := d.parsePlayerLine(line)
if err != nil {
return nil, errors.Wrapf(err, "unable to parse line, url %s, line %s", url, strings.Join(line, ","))
}
players = append(players, player)
}
return players, nil
}
func (d *dataLoader) parseTribeLine(line []string) (*models.Tribe, error) {
if len(line) != 8 {
return nil, fmt.Errorf("Invalid line format (should be id,name,tag,members,villages,points,allpoints,rank)")
}
var err error
ex := true
tribe := &models.Tribe{
Exists: &ex,
}
tribe.ID, err = strconv.Atoi(line[0])
if err != nil {
return nil, errors.Wrap(err, "tribe.ID")
}
tribe.Name, err = url.QueryUnescape(line[1])
if err != nil {
return nil, errors.Wrap(err, "tribe.Name")
}
tribe.Tag, err = url.QueryUnescape(line[2])
if err != nil {
return nil, errors.Wrap(err, "tribe.Tag")
}
tribe.TotalMembers, err = strconv.Atoi(line[3])
if err != nil {
return nil, errors.Wrap(err, "tribe.TotalMembers")
}
tribe.TotalVillages, err = strconv.Atoi(line[4])
if err != nil {
return nil, errors.Wrap(err, "tribe.TotalVillages")
}
tribe.Points, err = strconv.Atoi(line[5])
if err != nil {
return nil, errors.Wrap(err, "tribe.Points")
}
tribe.AllPoints, err = strconv.Atoi(line[6])
if err != nil {
return nil, errors.Wrap(err, "tribe.AllPoints")
}
tribe.Rank, err = strconv.Atoi(line[7])
if err != nil {
return nil, errors.Wrap(err, "tribe.Rank")
}
return tribe, nil
}
func (d *dataLoader) LoadTribes() ([]*models.Tribe, error) {
url := d.baseURL + EndpointTribe
lines, err := d.getCSVData(url, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointTribeNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "unable to to get data, url %s", url)
}
}
tribes := []*models.Tribe{}
for _, line := range lines {
tribe, err := d.parseTribeLine(line)
if err != nil {
return nil, errors.Wrapf(err, "unable to parse line, url %s, line %s", url, strings.Join(line, ","))
}
tribes = append(tribes, tribe)
}
return tribes, nil
}
func (d *dataLoader) parseVillageLine(line []string) (*models.Village, error) {
if len(line) != 7 {
return nil, fmt.Errorf("Invalid line format (should be id,name,x,y,playerID,points,bonus)")
}
var err error
village := &models.Village{}
village.ID, err = strconv.Atoi(line[0])
if err != nil {
return nil, errors.Wrap(err, "village.ID")
}
village.Name, err = url.QueryUnescape(line[1])
if err != nil {
return nil, errors.Wrap(err, "village.Name")
}
village.X, err = strconv.Atoi(line[2])
if err != nil {
return nil, errors.Wrap(err, "village.X")
}
village.Y, err = strconv.Atoi(line[3])
if err != nil {
return nil, errors.Wrap(err, "village.Y")
}
village.PlayerID, err = strconv.Atoi(line[4])
if err != nil {
return nil, errors.Wrap(err, "village.PlayerID")
}
village.Points, err = strconv.Atoi(line[5])
if err != nil {
return nil, errors.Wrap(err, "village.Points")
}
village.Bonus, err = strconv.Atoi(line[6])
if err != nil {
return nil, errors.Wrap(err, "village.Bonus")
}
return village, nil
}
func (d *dataLoader) LoadVillages() ([]*models.Village, error) {
url := d.baseURL + EndpointVillage
lines, err := d.getCSVData(url, true)
if err != nil {
lines, err = d.getCSVData(d.baseURL+EndpointVillageNotGzipped, false)
if err != nil {
return nil, errors.Wrapf(err, "unable to get data, url %s", url)
}
}
villages := []*models.Village{}
for _, line := range lines {
village, err := d.parseVillageLine(line)
if err != nil {
return nil, errors.Wrapf(err, "unable to parse line, url %s, line %s", url, strings.Join(line, ","))
}
villages = append(villages, village)
}
return villages, nil
}
func (d *dataLoader) parseEnnoblementLine(line []string) (*models.Ennoblement, error) {
if len(line) != 4 {
return nil, fmt.Errorf("Invalid line format (should be village_id,timestamp,new_owner_id,old_owner_id)")
}
var err error
ennoblement := &models.Ennoblement{}
ennoblement.VillageID, err = strconv.Atoi(line[0])
if err != nil {
return nil, errors.Wrap(err, "ennoblement.VillageID")
}
timestamp, err := strconv.Atoi(line[1])
if err != nil {
return nil, errors.Wrap(err, "timestamp")
}
ennoblement.EnnobledAt = time.Unix(int64(timestamp), 0)
ennoblement.NewOwnerID, err = strconv.Atoi(line[2])
if err != nil {
return nil, errors.Wrap(err, "ennoblement.NewOwnerID")
}
ennoblement.OldOwnerID, err = strconv.Atoi(line[3])
if err != nil {
return nil, errors.Wrap(err, "ennoblement.OldOwnerID")
}
return ennoblement, nil
}
type LoadEnnoblementsConfig struct {
EnnobledAtGTE time.Time
}
func (d *dataLoader) LoadEnnoblements(cfg *LoadEnnoblementsConfig) ([]*models.Ennoblement, error) {
if cfg == nil {
cfg = &LoadEnnoblementsConfig{}
}
yesterdayDate := time.Now().Add(-23 * time.Hour)
url := d.baseURL + EndpointConquer
compressed := true
if cfg.EnnobledAtGTE.After(yesterdayDate) || cfg.EnnobledAtGTE.Equal(yesterdayDate) {
compressed = false
url = d.baseURL + fmt.Sprintf(EndpointGetConquer, cfg.EnnobledAtGTE.Unix())
}
lines, err := d.getCSVData(url, compressed)
if err != nil && compressed {
lines, err = d.getCSVData(d.baseURL+EndpointConquerNotGzipped, false)
}
if err != nil {
return nil, errors.Wrapf(err, "unable to get data, url %s", url)
}
ennoblements := []*models.Ennoblement{}
for _, line := range lines {
ennoblement, err := d.parseEnnoblementLine(line)
if err != nil {
return nil, errors.Wrapf(err, "unable to parse line, url %s, line %s", url, strings.Join(line, ","))
}
if ennoblement.EnnobledAt.After(cfg.EnnobledAtGTE) {
ennoblements = append(ennoblements, ennoblement)
}
}
return ennoblements, nil
}
func (d *dataLoader) GetConfig() (*models.ServerConfig, error) {
url := d.baseURL + EndpointConfig
cfg := &models.ServerConfig{}
err := d.getXML(url, cfg)
if err != nil {
return nil, errors.Wrap(err, "getConfig")
}
return cfg, nil
}
func (d *dataLoader) GetBuildingConfig() (*models.BuildingConfig, error) {
url := d.baseURL + EndpointBuildingConfig
cfg := &models.BuildingConfig{}
err := d.getXML(url, cfg)
if err != nil {
return nil, errors.Wrap(err, "getBuildingConfig")
}
return cfg, nil
}
func (d *dataLoader) GetUnitConfig() (*models.UnitConfig, error) {
url := d.baseURL + EndpointUnitConfig
cfg := &models.UnitConfig{}
err := d.getXML(url, cfg)
if err != nil {
return nil, errors.Wrap(err, "getUnitConfig")
}
return cfg, nil
}

51
tw/dataloader/helpers.go Normal file
View File

@ -0,0 +1,51 @@
package dataloader
import (
"compress/gzip"
"encoding/csv"
"encoding/xml"
"io"
"io/ioutil"
)
func newCsvReader(r io.Reader) *csv.Reader {
csvReader := csv.NewReader(r)
csvReader.LazyQuotes = true
csvReader.FieldsPerRecord = -1
return csvReader
}
func uncompressAndReadCsvLines(r io.Reader) ([][]string, error) {
uncompressedStream, err := gzip.NewReader(r)
if err != nil {
return [][]string{}, err
}
defer uncompressedStream.Close()
return newCsvReader(uncompressedStream).ReadAll()
}
func (d *dataLoader) getCSVData(url string, compressed bool) ([][]string, error) {
resp, err := d.client.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if !compressed {
return newCsvReader(resp.Body).ReadAll()
}
return uncompressAndReadCsvLines(resp.Body)
}
func (d *dataLoader) getXML(url string, decode interface{}) error {
resp, err := d.client.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return xml.Unmarshal(bytes, decode)
}

2
utils/language_tag_from_world_name.go → tw/language_tag_from_server_key.go
View File

@ -1,4 +1,4 @@
package utils
package tw
import (
"github.com/tribalwarshelp/shared/models"

21
tw/urls.go Normal file
View File

@ -0,0 +1,21 @@
package tw
import "fmt"
const (
EndpointTribeProfile = "/game.php?screen=info_ally&id=%d"
EndpointPlayerProfile = "/game.php?screen=info_player&id=%d"
EndpointVillageProfile = "/game.php?screen=info_village&id=%d"
)
func BuildVillageURL(server, host string, id int) string {
return fmt.Sprintf("https://%s.%s"+EndpointVillageProfile, server, host, id)
}
func BuildTribeURL(server, host string, id int) string {
return fmt.Sprintf("https://%s.%s"+EndpointTribeProfile, server, host, id)
}
func BuildPlayerURL(server, host string, id int) string {
return fmt.Sprintf("https://%s.%s"+EndpointPlayerProfile, server, host, id)
}