From a505cc9f31799496be4da96659226f86d565ec14 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 14 Feb 2021 19:19:30 +0100 Subject: [PATCH 01/31] add a new package - queue --- cron/queue/queue.go | 54 +++++++++++++++++++++++++++ go.mod | 5 +++ go.sum | 89 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 cron/queue/queue.go diff --git a/cron/queue/queue.go b/cron/queue/queue.go new file mode 100644 index 0000000..24596ff --- /dev/null +++ b/cron/queue/queue.go @@ -0,0 +1,54 @@ +package queue + +import ( + "time" + + "github.com/go-pg/pg/v10" + "github.com/go-redis/redis/v8" + "github.com/vmihailenco/taskq/v3" + "github.com/vmihailenco/taskq/v3/redisq" +) + +type Config struct { + DB *pg.DB + Redis redis.UniversalClient + WorkerLimit int +} + +type queue struct { + db *pg.DB + redis redis.UniversalClient + mainQueue taskq.Queue + ennoblementsQueue taskq.Queue + factory taskq.Factory +} + +func New(cfg *Config) error { + q := &queue{ + db: cfg.DB, + redis: cfg.Redis, + } + + if err := q.init(cfg); err != nil { + return err + } + + return nil +} + +func (q *queue) init(cfg *Config) error { + q.factory = redisq.NewFactory() + q.mainQueue = q.newQueue("main", cfg.WorkerLimit) + q.ennoblementsQueue = q.newQueue("ennoblements", cfg.WorkerLimit*2) + + return nil +} + +func (q *queue) newQueue(name string, limit int) taskq.Queue { + return q.factory.RegisterQueue(&taskq.QueueOptions{ + Name: name, + ReservationTimeout: time.Minute * 2, + Redis: q.redis, + MaxNumWorker: int32(limit), + }) +} diff --git a/go.mod b/go.mod index 1209068..de67db8 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,14 @@ require ( github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 github.com/go-pg/pg/extra/pgdebug v0.2.0 github.com/go-pg/pg/v10 v10.7.3 + github.com/go-redis/redis/v8 v8.5.0 + github.com/go-redis/redis_rate/v9 v9.1.1 // indirect github.com/joho/godotenv v1.3.0 + github.com/klauspost/compress v1.11.7 // indirect github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.7.0 github.com/tribalwarshelp/shared v0.0.0-20210116195235-d8d19c3e6558 + github.com/vmihailenco/msgpack/v5 v5.2.0 // indirect + github.com/vmihailenco/taskq/v3 v3.2.3 ) diff --git a/go.sum b/go.sum index fb5af42..7bd9270 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,33 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83/go.mod h1:+iGkf5HfOVeRVd9K7qQDucIl+/Kt3MyenMa90b/O/c4= +github.com/aws/aws-sdk-go v1.35.28 h1:S2LuRnfC8X05zgZLC8gy/Sb82TGv2Cpytzbzz7tkeHc= +github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/bsm/redislock v0.7.0 h1:RL7aZJhCKkuBjQbnSTKCeedTRifBWxd/ffP+GZ599Mo= +github.com/bsm/redislock v0.7.0/go.mod h1:3Kgu+cXw0JrkZ5pmY/JbcFpixGZ5M9v9G2PGWYqku+k= +github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= +github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-pg/pg/extra/pgdebug v0.2.0 h1:t62UhMiV6KYAxSWojwIJiyX06TdepkzCeIzdeb00184= github.com/go-pg/pg/extra/pgdebug v0.2.0/go.mod h1:KmW//PLshMAQunfInLv9mFIbYXuGplOY9bc6qo3CaY0= github.com/go-pg/pg/v10 v10.6.2/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= @@ -20,6 +35,15 @@ github.com/go-pg/pg/v10 v10.7.3 h1:oL/Hz5MJie/9epmwxlZjfReO+2wlLPOK6BeGb9I+XHk= github.com/go-pg/pg/v10 v10.7.3/go.mod h1:UsDYtA+ihbBNX1OeIvDejxkL4RXzL3wsZYoEv5NUEqM= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/go-redis/redis/v8 v8.1.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo= +github.com/go-redis/redis/v8 v8.3.4/go.mod h1:jszGxBCez8QA1HWSmQxJO9Y82kNibbUmeYhKWrBejTU= +github.com/go-redis/redis/v8 v8.4.0/go.mod h1:A1tbYoHSa1fXwN+//ljcCYYJeLmVrwL9hbQN45Jdy0M= +github.com/go-redis/redis/v8 v8.5.0 h1:L3r1Q3I5WOUdXZGCP6g44EruKh0u3n6co5Hl5xWkdGA= +github.com/go-redis/redis/v8 v8.5.0/go.mod h1:YmEcgBDttjnkbMzDAhDtQxY9yVA7jMN6PCR5HeMvqFE= +github.com/go-redis/redis_rate/v9 v9.1.0 h1:5FOi8UWgxTQ2KlvUg25xPdv2UBxyeRIJJtMZJhBF78U= +github.com/go-redis/redis_rate/v9 v9.1.0/go.mod h1:jjU9YxOSZ3cz0yj1QJVAJiy5ueKmL9o4AySJHcKyTSE= +github.com/go-redis/redis_rate/v9 v9.1.1 h1:7SIrbnhQ7zsTNEgIvprFhJf7/+l3wSpZc2iRVwUmaq8= +github.com/go-redis/redis_rate/v9 v9.1.1/go.mod h1:jjU9YxOSZ3cz0yj1QJVAJiy5ueKmL9o4AySJHcKyTSE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -43,17 +67,34 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= +github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= +github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74 h1:gyfyP8SEIZHs1u2ivTdIbWRtfaKbg5K79d06vnqroJo= +github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -65,12 +106,18 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -78,6 +125,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -99,17 +148,28 @@ github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oT github.com/vmihailenco/msgpack/v5 v5.0.0/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/msgpack/v5 v5.1.4 h1:6K44/cU6dMNGkVTGGuu7ef2NdSRFMhAFGGLfE3cqtHM= github.com/vmihailenco/msgpack/v5 v5.1.4/go.mod h1:C5gboKD0TJPqWDTVTtrQNfRbiBwHZGo8UTqP/9/XvLI= +github.com/vmihailenco/msgpack/v5 v5.2.0 h1:ZhIAtVUP1mme8GIlpiAnmTzjSWMexA/uNF2We85DR0w= +github.com/vmihailenco/msgpack/v5 v5.2.0/go.mod h1:fEM7KuHcnm0GvDCztRpw9hV0PuoO2ciTismP6vjggcM= github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/vmihailenco/taskq/v3 v3.2.3 h1:6VeFiv0o9iXbKULgPFy4JzWzFZvrEjifF/lLMLUNbm0= +github.com/vmihailenco/taskq/v3 v3.2.3/go.mod h1:RQHAGqec6G4RcjZ5sLNqkWWjhLddBQPLxU/x1+RLsCA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= +go.opentelemetry.io/otel v0.16.0 h1:uIWEbdeb4vpKPGITLsRVUS44L5oDbDUCZxn8lkxhmgw= +go.opentelemetry.io/otel v0.16.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -117,9 +177,17 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -127,32 +195,44 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201027133719-8eef5233e2a1/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 h1:/dSxr6gT0FNI1MO5WLJo8mTmItROeOKTkDn+7OwWBos= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -164,8 +244,15 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= @@ -189,6 +276,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -197,6 +285,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= From 55d2a5b0ab4d3d18f24a74fee8344ca5885af90f Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 14:21:51 +0200 Subject: [PATCH 02/31] add github.com/go-redis/redis/v8 and github.com/vmihailenco/taskq/v3 --- cron/queue/queue.go | 6 ++-- go.mod | 4 +++ go.sum | 75 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/cron/queue/queue.go b/cron/queue/queue.go index 24596ff..c2d04a7 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -38,13 +38,13 @@ func New(cfg *Config) error { func (q *queue) init(cfg *Config) error { q.factory = redisq.NewFactory() - q.mainQueue = q.newQueue("main", cfg.WorkerLimit) - q.ennoblementsQueue = q.newQueue("ennoblements", cfg.WorkerLimit*2) + q.mainQueue = q.registerQueue("main", cfg.WorkerLimit) + q.ennoblementsQueue = q.registerQueue("ennoblements", cfg.WorkerLimit*2) return nil } -func (q *queue) newQueue(name string, limit int) taskq.Queue { +func (q *queue) registerQueue(name string, limit int) taskq.Queue { return q.factory.RegisterQueue(&taskq.QueueOptions{ Name: name, ReservationTimeout: time.Minute * 2, diff --git a/go.mod b/go.mod index c5d46ec..d1a3015 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,14 @@ require ( github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 github.com/go-pg/pg/v10 v10.9.1 + github.com/go-redis/redis/v8 v8.8.2 // indirect + github.com/go-redis/redis_rate/v9 v9.1.1 // indirect github.com/joho/godotenv v1.3.0 + github.com/klauspost/compress v1.12.1 // indirect github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 github.com/tribalwarshelp/shared v0.0.0-20210423190057-03d8445d35dc + github.com/vmihailenco/taskq/v3 v3.2.3 // indirect golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect ) diff --git a/go.sum b/go.sum index f5fa45b..6de7c76 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,44 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 h1:7IdSzhdupqm4AC3UDH9b5gdCDE2SlX6qkVC0zwqAuLA= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83/go.mod h1:+iGkf5HfOVeRVd9K7qQDucIl+/Kt3MyenMa90b/O/c4= +github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/bsm/redislock v0.7.0 h1:RL7aZJhCKkuBjQbnSTKCeedTRifBWxd/ffP+GZ599Mo= +github.com/bsm/redislock v0.7.0/go.mod h1:3Kgu+cXw0JrkZ5pmY/JbcFpixGZ5M9v9G2PGWYqku+k= +github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= +github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3/go.mod h1:M5XHQLu90v2JNm/bW2tdsYar+5vhV0gEcBcmDBNAN1Y= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-pg/pg/v10 v10.9.1 h1:kU4t84zWGGaU0Qsu49FbNtToUVrlSTkNOngW8aQmwvk= github.com/go-pg/pg/v10 v10.9.1/go.mod h1:rgmTPgHgl5EN2CNKKoMwC7QT62t8BqsdpEkUQuiZMQs= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/go-redis/redis/v8 v8.1.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo= +github.com/go-redis/redis/v8 v8.3.4/go.mod h1:jszGxBCez8QA1HWSmQxJO9Y82kNibbUmeYhKWrBejTU= +github.com/go-redis/redis/v8 v8.4.0/go.mod h1:A1tbYoHSa1fXwN+//ljcCYYJeLmVrwL9hbQN45Jdy0M= +github.com/go-redis/redis/v8 v8.8.2 h1:O/NcHqobw7SEptA0yA6up6spZVFtwE06SXM8rgLtsP8= +github.com/go-redis/redis/v8 v8.8.2/go.mod h1:F7resOH5Kdug49Otu24RjHWwgK7u9AmtqWMnCV1iP5Y= +github.com/go-redis/redis_rate/v9 v9.1.0/go.mod h1:jjU9YxOSZ3cz0yj1QJVAJiy5ueKmL9o4AySJHcKyTSE= +github.com/go-redis/redis_rate/v9 v9.1.1 h1:7SIrbnhQ7zsTNEgIvprFhJf7/+l3wSpZc2iRVwUmaq8= +github.com/go-redis/redis_rate/v9 v9.1.1/go.mod h1:jjU9YxOSZ3cz0yj1QJVAJiy5ueKmL9o4AySJHcKyTSE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -30,18 +51,33 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= +github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= +github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -52,12 +88,16 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -65,6 +105,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -80,6 +121,7 @@ github.com/tribalwarshelp/shared v0.0.0-20210423190057-03d8445d35dc h1:giWPsD/6W github.com/tribalwarshelp/shared v0.0.0-20210423190057-03d8445d35dc/go.mod h1:CDQvesBYmSyGDl5X37xfa+ub55ZbikrHDuIZ4AcfM8I= github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.0.0/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/msgpack/v5 v5.3.0/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/msgpack/v5 v5.3.1 h1:0i85a4dsZh8mC//wmyyTEzidDLPQfQAxZIOLtafGbFY= github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= @@ -87,6 +129,12 @@ github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vb github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/vmihailenco/taskq/v3 v3.2.3 h1:6VeFiv0o9iXbKULgPFy4JzWzFZvrEjifF/lLMLUNbm0= +github.com/vmihailenco/taskq/v3 v3.2.3/go.mod h1:RQHAGqec6G4RcjZ5sLNqkWWjhLddBQPLxU/x1+RLsCA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= +go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= +go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= go.opentelemetry.io/otel v0.19.0/go.mod h1:j9bF567N9EfomkSidSfmMwIwIBuP37AMAIzVW85OxSg= go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= @@ -101,40 +149,59 @@ go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52l go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201027133719-8eef5233e2a1/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw= @@ -149,8 +216,14 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -170,6 +243,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -177,6 +251,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= From 4e051b0a5fa7d9b41f4628e7c4c431491b5e12ca Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 14:51:41 +0200 Subject: [PATCH 03/31] move queue names to constants, add the Queue interface, implement the Queue interface --- cron/queue/queue.go | 82 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/cron/queue/queue.go b/cron/queue/queue.go index c2d04a7..0b90fd6 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -1,54 +1,114 @@ package queue import ( + "context" + "github.com/pkg/errors" "time" - "github.com/go-pg/pg/v10" "github.com/go-redis/redis/v8" "github.com/vmihailenco/taskq/v3" "github.com/vmihailenco/taskq/v3/redisq" ) +type QueueName string + +const ( + MainQueue QueueName = "main" + EnnoblementsQueue QueueName = "ennoblements" +) + type Config struct { - DB *pg.DB Redis redis.UniversalClient WorkerLimit int } +type Queue interface { + Start(ctx context.Context) error + Close() error + Add(name QueueName, msg *taskq.Message) error +} + type queue struct { - db *pg.DB redis redis.UniversalClient mainQueue taskq.Queue ennoblementsQueue taskq.Queue factory taskq.Factory } -func New(cfg *Config) error { +func New(cfg *Config) (Queue, error) { + if err := validateConfig(cfg); err != nil { + return nil, err + } + q := &queue{ - db: cfg.DB, redis: cfg.Redis, } if err := q.init(cfg); err != nil { - return err + return nil, err } - return nil + return q, nil } func (q *queue) init(cfg *Config) error { q.factory = redisq.NewFactory() - q.mainQueue = q.registerQueue("main", cfg.WorkerLimit) - q.ennoblementsQueue = q.registerQueue("ennoblements", cfg.WorkerLimit*2) + q.mainQueue = q.registerQueue(MainQueue, cfg.WorkerLimit) + q.ennoblementsQueue = q.registerQueue(EnnoblementsQueue, cfg.WorkerLimit*2) return nil } -func (q *queue) registerQueue(name string, limit int) taskq.Queue { +func (q *queue) registerQueue(name QueueName, limit int) taskq.Queue { return q.factory.RegisterQueue(&taskq.QueueOptions{ - Name: name, + Name: string(name), ReservationTimeout: time.Minute * 2, Redis: q.redis, MaxNumWorker: int32(limit), }) } + +func (q *queue) Start(ctx context.Context) error { + if err := q.factory.StartConsumers(ctx); err != nil { + return errors.Wrap(err, "Couldn't start the queue") + } + return nil +} + +func (q *queue) Close() error { + if err := q.factory.Close(); err != nil { + return errors.Wrap(err, "Couldn't close the queue") + } + return nil +} + +func (q *queue) Add(name QueueName, msg *taskq.Message) error { + queue := q.getQueueByName(name) + if queue == nil { + return errors.Errorf("Couldn't add the message to the queue: unknown queue name '%s'", name) + } + if err := queue.Add(msg); err != nil { + return errors.Wrap(err, "Couldn't add the message to the queue") + } + return nil +} + +func (q *queue) getQueueByName(name QueueName) taskq.Queue { + switch name { + case MainQueue: + return q.mainQueue + case EnnoblementsQueue: + return q.ennoblementsQueue + } + return nil +} + +func validateConfig(cfg *Config) error { + if cfg == nil { + return errors.New("Config hasn't been provided") + } + if cfg.Redis == nil { + return errors.New("cfg.Redis is a required field") + } + return nil +} From dea3d4616a1f8e435ab2b0350715000b0d5d85dd Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 17:02:58 +0200 Subject: [PATCH 04/31] huge refactor --- cron/cron.go | 26 +-- cron/handler.go | 259 +-------------------------- cron/tasks/task.go | 12 ++ cron/tasks/task_load_servers.go | 104 +++++++++++ cron/tasks/tasks.go | 59 ++++++ db/db.go | 189 ++++++++++++++++++++ db/sql_statements.go | 306 ++++++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 10 +- main.go | 98 +++++----- utils/env/env.go | 30 ++++ 11 files changed, 787 insertions(+), 310 deletions(-) create mode 100644 cron/tasks/task.go create mode 100644 cron/tasks/task_load_servers.go create mode 100644 cron/tasks/tasks.go create mode 100644 db/db.go create mode 100644 db/sql_statements.go create mode 100644 utils/env/env.go diff --git a/cron/cron.go b/cron/cron.go index d9f8023..38b5788 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -9,6 +9,8 @@ import ( "github.com/go-pg/pg/v10" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" + + "github.com/tribalwarshelp/cron/cron/queue" ) var log = logrus.WithField("package", "cron") @@ -17,19 +19,23 @@ type Config struct { DB *pg.DB MaxConcurrentWorkers int RunOnStartup bool + Queue queue.Queue } func Attach(c *cron.Cron, cfg Config) error { if cfg.DB == nil { return fmt.Errorf("cfg.DB cannot be nil, expected *pg.DB") } + if cfg.Queue == nil { + return fmt.Errorf("cfg.Queue cannot be nil, expected queue.Queue") + } h := &handler{db: cfg.DB, maxConcurrentWorkers: cfg.MaxConcurrentWorkers} if err := h.init(); err != nil { return err } - versions := []*models.Version{} + var versions []*models.Version if err := cfg.DB.Model(&versions).DistinctOn("timezone").Select(); err != nil { return err } @@ -37,8 +43,8 @@ func Attach(c *cron.Cron, cfg Config) error { updateServerData := utils.TrackExecutionTime(log, h.updateServerData, "updateServerData") vacuumDatabase := utils.TrackExecutionTime(log, h.vacuumDatabase, "vacuumDatabase") updateServerEnnoblements := utils.TrackExecutionTime(log, h.updateServerEnnoblements, "updateServerEnnoblements") - updateHistoryFuncs := []func(){} - updateStatsFuncs := []func(){} + var updateHistoryFuncs []func() + var updateStatsFuncs []func() for _, version := range versions { updateHistory := utils.TrackExecutionTime(log, createFnWithTimezone(version.Timezone, h.updateHistory), @@ -69,13 +75,13 @@ func Attach(c *cron.Cron, cfg Config) error { if cfg.RunOnStartup { go func() { updateServerData() - vacuumDatabase() - for _, fn := range updateHistoryFuncs { - go fn() - } - for _, fn := range updateStatsFuncs { - go fn() - } + //vacuumDatabase() + //for _, fn := range updateHistoryFuncs { + // go fn() + //} + //for _, fn := range updateStatsFuncs { + // go fn() + //} }() } diff --git a/cron/handler.go b/cron/handler.go index 5376b66..d264201 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -1,32 +1,27 @@ package cron import ( + "context" "fmt" - "io/ioutil" - "net/http" "runtime" "sync" "time" - "github.com/sirupsen/logrus" "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/shared/tw/dataloader" - phpserialize "github.com/Kichiyaki/go-php-serialize" - "github.com/go-pg/pg/v10" - "github.com/go-pg/pg/v10/orm" "github.com/pkg/errors" -) -const ( - endpointGetServers = "/backend/get_servers.php" + "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/cron/tasks" ) type handler struct { db *pg.DB maxConcurrentWorkers int pool *pool + queue queue.Queue } func (h *handler) init() error { @@ -38,255 +33,11 @@ func (h *handler) init() error { h.pool = newPool(h.maxConcurrentWorkers) } - tx, err := h.db.Begin() - if err != nil { - return err - } - defer tx.Close() - - dbModels := []interface{}{ - (*models.SpecialServer)(nil), - (*models.Server)(nil), - (*models.Version)(nil), - (*models.PlayerToServer)(nil), - (*models.PlayerNameChange)(nil), - } - - for _, model := range dbModels { - err := tx.Model(model).CreateTable(&orm.CreateTableOptions{ - IfNotExists: true, - }) - if err != nil { - return err - } - } - - type statementWithParams struct { - statement string - params []interface{} - } - - for _, s := range []statementWithParams{ - { - statement: pgDefaultValues, - }, - { - statement: allVersionsPGInsertStatements, - }, - { - statement: allSpecialServersPGInsertStatements, - }, - { - statement: pgDropSchemaFunctions, - params: []interface{}{pg.Safe("public")}, - }, - { - statement: pgFunctions, - }, - } { - if _, err := tx.Exec(s.statement, s.params...); err != nil { - return err - } - } - - if err := tx.Commit(); err != nil { - return err - } - - servers := []*models.Server{} - if err := h.db.Model(&servers).Select(); err != nil { - return err - } - - for _, server := range servers { - if err := h.createSchema(server, true); err != nil { - return err - } - } - return nil } -func (h *handler) createSchema(server *models.Server, init bool) error { - if !init { - exists, err := h.db.Model().Table("information_schema.schemata").Where("schema_name = ?", server.Key).Exists() - if err != nil { - return err - } - - if exists { - return nil - } - } - - tx, err := h.db.WithParam("SERVER", pg.Safe(server.Key)).Begin() - if err != nil { - return err - } - defer tx.Close() - - if _, err := tx.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", server.Key)); err != nil { - return err - } - - dbModels := []interface{}{ - (*models.Tribe)(nil), - (*models.Player)(nil), - (*models.Village)(nil), - (*models.Ennoblement)(nil), - (*models.ServerStats)(nil), - (*models.TribeHistory)(nil), - (*models.PlayerHistory)(nil), - (*models.TribeChange)(nil), - (*models.DailyPlayerStats)(nil), - (*models.DailyTribeStats)(nil), - } - - for _, model := range dbModels { - err := tx.Model(model).CreateTable(&orm.CreateTableOptions{ - IfNotExists: true, - }) - if err != nil { - return err - } - } - - statements := []string{ - serverPGFunctions, - serverPGTriggers, - serverPGDefaultValues, - } - if init { - statements = append([]string{pgDropSchemaFunctions}, statements...) - } - for _, statement := range statements { - if _, err := tx.Exec(statement, pg.Safe(server.Key), server.VersionCode); err != nil { - return err - } - } - - return tx.Commit() -} - -func (h *handler) getServers() ([]*models.Server, map[string]string, error) { - versions := []*models.Version{} - if err := h.db.Model(&versions).Relation("SpecialServers").Order("code ASC").Select(); err != nil { - return nil, nil, errors.Wrap(err, "getServers") - } - - serverKeys := []string{} - servers := []*models.Server{} - urls := make(map[string]string) - loadedVersions := []models.VersionCode{} - for _, version := range versions { - log := log.WithField("host", version.Host) - log.Infof("Loading servers from %s", version.Host) - resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers)) - if err != nil { - log.Errorln(errors.Wrapf(err, "fetching servers from %s", version.Host)) - continue - } - defer resp.Body.Close() - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Errorln(errors.Wrapf(err, "reading response body from %s", version.Host)) - continue - } - body, err := phpserialize.Decode(string(bodyBytes)) - if err != nil { - log.Errorln(errors.Wrapf(err, "serializing body from %s into go value", version.Host)) - continue - } - for serverKey, url := range body.(map[interface{}]interface{}) { - serverKeyStr := serverKey.(string) - if version.SpecialServers.Contains(serverKeyStr) { - continue - } - server := &models.Server{ - Key: serverKeyStr, - Status: models.ServerStatusOpen, - VersionCode: version.Code, - Version: version, - } - if err := h.createSchema(server, false); err != nil { - log.WithField("serverKey", serverKey).Errorln(errors.Wrapf(err, "cannot create schema for %s", serverKey)) - continue - } - serverKeys = append(serverKeys, serverKeyStr) - urls[serverKeyStr] = url.(string) - servers = append(servers, server) - } - loadedVersions = append(loadedVersions, version.Code) - } - - if len(servers) > 0 { - if _, err := h.db.Model(&servers). - OnConflict("(key) DO UPDATE"). - Set("status = ?", models.ServerStatusOpen). - Set("version_code = EXCLUDED.version_code"). - Returning("*"). - Insert(); err != nil { - return nil, nil, err - } - } - - if _, err := h.db.Model(&models.Server{}). - Set("status = ?", models.ServerStatusClosed). - Where("key NOT IN (?) AND version_code IN (?)", pg.In(serverKeys), pg.In(loadedVersions)). - Update(); err != nil { - return nil, nil, err - } - - return servers, urls, nil -} - func (h *handler) updateServerData() { - servers, urls, err := h.getServers() - if err != nil { - log.Errorln("updateServerData:", err.Error()) - return - } - log. - WithField("numberOfServers", len(servers)). - Info("updateServerData: servers loaded") - - var wg sync.WaitGroup - - for _, server := range servers { - log := log.WithField("serverKey", server.Key) - url, ok := urls[server.Key] - if !ok { - log.Warnf("No one URL associated with key: %s, skipping...", server.Key) - continue - } - h.pool.waitForWorker() - wg.Add(1) - sh := &updateServerDataWorker{ - db: h.db.WithParam("SERVER", pg.Safe(server.Key)), - server: server, - dataloader: dataloader.New(&dataloader.Config{ - BaseURL: url, - Client: &http.Client{ - Timeout: 10 * time.Second, - }, - }), - } - go func(worker *updateServerDataWorker, server *models.Server, url string, log *logrus.Entry) { - defer func() { - h.pool.releaseWorker() - wg.Done() - }() - log.Infof("updateServerData: %s: updating data", server.Key) - err := sh.update() - if err != nil { - log.Errorln("updateServerData:", errors.Wrap(err, server.Key)) - return - } - log.Infof("updateServerData: %s: data updated", server.Key) - }(sh, server, url, log) - } - - wg.Wait() + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersions).WithArgs(context.Background())) } func (h *handler) updateServerEnnoblements() { diff --git a/cron/tasks/task.go b/cron/tasks/task.go new file mode 100644 index 0000000..03e6d84 --- /dev/null +++ b/cron/tasks/task.go @@ -0,0 +1,12 @@ +package tasks + +import ( + "github.com/go-pg/pg/v10" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type task struct { + db *pg.DB + queue queue.Queue +} diff --git a/cron/tasks/task_load_servers.go b/cron/tasks/task_load_servers.go new file mode 100644 index 0000000..8190f7f --- /dev/null +++ b/cron/tasks/task_load_servers.go @@ -0,0 +1,104 @@ +package tasks + +import ( + "fmt" + phpserialize "github.com/Kichiyaki/go-php-serialize" + "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/tribalwarshelp/shared/models" + "io/ioutil" + "net/http" + + "github.com/tribalwarshelp/cron/db" +) + +const ( + endpointGetServers = "/backend/get_servers.php" +) + +type taskLoadServers struct { + *task +} + +func (t *taskLoadServers) execute(version *models.Version) error { + if err := t.validatePayload(version); err != nil { + return nil + } + entry := log.WithField("host", version.Host) + entry.Infof("%s: Loading servers", version.Host) + data, err := t.getServers(version) + if err != nil { + log.Errorln(err) + return err + } + + var serverKeys []string + var servers []*models.Server + for serverKey := range data { + if version.SpecialServers.Contains(serverKey) { + continue + } + server := &models.Server{ + Key: serverKey, + Status: models.ServerStatusOpen, + VersionCode: version.Code, + Version: version, + } + if err := db.CreateSchema(t.db, server); err != nil { + logrus.Warn(errors.Wrapf(err, "%s: couldn't create the schema", server.Key)) + continue + } + servers = append(servers, server) + serverKeys = append(serverKeys, serverKey) + } + + if len(servers) > 0 { + if _, err := t.db.Model(&servers). + OnConflict("(key) DO UPDATE"). + Set("status = ?", models.ServerStatusOpen). + Set("version_code = EXCLUDED.version_code"). + Returning("*"). + Insert(); err != nil { + err = errors.Wrap(err, "couldn't insert/update servers") + logrus.Fatal(err) + return err + } + } + + if _, err := t.db.Model(&models.Server{}). + Set("status = ?", models.ServerStatusClosed). + Where("key NOT IN (?) AND version_code =", pg.In(serverKeys), version.Code). + Update(); err != nil { + err = errors.Wrap(err, "couldn't update server statuses") + logrus.Fatal(err) + return err + } + + entry.Infof("%s: Servers have been loaded", version.Host) + return nil +} + +func (t *taskLoadServers) validatePayload(version *models.Version) error { + if version == nil { + return errors.Errorf("taskLoadServers.validatePayload: Expected *models.Version, got nil") + } + return nil +} + +func (t *taskLoadServers) getServers(version *models.Version) (map[string]string, error) { + resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers)) + if err != nil { + return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't load servers", version.Host) + } + defer resp.Body.Close() + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't read the body", version.Host) + } + body, err := phpserialize.Decode(string(bodyBytes)) + if err != nil { + return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't decode the body into the go value", version.Host) + } + return body.(map[string]string), nil +} diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go new file mode 100644 index 0000000..10bda4e --- /dev/null +++ b/cron/tasks/tasks.go @@ -0,0 +1,59 @@ +package tasks + +import ( + "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vmihailenco/taskq/v3" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +const ( + TaskNameLoadVersions = "loadVersions" + TaskNameLoadServers = "loadServers" + TaskNameUpdateServerData = "updateServerData" + retryLimitLoadServers = 3 +) + +var log = logrus.WithField("package", "tasks") + +type Config struct { + DB *pg.DB + Queue queue.Queue +} + +func RegisterTasks(cfg *Config) error { + if err := validateConfig(cfg); err != nil { + return errors.Wrap(err, "RegisterTasks") + } + + t := &task{ + db: cfg.DB, + queue: cfg.Queue, + } + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskNameLoadServers, + RetryLimit: retryLimitLoadServers, + Handler: (&taskLoadServers{t}).execute, + }) + + return nil +} + +func Get(taskName string) *taskq.Task { + return taskq.Tasks.Get(taskName) +} + +func validateConfig(cfg *Config) error { + if cfg == nil { + return errors.New("Config hasn't been provided") + } + if cfg.DB == nil { + return errors.New("cfg.DB is required") + } + if cfg.Queue == nil { + return errors.New("cfg.Queue is required") + } + return nil +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..d946e48 --- /dev/null +++ b/db/db.go @@ -0,0 +1,189 @@ +package db + +import ( + "fmt" + gopglogrusquerylogger "github.com/Kichiyaki/go-pg-logrus-query-logger/v10" + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/tribalwarshelp/shared/models" + + envutils "github.com/tribalwarshelp/cron/utils/env" +) + +var log = logrus.WithField("package", "db") + +type Config struct { + LogQueries bool +} + +func New(cfg *Config) (*pg.DB, error) { + db := pg.Connect(prepareOptions()) + + if cfg != nil && cfg.LogQueries { + db.AddQueryHook(gopglogrusquerylogger.QueryLogger{ + Entry: log, + MaxQueryLength: 5000, + }) + } + + if err := prepareDB(db); err != nil { + return nil, errors.Wrap(err, "New") + } + + return db, nil +} + +func prepareOptions() *pg.Options { + return &pg.Options{ + User: envutils.GetenvString("DB_USER"), + Password: envutils.GetenvString("DB_PASSWORD"), + Database: envutils.GetenvString("DB_NAME"), + Addr: envutils.GetenvString("DB_HOST") + ":" + envutils.GetenvString("DB_PORT"), + PoolSize: envutils.GetenvInt("DB_POOL_SIZE"), + } +} + +func prepareDB(db *pg.DB) error { + tx, err := db.Begin() + if err != nil { + return errors.Wrap(err, "Couldn't start a transaction") + } + defer tx.Close() + + dbModels := []interface{}{ + (*models.SpecialServer)(nil), + (*models.Server)(nil), + (*models.Version)(nil), + (*models.PlayerToServer)(nil), + (*models.PlayerNameChange)(nil), + } + + for _, model := range dbModels { + err := tx.Model(model).CreateTable(&orm.CreateTableOptions{ + IfNotExists: true, + }) + if err != nil { + return errors.Wrap(err, "Couldn't create the table") + } + } + + type statementWithParams struct { + statement string + params []interface{} + } + + for _, s := range []statementWithParams{ + { + statement: pgDefaultValues, + }, + { + statement: allVersionsPGInsertStatements, + }, + { + statement: allSpecialServersPGInsertStatements, + }, + { + statement: pgDropSchemaFunctions, + params: []interface{}{pg.Safe("public")}, + }, + { + statement: pgFunctions, + }, + } { + if _, err := tx.Exec(s.statement, s.params...); err != nil { + return errors.Wrap(err, "Couldn't initialize the db") + } + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "Couldn't commit changes") + } + + var servers []*models.Server + if err := db.Model(&servers).Select(); err != nil { + return errors.Wrap(err, "Couldn't load servers") + } + + for _, server := range servers { + if err := createSchema(db, server, true); err != nil { + return err + } + } + + return nil +} + +func CreateSchema(db *pg.DB, server *models.Server) error { + return createSchema(db, server, false) +} + +func SchemaExists(db *pg.DB, schemaName string) bool { + exists, err := db. + Model(). + Table("information_schema.schemata"). + Where("schema_name = ?", schemaName). + Exists() + if err != nil { + return false + } + return exists +} + +func createSchema(db *pg.DB, server *models.Server, init bool) error { + if !init && SchemaExists(db, server.Key) { + return nil + } + + tx, err := db.WithParam("SERVER", pg.Safe(server.Key)).Begin() + if err != nil { + return errors.Wrap(err, "CreateSchema: couldn't start a transaction") + } + defer tx.Close() + + if _, err := tx.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", server.Key)); err != nil { + return errors.Wrap(err, "CreateSchema: couldn't create the schema") + } + + dbModels := []interface{}{ + (*models.Tribe)(nil), + (*models.Player)(nil), + (*models.Village)(nil), + (*models.Ennoblement)(nil), + (*models.ServerStats)(nil), + (*models.TribeHistory)(nil), + (*models.PlayerHistory)(nil), + (*models.TribeChange)(nil), + (*models.DailyPlayerStats)(nil), + (*models.DailyTribeStats)(nil), + } + + for _, model := range dbModels { + err := tx.Model(model).CreateTable(&orm.CreateTableOptions{ + IfNotExists: true, + }) + if err != nil { + return err + } + } + + statements := []string{ + serverPGFunctions, + serverPGTriggers, + serverPGDefaultValues, + } + if init { + statements = append([]string{pgDropSchemaFunctions}, statements...) + } + for _, statement := range statements { + if _, err := tx.Exec(statement, pg.Safe(server.Key), server.VersionCode); err != nil { + return errors.Wrap(err, "CreateSchema: couldn't initialize the schema") + } + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "CreateSchema: couldn't commit changes") + } + return nil +} diff --git a/db/sql_statements.go b/db/sql_statements.go new file mode 100644 index 0000000..583d57e --- /dev/null +++ b/db/sql_statements.go @@ -0,0 +1,306 @@ +package db + +const ( + allSpecialServersPGInsertStatements = ` + INSERT INTO public.special_servers (version_code, key) VALUES ('pl', 'pls1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('en', 'ens1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('uk', 'uks1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('uk', 'master') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('it', 'its1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('hu', 'hus1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('fr', 'frs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('us', 'uss1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('nl', 'nls1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('es', 'ess1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('ro', 'ros1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('gr', 'grs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('br', 'brs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('tr', 'trs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('cs', 'css1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('de', 'des1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('ru', 'rus1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('ch', 'chs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('pt', 'pts1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + INSERT INTO public.special_servers (version_code, key) VALUES ('sk', 'sks1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; + ` + allVersionsPGInsertStatements = ` + INSERT INTO public.versions (code, name, host, timezone) VALUES ('pl', 'Polska', 'plemiona.pl', 'Europe/Warsaw') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('uk', 'United Kingdom', 'tribalwars.co.uk', 'Europe/London') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('hu', 'Hungary', 'klanhaboru.hu', 'Europe/Budapest') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('it', 'Italy', 'tribals.it', 'Europe/Rome') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('fr', 'France', 'guerretribale.fr', 'Europe/Paris') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('us', 'United States', 'tribalwars.us', 'America/New_York') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('nl', 'The Netherlands', 'tribalwars.nl', 'Europe/Amsterdam') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('es', 'Spain', 'guerrastribales.es', 'Europe/Madrid') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('ro', 'Romania', 'triburile.ro', 'Europe/Bucharest') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('gr', 'Greece', 'fyletikesmaxes.gr', 'Europe/Athens') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('br', 'Brazil', 'tribalwars.com.br', 'America/Sao_Paulo') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('tr', 'Turkey', 'klanlar.org', 'Europe/Istanbul') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('cs', 'Czech Republic', 'divokekmeny.cz', 'Europe/Prague') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('ru', 'Russia', 'voyna-plemyon.ru', 'Europe/Moscow') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('ch', 'Switerzland', 'staemme.ch', 'Europe/Zurich') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('pt', 'Portugal', 'tribalwars.com.pt', 'Europe/Lisbon') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('en', 'International', 'tribalwars.net', 'Europe/London') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('de', 'Germany', 'die-staemme.de', 'Europe/Berlin') ON CONFLICT (code) DO NOTHING; + INSERT INTO public.versions (code, name, host, timezone) VALUES ('sk', 'Slovakia', 'divoke-kmene.sk', 'Europe/Bratislava') ON CONFLICT (code) DO NOTHING; + ` + + pgDropSchemaFunctions = ` + DO + $do$ + DECLARE + funcTableRecord RECORD; + BEGIN + FOR funcTableRecord IN SELECT routine_schema, routine_name from information_schema.routines where routine_type = 'FUNCTION' AND specific_schema = '?0' LOOP + EXECUTE 'DROP FUNCTION IF EXISTS ' || funcTableRecord.routine_schema || '.' || funcTableRecord.routine_name || ' CASCADE;'; + END LOOP; + END + $do$; + ` + + pgFunctions = ` + CREATE OR REPLACE FUNCTION check_daily_growth() + RETURNS trigger AS + $BODY$ + BEGIN + IF NEW.exists = false THEN + NEW.daily_growth = 0; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION check_existence() + RETURNS trigger AS + $BODY$ + BEGIN + IF NEW.exists = false AND OLD.exists = true THEN + NEW.deleted_at = now(); + END IF; + IF NEW.exists = true THEN + NEW.deleted_at = null; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION check_dominance() + RETURNS trigger AS + $BODY$ + BEGIN + IF NEW.exists = false THEN + NEW.dominance = 0; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + + CREATE OR REPLACE FUNCTION update_most_points_most_villages_best_rank_last_activity() + RETURNS trigger AS + $BODY$ + BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.most_points IS null OR NEW.points > NEW.most_points THEN + NEW.most_points = NEW.points; + NEW.most_points_at = now(); + END IF; + IF NEW.most_villages IS null OR NEW.total_villages > NEW.most_villages THEN + NEW.most_villages = NEW.total_villages; + NEW.most_villages_at = now(); + END IF; + IF NEW.best_rank IS null OR NEW.rank < NEW.best_rank OR NEW.best_rank = 0 THEN + NEW.best_rank = NEW.rank; + NEW.best_rank_at = now(); + END IF; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF NEW.most_points IS null OR NEW.points > OLD.most_points THEN + NEW.most_points = NEW.points; + NEW.most_points_at = now(); + END IF; + IF NEW.most_villages IS null OR NEW.total_villages > OLD.most_villages THEN + NEW.most_villages = NEW.total_villages; + NEW.most_villages_at = now(); + END IF; + IF NEW.best_rank IS null OR NEW.rank < OLD.best_rank OR OLD.best_rank = 0 THEN + NEW.best_rank = NEW.rank; + NEW.best_rank_at = now(); + END IF; + if TG_TABLE_NAME = 'players' THEN + IF NEW.points > OLD.points OR NEW.score_att > OLD.score_att THEN + NEW.last_activity_at = now(); + END IF; + END IF; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + ` + + serverPGFunctions = ` + CREATE OR REPLACE FUNCTION ?0.log_tribe_change() + RETURNS trigger AS + $BODY$ + BEGIN + IF TG_OP = 'INSERT' THEN + IF NEW.tribe_id <> 0 THEN + INSERT INTO ?0.tribe_changes(player_id,old_tribe_id,new_tribe_id,created_at) + VALUES(NEW.id,0,NEW.tribe_id,now()); + END IF; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF NEW.tribe_id <> OLD.tribe_id THEN + INSERT INTO ?0.tribe_changes(player_id,old_tribe_id,new_tribe_id,created_at) + VALUES(OLD.id,OLD.tribe_id,NEW.tribe_id,now()); + END IF; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql VOLATILE; + + CREATE OR REPLACE FUNCTION ?0.log_player_name_change() + RETURNS trigger AS + $BODY$ + BEGIN + IF NEW.name <> OLD.name AND old.exists = true THEN + INSERT INTO player_name_changes(version_code,player_id,old_name,new_name,change_date) + VALUES(?1,NEW.id,OLD.name,NEW.name,CURRENT_DATE) + ON CONFLICT DO NOTHING; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql VOLATILE; + + CREATE OR REPLACE FUNCTION ?0.get_old_and_new_owner_tribe_id() + RETURNS trigger AS + $BODY$ + BEGIN + IF NEW.old_owner_id <> 0 THEN + SELECT tribe_id INTO NEW.old_owner_tribe_id + FROM ?0.players + WHERE id = NEW.old_owner_id; + END IF; + IF NEW.old_owner_tribe_id IS NULL THEN + NEW.old_owner_tribe_id = 0; + END IF; + IF NEW.new_owner_id <> 0 THEN + SELECT tribe_id INTO NEW.new_owner_tribe_id + FROM ?0.players + WHERE id = NEW.new_owner_id; + END IF; + IF NEW.new_owner_tribe_id IS NULL THEN + NEW.new_owner_tribe_id = 0; + END IF; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql VOLATILE; + + CREATE OR REPLACE FUNCTION ?0.insert_into_player_to_servers() + RETURNS trigger AS + $BODY$ + BEGIN + INSERT INTO player_to_servers(server_key,player_id) + VALUES('?0', NEW.id) + ON CONFLICT DO NOTHING; + + RETURN NEW; + END; + $BODY$ + LANGUAGE plpgsql; + ` + serverPGTriggers = ` + CREATE TRIGGER ?0_log_tribe_change_on_insert + AFTER INSERT + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE ?0.log_tribe_change(); + + CREATE TRIGGER ?0_log_tribe_change_on_update + AFTER UPDATE + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE ?0.log_tribe_change(); + + CREATE TRIGGER ?0_name_change + AFTER UPDATE + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE ?0.log_player_name_change(); + + CREATE TRIGGER ?0_check_daily_growth + BEFORE UPDATE + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE check_daily_growth(); + + CREATE TRIGGER ?0_check_player_existence + BEFORE UPDATE + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE check_existence(); + + CREATE TRIGGER ?0_check_tribe_existence + BEFORE UPDATE + ON ?0.tribes + FOR EACH ROW + EXECUTE PROCEDURE check_existence(); + + CREATE TRIGGER ?0_check_dominance + BEFORE UPDATE + ON ?0.tribes + FOR EACH ROW + EXECUTE PROCEDURE check_dominance(); + + CREATE TRIGGER ?0_update_ennoblement_old_and_new_owner_tribe_id + BEFORE INSERT + ON ?0.ennoblements + FOR EACH ROW + EXECUTE PROCEDURE ?0.get_old_and_new_owner_tribe_id(); + + CREATE TRIGGER ?0_insert_into_player_to_servers + AFTER INSERT + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE ?0.insert_into_player_to_servers(); + + CREATE TRIGGER ?0_update_most_points_most_villages_best_rank_last_activity + BEFORE INSERT OR UPDATE + ON ?0.players + FOR EACH ROW + EXECUTE PROCEDURE update_most_points_most_villages_best_rank_last_activity(); + + CREATE TRIGGER ?0_update_most_points_most_villages_best_rank_last_activity + BEFORE INSERT OR UPDATE + ON ?0.tribes + FOR EACH ROW + EXECUTE PROCEDURE update_most_points_most_villages_best_rank_last_activity(); + ` + + serverPGDefaultValues = ` + ALTER TABLE ?0.daily_player_stats ALTER COLUMN create_date set default CURRENT_DATE; + ALTER TABLE ?0.daily_tribe_stats ALTER COLUMN create_date set default CURRENT_DATE; + ALTER TABLE ?0.player_history ALTER COLUMN create_date set default CURRENT_DATE; + ALTER TABLE ?0.tribe_history ALTER COLUMN create_date set default CURRENT_DATE; + ALTER TABLE ?0.stats ALTER COLUMN create_date set default CURRENT_DATE; + ` + + pgDefaultValues = ` + ALTER TABLE player_name_changes ALTER COLUMN change_date set default CURRENT_DATE; + ` +) diff --git a/go.mod b/go.mod index d1a3015..ef15513 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 github.com/go-pg/pg/v10 v10.9.1 - github.com/go-redis/redis/v8 v8.8.2 // indirect + github.com/go-redis/redis/v8 v8.8.2 github.com/go-redis/redis_rate/v9 v9.1.1 // indirect github.com/joho/godotenv v1.3.0 github.com/klauspost/compress v1.12.1 // indirect @@ -14,6 +14,6 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.8.1 github.com/tribalwarshelp/shared v0.0.0-20210423190057-03d8445d35dc - github.com/vmihailenco/taskq/v3 v3.2.3 // indirect + github.com/vmihailenco/taskq/v3 v3.2.3 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 // indirect ) diff --git a/go.sum b/go.sum index 6de7c76..65be078 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa0 github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83/go.mod h1:+iGkf5HfOVeRVd9K7qQDucIl+/Kt3MyenMa90b/O/c4= +github.com/aws/aws-sdk-go v1.35.28 h1:S2LuRnfC8X05zgZLC8gy/Sb82TGv2Cpytzbzz7tkeHc= github.com/aws/aws-sdk-go v1.35.28/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/bsm/redislock v0.7.0 h1:RL7aZJhCKkuBjQbnSTKCeedTRifBWxd/ffP+GZ599Mo= github.com/bsm/redislock v0.7.0/go.mod h1:3Kgu+cXw0JrkZ5pmY/JbcFpixGZ5M9v9G2PGWYqku+k= @@ -18,6 +19,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= @@ -63,14 +65,17 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -89,14 +94,14 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -221,7 +226,6 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/main.go b/main.go index ae6ca10..4d35874 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,23 @@ package main import ( + "context" + "github.com/go-redis/redis/v8" + "github.com/pkg/errors" "os" "os/signal" - "strconv" - "strings" "syscall" + "time" "github.com/sirupsen/logrus" "github.com/tribalwarshelp/shared/mode" - _cron "github.com/tribalwarshelp/cron/cron" + twhelpcron "github.com/tribalwarshelp/cron/cron" + "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/cron/tasks" + "github.com/tribalwarshelp/cron/db" + envutils "github.com/tribalwarshelp/cron/utils/env" - gopglogrusquerylogger "github.com/Kichiyaki/go-pg-logrus-query-logger/v10" - "github.com/go-pg/pg/v10" "github.com/joho/godotenv" "github.com/robfig/cron/v3" ) @@ -29,38 +33,44 @@ func init() { } func main() { - dbOptions := &pg.Options{ - User: os.Getenv("DB_USER"), - Password: os.Getenv("DB_PASSWORD"), - Database: os.Getenv("DB_NAME"), - Addr: os.Getenv("DB_HOST") + ":" + os.Getenv("DB_PORT"), - PoolSize: mustParseEnvToInt("DB_POOL_SIZE"), + client, err := initializeRedis() + if err != nil { + logrus.Fatal(errors.Wrap(err, "Error establishing a redis connection")) } - dbFields := logrus.Fields{ - "user": dbOptions.User, - "database": dbOptions.Database, - "addr": dbOptions.Addr, + defer client.Close() + + conn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")}) + if err != nil { + logrus.Fatal(errors.Wrap(err, "Error establishing a database connection")) } - db := pg.Connect(dbOptions) - defer func() { - if err := db.Close(); err != nil { - logrus.WithFields(dbFields).Fatalln(err) - } - }() - if strings.ToUpper(os.Getenv("LOG_DB_QUERIES")) == "TRUE" { - db.AddQueryHook(gopglogrusquerylogger.QueryLogger{ - Entry: logrus.NewEntry(logrus.StandardLogger()), - }) + defer conn.Close() + logrus.Info("Connection with the database has been established") + + queue, err := queue.New(&queue.Config{ + WorkerLimit: envutils.GetenvInt("MAX_CONCURRENT_WORKERS"), + Redis: client, + }) + if err != nil { + logrus.Fatal(errors.Wrap(err, "Couldn't create the task queue")) + } + tasks.RegisterTasks(&tasks.Config{ + DB: conn, + Queue: queue, + }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := queue.Start(ctx); err != nil { + logrus.Fatal(err) } - logrus.WithFields(dbFields).Info("Connection with the database has been established") c := cron.New(cron.WithChain( cron.SkipIfStillRunning(cron.PrintfLogger(logrus.WithField("package", "cron"))), )) - if err := _cron.Attach(c, _cron.Config{ - DB: db, - MaxConcurrentWorkers: mustParseEnvToInt("MAX_CONCURRENT_WORKERS"), - RunOnStartup: os.Getenv("RUN_ON_STARTUP") == "true", + if err := twhelpcron.Attach(c, twhelpcron.Config{ + DB: conn, + MaxConcurrentWorkers: envutils.GetenvInt("MAX_CONCURRENT_WORKERS"), + RunOnStartup: envutils.GetenvBool("RUN_ON_STARTUP"), + Queue: queue, }); err != nil { logrus.Fatal(err) } @@ -74,18 +84,9 @@ func main() { <-channel logrus.Info("shutting down") -} - -func mustParseEnvToInt(key string) int { - str := os.Getenv(key) - if str == "" { - return 0 + if err := queue.Close(); err != nil { + logrus.Fatal(err) } - i, err := strconv.Atoi(str) - if err != nil { - return 0 - } - return i } func setupLogger() { @@ -105,3 +106,18 @@ func setupLogger() { logrus.SetFormatter(customFormatter) } } + +func initializeRedis() (redis.UniversalClient, error) { + client := redis.NewClient(&redis.Options{ + Addr: envutils.GetenvString("REDIS_ADDR"), + Username: envutils.GetenvString("REDIS_USERNAME"), + Password: envutils.GetenvString("REDIS_PASSWORD"), + DB: envutils.GetenvInt("REDIS_DB"), + }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + return nil, errors.Wrap(err, "initializeRedis") + } + return client, nil +} diff --git a/utils/env/env.go b/utils/env/env.go new file mode 100644 index 0000000..626395c --- /dev/null +++ b/utils/env/env.go @@ -0,0 +1,30 @@ +package envutils + +import ( + "os" + "strconv" +) + +func GetenvInt(key string) int { + str := GetenvString(key) + if str == "" { + return 0 + } + i, err := strconv.Atoi(str) + if err != nil { + return 0 + } + return i +} + +func GetenvBool(key string) bool { + str := GetenvString(key) + if str == "" { + return false + } + return str == "true" || str == "1" +} + +func GetenvString(key string) string { + return os.Getenv(key) +} From cdd8c9b03f099b1a3f860682e33124c5f1b1f9fd Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 17:21:32 +0200 Subject: [PATCH 05/31] add taskLodVersions --- cron/cron.go | 6 +++++- cron/tasks/task_load_servers.go | 14 ++++++++++++-- cron/tasks/task_load_versions.go | 28 ++++++++++++++++++++++++++++ cron/tasks/tasks.go | 9 +++++++-- main.go | 4 +--- 5 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 cron/tasks/task_load_versions.go diff --git a/cron/cron.go b/cron/cron.go index 38b5788..0a1e618 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -30,7 +30,11 @@ func Attach(c *cron.Cron, cfg Config) error { return fmt.Errorf("cfg.Queue cannot be nil, expected queue.Queue") } - h := &handler{db: cfg.DB, maxConcurrentWorkers: cfg.MaxConcurrentWorkers} + h := &handler{ + db: cfg.DB, + maxConcurrentWorkers: cfg.MaxConcurrentWorkers, + queue: cfg.Queue, + } if err := h.init(); err != nil { return err } diff --git a/cron/tasks/task_load_servers.go b/cron/tasks/task_load_servers.go index 8190f7f..9475e89 100644 --- a/cron/tasks/task_load_servers.go +++ b/cron/tasks/task_load_servers.go @@ -68,7 +68,7 @@ func (t *taskLoadServers) execute(version *models.Version) error { if _, err := t.db.Model(&models.Server{}). Set("status = ?", models.ServerStatusClosed). - Where("key NOT IN (?) AND version_code =", pg.In(serverKeys), version.Code). + Where("key NOT IN (?) AND version_code = ?", pg.In(serverKeys), version.Code). Update(); err != nil { err = errors.Wrap(err, "couldn't update server statuses") logrus.Fatal(err) @@ -92,6 +92,7 @@ func (t *taskLoadServers) getServers(version *models.Version) (map[string]string return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't load servers", version.Host) } defer resp.Body.Close() + bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't read the body", version.Host) @@ -100,5 +101,14 @@ func (t *taskLoadServers) getServers(version *models.Version) (map[string]string if err != nil { return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't decode the body into the go value", version.Host) } - return body.(map[string]string), nil + + result := make(map[string]string) + for serverKey, url := range body.(map[interface{}]interface{}) { + serverKeyStr := serverKey.(string) + urlStr := url.(string) + if serverKeyStr != "" && urlStr != "" { + result[serverKeyStr] = urlStr + } + } + return result, nil } diff --git a/cron/tasks/task_load_versions.go b/cron/tasks/task_load_versions.go new file mode 100644 index 0000000..6ee2fb1 --- /dev/null +++ b/cron/tasks/task_load_versions.go @@ -0,0 +1,28 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskLoadVersions struct { + *task +} + +func (t *taskLoadVersions) execute() error { + var versions []*models.Version + log.Debug("taskLoadVersions.execute: Loading versions...") + if err := t.db.Model(&versions).Relation("SpecialServers").Select(); err != nil { + err = errors.Wrap(err, "taskLoadVersions.execute: couldn't load versions") + log.Fatal(err) + return err + } + for _, version := range versions { + t.queue.Add(queue.MainQueue, Get(TaskNameLoadServers).WithArgs(context.Background(), version)) + } + log.Debug("taskLoadVersions.execute: Versions have been loaded") + return nil +} diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 10bda4e..919d5e8 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -13,7 +13,7 @@ const ( TaskNameLoadVersions = "loadVersions" TaskNameLoadServers = "loadServers" TaskNameUpdateServerData = "updateServerData" - retryLimitLoadServers = 3 + defaultRetryLimit = 3 ) var log = logrus.WithField("package", "tasks") @@ -32,9 +32,14 @@ func RegisterTasks(cfg *Config) error { db: cfg.DB, queue: cfg.Queue, } + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskNameLoadVersions, + RetryLimit: defaultRetryLimit, + Handler: (&taskLoadVersions{t}).execute, + }) taskq.RegisterTask(&taskq.TaskOptions{ Name: TaskNameLoadServers, - RetryLimit: retryLimitLoadServers, + RetryLimit: defaultRetryLimit, Handler: (&taskLoadServers{t}).execute, }) diff --git a/main.go b/main.go index 4d35874..dc57079 100644 --- a/main.go +++ b/main.go @@ -57,9 +57,7 @@ func main() { DB: conn, Queue: queue, }) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := queue.Start(ctx); err != nil { + if err := queue.Start(context.Background()); err != nil { logrus.Fatal(err) } From 349d8e9b595d2fd3bc2b3b9fd3a06b29882697fe Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 17:37:08 +0200 Subject: [PATCH 06/31] rename taskLoadServers -> taskLoadServersAndUpdateData, taskLoadVersions -> taskLoadVersionsAndUpdateServerData --- cron/handler.go | 2 +- ...o => task_load_servers_and_update_data.go} | 16 +++++------ cron/tasks/task_load_versions.go | 28 ------------------- ...sk_load_versions_and_update_server_data.go | 28 +++++++++++++++++++ cron/tasks/task_update_server_data.go | 1 + cron/tasks/tasks.go | 16 +++++------ 6 files changed, 46 insertions(+), 45 deletions(-) rename cron/tasks/{task_load_servers.go => task_load_servers_and_update_data.go} (75%) delete mode 100644 cron/tasks/task_load_versions.go create mode 100644 cron/tasks/task_load_versions_and_update_server_data.go create mode 100644 cron/tasks/task_update_server_data.go diff --git a/cron/handler.go b/cron/handler.go index d264201..f03aae8 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -37,7 +37,7 @@ func (h *handler) init() error { } func (h *handler) updateServerData() { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersions).WithArgs(context.Background())) + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) } func (h *handler) updateServerEnnoblements() { diff --git a/cron/tasks/task_load_servers.go b/cron/tasks/task_load_servers_and_update_data.go similarity index 75% rename from cron/tasks/task_load_servers.go rename to cron/tasks/task_load_servers_and_update_data.go index 9475e89..b3798f5 100644 --- a/cron/tasks/task_load_servers.go +++ b/cron/tasks/task_load_servers_and_update_data.go @@ -17,11 +17,11 @@ const ( endpointGetServers = "/backend/get_servers.php" ) -type taskLoadServers struct { +type taskLoadServersAndUpdateData struct { *task } -func (t *taskLoadServers) execute(version *models.Version) error { +func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { if err := t.validatePayload(version); err != nil { return nil } @@ -79,27 +79,27 @@ func (t *taskLoadServers) execute(version *models.Version) error { return nil } -func (t *taskLoadServers) validatePayload(version *models.Version) error { +func (t *taskLoadServersAndUpdateData) validatePayload(version *models.Version) error { if version == nil { - return errors.Errorf("taskLoadServers.validatePayload: Expected *models.Version, got nil") + return errors.Errorf("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil") } return nil } -func (t *taskLoadServers) getServers(version *models.Version) (map[string]string, error) { +func (t *taskLoadServersAndUpdateData) getServers(version *models.Version) (map[string]string, error) { resp, err := http.Get(fmt.Sprintf("https://%s%s", version.Host, endpointGetServers)) if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't load servers", version.Host) + return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't load servers", version.Host) } defer resp.Body.Close() bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't read the body", version.Host) + return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the body", version.Host) } body, err := phpserialize.Decode(string(bodyBytes)) if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServers.loadServers couldn't decode the body into the go value", version.Host) + return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the body into the go value", version.Host) } result := make(map[string]string) diff --git a/cron/tasks/task_load_versions.go b/cron/tasks/task_load_versions.go deleted file mode 100644 index 6ee2fb1..0000000 --- a/cron/tasks/task_load_versions.go +++ /dev/null @@ -1,28 +0,0 @@ -package tasks - -import ( - "context" - "github.com/pkg/errors" - "github.com/tribalwarshelp/shared/models" - - "github.com/tribalwarshelp/cron/cron/queue" -) - -type taskLoadVersions struct { - *task -} - -func (t *taskLoadVersions) execute() error { - var versions []*models.Version - log.Debug("taskLoadVersions.execute: Loading versions...") - if err := t.db.Model(&versions).Relation("SpecialServers").Select(); err != nil { - err = errors.Wrap(err, "taskLoadVersions.execute: couldn't load versions") - log.Fatal(err) - return err - } - for _, version := range versions { - t.queue.Add(queue.MainQueue, Get(TaskNameLoadServers).WithArgs(context.Background(), version)) - } - log.Debug("taskLoadVersions.execute: Versions have been loaded") - return nil -} diff --git a/cron/tasks/task_load_versions_and_update_server_data.go b/cron/tasks/task_load_versions_and_update_server_data.go new file mode 100644 index 0000000..847efd5 --- /dev/null +++ b/cron/tasks/task_load_versions_and_update_server_data.go @@ -0,0 +1,28 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskLoadVersionsAndUpdateServerData struct { + *task +} + +func (t *taskLoadVersionsAndUpdateServerData) execute() error { + var versions []*models.Version + log.Debug("taskLoadVersionsAndUpdateServerData.execute: Loading versions...") + if err := t.db.Model(&versions).Relation("SpecialServers").Select(); err != nil { + err = errors.Wrap(err, "taskLoadVersionsAndUpdateServerData.execute: couldn't load versions") + log.Fatal(err) + return err + } + for _, version := range versions { + t.queue.Add(queue.MainQueue, Get(TaskNameLoadServersAndUpdateData).WithArgs(context.Background(), version)) + } + log.Debug("taskLoadVersionsAndUpdateServerData.execute: Versions have been loaded") + return nil +} diff --git a/cron/tasks/task_update_server_data.go b/cron/tasks/task_update_server_data.go new file mode 100644 index 0000000..9b29ce4 --- /dev/null +++ b/cron/tasks/task_update_server_data.go @@ -0,0 +1 @@ +package tasks diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 919d5e8..ac68916 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -10,10 +10,10 @@ import ( ) const ( - TaskNameLoadVersions = "loadVersions" - TaskNameLoadServers = "loadServers" - TaskNameUpdateServerData = "updateServerData" - defaultRetryLimit = 3 + TaskNameLoadVersionsAndUpdateServerData = "loadVersions" + TaskNameLoadServersAndUpdateData = "loadServers" + TaskNameUpdateServerData = "updateServerData" + defaultRetryLimit = 3 ) var log = logrus.WithField("package", "tasks") @@ -33,14 +33,14 @@ func RegisterTasks(cfg *Config) error { queue: cfg.Queue, } taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameLoadVersions, + Name: TaskNameLoadVersionsAndUpdateServerData, RetryLimit: defaultRetryLimit, - Handler: (&taskLoadVersions{t}).execute, + Handler: (&taskLoadVersionsAndUpdateServerData{t}).execute, }) taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameLoadServers, + Name: TaskNameLoadServersAndUpdateData, RetryLimit: defaultRetryLimit, - Handler: (&taskLoadServers{t}).execute, + Handler: (&taskLoadServersAndUpdateData{t}).execute, }) return nil From 072554cef8f1973518b447bffd98af738e876dd2 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 18:21:34 +0200 Subject: [PATCH 07/31] add taskUpdateServerData --- cron/cron.go | 5 +- cron/queue/queue.go | 1 + cron/tasks/helpers.go | 103 +++++ .../task_load_servers_and_update_data.go | 12 +- cron/tasks/task_update_server_data.go | 369 ++++++++++++++++++ cron/tasks/tasks.go | 5 + 6 files changed, 490 insertions(+), 5 deletions(-) create mode 100644 cron/tasks/helpers.go diff --git a/cron/cron.go b/cron/cron.go index 0a1e618..aae5120 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -44,7 +44,6 @@ func Attach(c *cron.Cron, cfg Config) error { return err } - updateServerData := utils.TrackExecutionTime(log, h.updateServerData, "updateServerData") vacuumDatabase := utils.TrackExecutionTime(log, h.vacuumDatabase, "vacuumDatabase") updateServerEnnoblements := utils.TrackExecutionTime(log, h.updateServerEnnoblements, "updateServerEnnoblements") var updateHistoryFuncs []func() @@ -67,7 +66,7 @@ func Attach(c *cron.Cron, cfg Config) error { return err } } - if _, err := c.AddFunc("0 * * * *", updateServerData); err != nil { + if _, err := c.AddFunc("0 * * * *", h.updateServerData); err != nil { return err } if _, err := c.AddFunc("20 1 * * *", vacuumDatabase); err != nil { @@ -78,7 +77,7 @@ func Attach(c *cron.Cron, cfg Config) error { } if cfg.RunOnStartup { go func() { - updateServerData() + h.updateServerData() //vacuumDatabase() //for _, fn := range updateHistoryFuncs { // go fn() diff --git a/cron/queue/queue.go b/cron/queue/queue.go index 0b90fd6..75f32d2 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -64,6 +64,7 @@ func (q *queue) registerQueue(name QueueName, limit int) taskq.Queue { Name: string(name), ReservationTimeout: time.Minute * 2, Redis: q.redis, + MinNumWorker: int32(limit), MaxNumWorker: int32(limit), }) } diff --git a/cron/tasks/helpers.go b/cron/tasks/helpers.go new file mode 100644 index 0000000..d30542f --- /dev/null +++ b/cron/tasks/helpers.go @@ -0,0 +1,103 @@ +package tasks + +import ( + "time" + + "github.com/tribalwarshelp/shared/models" +) + +func countPlayerVillages(villages []*models.Village) int { + count := 0 + for _, village := range villages { + if village.PlayerID != 0 { + count++ + } + } + return count +} + +func getDateDifferenceInDays(t1, t2 time.Time) int { + diff := t1.Sub(t2) + return int(diff.Hours() / 24) +} + +func calcPlayerDailyGrowth(diffInDays, points int) int { + if diffInDays > 0 { + return points / diffInDays + } + return 0 +} + +type tribesSearchableByID struct { + tribes []*models.Tribe +} + +func (searchable tribesSearchableByID) GetID(index int) int { + return searchable.tribes[index].ID +} + +func (searchable tribesSearchableByID) Len() int { + return len(searchable.tribes) +} + +type playersSearchableByID struct { + players []*models.Player +} + +func (searchable playersSearchableByID) GetID(index int) int { + return searchable.players[index].ID +} + +func (searchable playersSearchableByID) Len() int { + return len(searchable.players) +} + +type ennoblementsSearchableByNewOwnerID struct { + ennoblements []*models.Ennoblement +} + +func (searchable ennoblementsSearchableByNewOwnerID) GetID(index int) int { + return searchable.ennoblements[index].NewOwnerID +} + +func (searchable ennoblementsSearchableByNewOwnerID) Len() int { + return len(searchable.ennoblements) +} + +type searchableByID interface { + GetID(index int) int + Len() int +} + +func makePlayersSearchable(players []*models.Player) searchableByID { + return playersSearchableByID{ + players: players, + } +} + +func makeTribesSearchable(tribes []*models.Tribe) searchableByID { + return tribesSearchableByID{ + tribes: tribes, + } +} + +func searchByID(haystack searchableByID, id int) int { + low := 0 + high := haystack.Len() - 1 + + for low <= high { + median := (low + high) / 2 + + if haystack.GetID(median) < id { + low = median + 1 + } else { + high = median - 1 + } + } + + if low == haystack.Len() || haystack.GetID(low) != id { + return -1 + } + + return low +} diff --git a/cron/tasks/task_load_servers_and_update_data.go b/cron/tasks/task_load_servers_and_update_data.go index b3798f5..30462f2 100644 --- a/cron/tasks/task_load_servers_and_update_data.go +++ b/cron/tasks/task_load_servers_and_update_data.go @@ -1,6 +1,7 @@ package tasks import ( + "context" "fmt" phpserialize "github.com/Kichiyaki/go-php-serialize" "github.com/go-pg/pg/v10" @@ -10,6 +11,7 @@ import ( "io/ioutil" "net/http" + "github.com/tribalwarshelp/cron/cron/queue" "github.com/tribalwarshelp/cron/db" ) @@ -23,6 +25,7 @@ type taskLoadServersAndUpdateData struct { func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { if err := t.validatePayload(version); err != nil { + log.Debug(err) return nil } entry := log.WithField("host", version.Host) @@ -60,7 +63,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { Set("version_code = EXCLUDED.version_code"). Returning("*"). Insert(); err != nil { - err = errors.Wrap(err, "couldn't insert/update servers") + err = errors.Wrap(err, "taskLoadServersAndUpdateData.execute: couldn't insert/update servers") logrus.Fatal(err) return err } @@ -70,11 +73,16 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { Set("status = ?", models.ServerStatusClosed). Where("key NOT IN (?) AND version_code = ?", pg.In(serverKeys), version.Code). Update(); err != nil { - err = errors.Wrap(err, "couldn't update server statuses") + err = errors.Wrap(err, "taskLoadServersAndUpdateData.execute: couldn't update server statuses") logrus.Fatal(err) return err } + for _, server := range servers { + s := server + t.queue.Add(queue.MainQueue, Get(TaskNameUpdateServerData).WithArgs(context.Background(), data[s.Key], s)) + } + entry.Infof("%s: Servers have been loaded", version.Host) return nil } diff --git a/cron/tasks/task_update_server_data.go b/cron/tasks/task_update_server_data.go index 9b29ce4..427307a 100644 --- a/cron/tasks/task_update_server_data.go +++ b/cron/tasks/task_update_server_data.go @@ -1 +1,370 @@ package tasks + +import ( + "github.com/go-pg/pg/v10" + "github.com/go-pg/pg/v10/orm" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + "github.com/tribalwarshelp/shared/tw/dataloader" + "net/http" + "time" +) + +type taskUpdateServerData struct { + *task +} + +func (t *taskUpdateServerData) execute(url string, server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + now := time.Now() + entry := log.WithField("key", server.Key) + entry.Infof("%s: updating data...", server.Key) + err := (&workerUpdateServerData{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + dataloader: dataloader.New(&dataloader.Config{ + BaseURL: url, + Client: &http.Client{ + Timeout: 10 * time.Second, + }, + }), + server: server, + }).update() + if err != nil { + err = errors.Wrap(err, "taskUpdateServerData.execute") + entry.Error(err) + return err + } + duration := time.Since(now) + entry. + WithFields(map[string]interface{}{ + "duration": duration.Nanoseconds(), + "durationPretty": duration.String(), + }). + Infof("%s has been updated", server.Key) + return nil +} + +func (t *taskUpdateServerData) validatePayload(server *models.Server) error { + if server == nil { + return errors.Errorf("taskLoadServersAndUpdateData.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + +type workerUpdateServerData struct { + db *pg.DB + dataloader dataloader.DataLoader + server *models.Server +} + +func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeated) ([]*models.Player, error) { + var ennoblements = []*models.Ennoblement{} + err := w.db.Model(&ennoblements).DistinctOn("new_owner_id").Order("new_owner_id ASC", "ennobled_at ASC").Select() + if err != nil { + return nil, errors.Wrap(err, "loadPlayers: couldn't load ennoblements") + } + + players, err := w.dataloader.LoadPlayers() + if err != nil { + return nil, err + } + + now := time.Now() + searchableByNewOwnerID := &ennoblementsSearchableByNewOwnerID{ennoblements: ennoblements} + for _, player := range players { + playerOD, ok := od[player.ID] + if ok { + player.OpponentsDefeated = *playerOD + } + firstEnnoblementIndex := searchByID(searchableByNewOwnerID, player.ID) + if firstEnnoblementIndex != -1 { + firstEnnoblement := ennoblements[firstEnnoblementIndex] + diffInDays := getDateDifferenceInDays(now, firstEnnoblement.EnnobledAt) + player.DailyGrowth = calcPlayerDailyGrowth(diffInDays, player.Points) + } + } + return players, nil +} + +func (w *workerUpdateServerData) loadTribes(od map[int]*models.OpponentsDefeated, numberOfVillages int) ([]*models.Tribe, error) { + tribes, err := w.dataloader.LoadTribes() + if err != nil { + return nil, err + } + for _, tribe := range tribes { + tribeOD, ok := od[tribe.ID] + if ok { + tribe.OpponentsDefeated = *tribeOD + } + if tribe.TotalVillages > 0 && numberOfVillages > 0 { + tribe.Dominance = float64(tribe.TotalVillages) / float64(numberOfVillages) * 100 + } else { + tribe.Dominance = 0 + } + } + return tribes, nil +} + +func (w *workerUpdateServerData) calculateODifference(od1 models.OpponentsDefeated, od2 models.OpponentsDefeated) models.OpponentsDefeated { + return models.OpponentsDefeated{ + RankAtt: (od1.RankAtt - od2.RankAtt) * -1, + ScoreAtt: od1.ScoreAtt - od2.ScoreAtt, + RankDef: (od1.RankDef - od2.RankDef) * -1, + ScoreDef: od1.ScoreDef - od2.ScoreDef, + RankSup: (od1.RankSup - od2.RankSup) * -1, + ScoreSup: od1.ScoreSup - od2.ScoreSup, + RankTotal: (od1.RankTotal - od2.RankTotal) * -1, + ScoreTotal: od1.ScoreTotal - od2.ScoreTotal, + } +} + +func (w *workerUpdateServerData) calculateTodaysTribeStats( + tribes []*models.Tribe, + history []*models.TribeHistory, +) []*models.DailyTribeStats { + var todaysStats []*models.DailyTribeStats + searchableTribes := makeTribesSearchable(tribes) + + for _, historyRecord := range history { + if index := searchByID(searchableTribes, historyRecord.TribeID); index != -1 { + tribe := tribes[index] + todaysStats = append(todaysStats, &models.DailyTribeStats{ + TribeID: tribe.ID, + Members: tribe.TotalMembers - historyRecord.TotalMembers, + Villages: tribe.TotalVillages - historyRecord.TotalVillages, + Points: tribe.Points - historyRecord.Points, + AllPoints: tribe.AllPoints - historyRecord.AllPoints, + Rank: (tribe.Rank - historyRecord.Rank) * -1, + Dominance: tribe.Dominance - historyRecord.Dominance, + CreateDate: historyRecord.CreateDate, + OpponentsDefeated: w.calculateODifference(tribe.OpponentsDefeated, historyRecord.OpponentsDefeated), + }) + } + } + + return todaysStats +} + +func (w *workerUpdateServerData) calculateDailyPlayerStats(players []*models.Player, + history []*models.PlayerHistory) []*models.DailyPlayerStats { + todaysStats := []*models.DailyPlayerStats{} + searchablePlayers := makePlayersSearchable(players) + + for _, historyRecord := range history { + if index := searchByID(searchablePlayers, historyRecord.PlayerID); index != -1 { + player := players[index] + todaysStats = append(todaysStats, &models.DailyPlayerStats{ + PlayerID: player.ID, + Villages: player.TotalVillages - historyRecord.TotalVillages, + Points: player.Points - historyRecord.Points, + Rank: (player.Rank - historyRecord.Rank) * -1, + CreateDate: historyRecord.CreateDate, + OpponentsDefeated: w.calculateODifference(player.OpponentsDefeated, historyRecord.OpponentsDefeated), + }) + } + } + + return todaysStats +} + +func (w *workerUpdateServerData) update() error { + pod, err := w.dataloader.LoadOD(false) + if err != nil { + return err + } + tod, err := w.dataloader.LoadOD(true) + if err != nil { + return err + } + + villages, err := w.dataloader.LoadVillages() + if err != nil { + return err + } + numberOfVillages := len(villages) + + tribes, err := w.loadTribes(tod, countPlayerVillages(villages)) + if err != nil { + return err + } + numberOfTribes := len(tribes) + + players, err := w.loadPlayers(pod) + if err != nil { + return err + } + numberOfPlayers := len(players) + + cfg, err := w.dataloader.GetConfig() + if err != nil { + return err + } + buildingCfg, err := w.dataloader.GetBuildingConfig() + if err != nil { + return err + } + unitCfg, err := w.dataloader.GetUnitConfig() + if err != nil { + return err + } + + tx, err := w.db.Begin() + if err != nil { + return err + } + defer tx.Close() + + if len(tribes) > 0 { + ids := []int{} + for _, tribe := range tribes { + ids = append(ids, tribe.ID) + } + + if _, err := tx.Model(&tribes). + OnConflict("(id) DO UPDATE"). + Set("name = EXCLUDED.name"). + Set("tag = EXCLUDED.tag"). + Set("total_members = EXCLUDED.total_members"). + Set("total_villages = EXCLUDED.total_villages"). + Set("points = EXCLUDED.points"). + Set("all_points = EXCLUDED.all_points"). + Set("rank = EXCLUDED.rank"). + Set("exists = EXCLUDED.exists"). + Set("dominance = EXCLUDED.dominance"). + Apply(appendODSetClauses). + Insert(); err != nil { + return errors.Wrap(err, "couldn't insert tribes") + } + if _, err := tx.Model(&tribes). + Where("NOT (tribe.id = ANY (?))", pg.Array(ids)). + Set("exists = false"). + Update(); err != nil && err != pg.ErrNoRows { + return errors.Wrap(err, "couldn't update non-existent tribes") + } + + tribesHistory := []*models.TribeHistory{} + if err := w.db.Model(&tribesHistory). + DistinctOn("tribe_id"). + Column("*"). + Where("tribe_id = ANY (?)", pg.Array(ids)). + Order("tribe_id DESC", "create_date DESC"). + Select(); err != nil && err != pg.ErrNoRows { + return errors.Wrap(err, "couldn't select tribe history records") + } + todaysTribeStats := w.calculateTodaysTribeStats(tribes, tribesHistory) + if len(todaysTribeStats) > 0 { + if _, err := tx. + Model(&todaysTribeStats). + OnConflict("ON CONSTRAINT daily_tribe_stats_tribe_id_create_date_key DO UPDATE"). + Set("members = EXCLUDED.members"). + Set("villages = EXCLUDED.villages"). + Set("points = EXCLUDED.points"). + Set("all_points = EXCLUDED.all_points"). + Set("rank = EXCLUDED.rank"). + Set("dominance = EXCLUDED.dominance"). + Apply(appendODSetClauses). + Insert(); err != nil { + return errors.Wrap(err, "couldn't insert today's tribe stats") + } + } + } + + if len(players) > 0 { + ids := []int{} + for _, player := range players { + ids = append(ids, player.ID) + } + if _, err := tx.Model(&players). + OnConflict("(id) DO UPDATE"). + Set("name = EXCLUDED.name"). + Set("total_villages = EXCLUDED.total_villages"). + Set("points = EXCLUDED.points"). + Set("rank = EXCLUDED.rank"). + Set("exists = EXCLUDED.exists"). + Set("tribe_id = EXCLUDED.tribe_id"). + Set("daily_growth = EXCLUDED.daily_growth"). + Apply(appendODSetClauses). + Insert(); err != nil { + return errors.Wrap(err, "couldn't insert players") + } + if _, err := tx.Model(&models.Player{}). + Where("NOT (player.id = ANY (?))", pg.Array(ids)). + Set("exists = false"). + Set("tribe_id = 0"). + Update(); err != nil && err != pg.ErrNoRows { + return errors.Wrap(err, "couldn't update non-existent players") + } + + playerHistory := []*models.PlayerHistory{} + if err := w.db.Model(&playerHistory). + DistinctOn("player_id"). + Column("*"). + Where("player_id = ANY (?)", pg.Array(ids)). + Order("player_id DESC", "create_date DESC").Select(); err != nil && err != pg.ErrNoRows { + return errors.Wrap(err, "couldn't select player history records") + } + todaysPlayerStats := w.calculateDailyPlayerStats(players, playerHistory) + if len(todaysPlayerStats) > 0 { + if _, err := tx. + Model(&todaysPlayerStats). + OnConflict("ON CONSTRAINT daily_player_stats_player_id_create_date_key DO UPDATE"). + Set("villages = EXCLUDED.villages"). + Set("points = EXCLUDED.points"). + Set("rank = EXCLUDED.rank"). + Apply(appendODSetClauses). + Insert(); err != nil { + return errors.Wrap(err, "couldn't insert today's player stats") + } + } + } + + if len(villages) > 0 { + if _, err := tx.Model(&villages). + OnConflict("(id) DO UPDATE"). + Set("name = EXCLUDED.name"). + Set("points = EXCLUDED.points"). + Set("x = EXCLUDED.x"). + Set("y = EXCLUDED.y"). + Set("bonus = EXCLUDED.bonus"). + Set("player_id = EXCLUDED.player_id"). + Insert(); err != nil { + return errors.Wrap(err, "couldn't insert villages") + } + } + + if _, err := tx.Model(w.server). + Set("data_updated_at = ?", time.Now()). + Set("unit_config = ?", unitCfg). + Set("building_config = ?", buildingCfg). + Set("config = ?", cfg). + Set("number_of_players = ?", numberOfPlayers). + Set("number_of_tribes = ?", numberOfTribes). + Set("number_of_villages = ?", numberOfVillages). + Returning("*"). + WherePK(). + Update(); err != nil { + return errors.Wrap(err, "couldn't update server") + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +func appendODSetClauses(q *orm.Query) (*orm.Query, error) { + return q.Set("rank_att = EXCLUDED.rank_att"). + Set("score_att = EXCLUDED.score_att"). + Set("rank_def = EXCLUDED.rank_def"). + Set("score_def = EXCLUDED.score_def"). + Set("rank_sup = EXCLUDED.rank_sup"). + Set("score_sup = EXCLUDED.score_sup"). + Set("rank_total = EXCLUDED.rank_total"). + Set("score_total = EXCLUDED.score_total"), + nil +} diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index ac68916..2b79c55 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -42,6 +42,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskLoadServersAndUpdateData{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskNameUpdateServerData, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateServerData{t}).execute, + }) return nil } From 579de07fb7fc70a7bca536f908a57c4e471f2e11 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 18:23:37 +0200 Subject: [PATCH 08/31] cleanup --- cron/helpers.go | 75 ------- cron/searchable_by_id.go | 77 ------- cron/update_server_data_worker.go | 329 ------------------------------ 3 files changed, 481 deletions(-) delete mode 100644 cron/searchable_by_id.go delete mode 100644 cron/update_server_data_worker.go diff --git a/cron/helpers.go b/cron/helpers.go index 034cad8..a700335 100644 --- a/cron/helpers.go +++ b/cron/helpers.go @@ -1,84 +1,9 @@ package cron import ( - "compress/gzip" - "encoding/csv" - "encoding/xml" - "io" - "io/ioutil" - "net/http" "time" - - "github.com/tribalwarshelp/shared/models" ) -var client = &http.Client{ - Timeout: 20 * time.Second, -} - -func newCsvReader(r io.Reader) *csv.Reader { - csvReader := csv.NewReader(r) - csvReader.LazyQuotes = true - return csvReader -} - -func uncompressAndReadCsvLines(r io.Reader) ([][]string, error) { - uncompressedStream, err := gzip.NewReader(r) - if err != nil { - return [][]string{}, err - } - defer uncompressedStream.Close() - return newCsvReader(uncompressedStream).ReadAll() -} - -func getCSVData(url string, compressed bool) ([][]string, error) { - resp, err := client.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if !compressed { - return newCsvReader(resp.Body).ReadAll() - } - return uncompressAndReadCsvLines(resp.Body) -} - -func getXML(url string, decode interface{}) error { - resp, err := client.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - bytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return xml.Unmarshal(bytes, decode) -} - -func countPlayerVillages(villages []*models.Village) int { - count := 0 - for _, village := range villages { - if village.PlayerID != 0 { - count++ - } - } - return count -} - -func getDateDifferenceInDays(t1, t2 time.Time) int { - diff := t1.Sub(t2) - return int(diff.Hours() / 24) -} - -func calcPlayerDailyGrowth(diffInDays, points int) int { - if diffInDays > 0 { - return points / diffInDays - } - return 0 -} - func createFnWithTimezone(timezone string, fn func(location *time.Location)) func() { tz, err := time.LoadLocation(timezone) if err != nil { diff --git a/cron/searchable_by_id.go b/cron/searchable_by_id.go deleted file mode 100644 index 5a1a535..0000000 --- a/cron/searchable_by_id.go +++ /dev/null @@ -1,77 +0,0 @@ -package cron - -import "github.com/tribalwarshelp/shared/models" - -type tribesSearchableByID struct { - tribes []*models.Tribe -} - -func (searchable tribesSearchableByID) GetID(index int) int { - return searchable.tribes[index].ID -} - -func (searchable tribesSearchableByID) Len() int { - return len(searchable.tribes) -} - -type playersSearchableByID struct { - players []*models.Player -} - -func (searchable playersSearchableByID) GetID(index int) int { - return searchable.players[index].ID -} - -func (searchable playersSearchableByID) Len() int { - return len(searchable.players) -} - -type ennoblementsSearchableByNewOwnerID struct { - ennoblements []*models.Ennoblement -} - -func (searchable ennoblementsSearchableByNewOwnerID) GetID(index int) int { - return searchable.ennoblements[index].NewOwnerID -} - -func (searchable ennoblementsSearchableByNewOwnerID) Len() int { - return len(searchable.ennoblements) -} - -type searchableByID interface { - GetID(index int) int - Len() int -} - -func makePlayersSearchable(players []*models.Player) searchableByID { - return playersSearchableByID{ - players: players, - } -} - -func makeTribesSearchable(tribes []*models.Tribe) searchableByID { - return tribesSearchableByID{ - tribes: tribes, - } -} - -func searchByID(haystack searchableByID, id int) int { - low := 0 - high := haystack.Len() - 1 - - for low <= high { - median := (low + high) / 2 - - if haystack.GetID(median) < id { - low = median + 1 - } else { - high = median - 1 - } - } - - if low == haystack.Len() || haystack.GetID(low) != id { - return -1 - } - - return low -} diff --git a/cron/update_server_data_worker.go b/cron/update_server_data_worker.go deleted file mode 100644 index a759411..0000000 --- a/cron/update_server_data_worker.go +++ /dev/null @@ -1,329 +0,0 @@ -package cron - -import ( - "time" - - "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/shared/tw/dataloader" - - "github.com/go-pg/pg/v10" - "github.com/go-pg/pg/v10/orm" - "github.com/pkg/errors" -) - -type updateServerDataWorker struct { - db *pg.DB - dataloader dataloader.DataLoader - server *models.Server -} - -func (w *updateServerDataWorker) loadPlayers(od map[int]*models.OpponentsDefeated) ([]*models.Player, error) { - ennoblements := []*models.Ennoblement{} - err := w.db.Model(&ennoblements).DistinctOn("new_owner_id").Order("new_owner_id ASC", "ennobled_at ASC").Select() - if err != nil { - return nil, errors.Wrap(err, "loadPlayers: cannot load ennoblements") - } - - players, err := w.dataloader.LoadPlayers() - if err != nil { - return nil, err - } - - now := time.Now() - searchableByNewOwnerID := &ennoblementsSearchableByNewOwnerID{ennoblements: ennoblements} - for _, player := range players { - playerOD, ok := od[player.ID] - if ok { - player.OpponentsDefeated = *playerOD - } - firstEnnoblementIndex := searchByID(searchableByNewOwnerID, player.ID) - if firstEnnoblementIndex != -1 { - firstEnnoblement := ennoblements[firstEnnoblementIndex] - diffInDays := getDateDifferenceInDays(now, firstEnnoblement.EnnobledAt) - player.DailyGrowth = calcPlayerDailyGrowth(diffInDays, player.Points) - } - } - return players, nil -} - -func (w *updateServerDataWorker) loadTribes(od map[int]*models.OpponentsDefeated, numberOfVillages int) ([]*models.Tribe, error) { - tribes, err := w.dataloader.LoadTribes() - if err != nil { - return nil, err - } - for _, tribe := range tribes { - tribeOD, ok := od[tribe.ID] - if ok { - tribe.OpponentsDefeated = *tribeOD - } - if tribe.TotalVillages > 0 && numberOfVillages > 0 { - tribe.Dominance = float64(tribe.TotalVillages) / float64(numberOfVillages) * 100 - } else { - tribe.Dominance = 0 - } - } - return tribes, nil -} - -func (w *updateServerDataWorker) calculateODifference(od1 models.OpponentsDefeated, od2 models.OpponentsDefeated) models.OpponentsDefeated { - return models.OpponentsDefeated{ - RankAtt: (od1.RankAtt - od2.RankAtt) * -1, - ScoreAtt: od1.ScoreAtt - od2.ScoreAtt, - RankDef: (od1.RankDef - od2.RankDef) * -1, - ScoreDef: od1.ScoreDef - od2.ScoreDef, - RankSup: (od1.RankSup - od2.RankSup) * -1, - ScoreSup: od1.ScoreSup - od2.ScoreSup, - RankTotal: (od1.RankTotal - od2.RankTotal) * -1, - ScoreTotal: od1.ScoreTotal - od2.ScoreTotal, - } -} - -func (w *updateServerDataWorker) calculateTodaysTribeStats(tribes []*models.Tribe, - history []*models.TribeHistory) []*models.DailyTribeStats { - todaysStats := []*models.DailyTribeStats{} - searchableTribes := makeTribesSearchable(tribes) - - for _, historyRecord := range history { - if index := searchByID(searchableTribes, historyRecord.TribeID); index != -1 { - tribe := tribes[index] - todaysStats = append(todaysStats, &models.DailyTribeStats{ - TribeID: tribe.ID, - Members: tribe.TotalMembers - historyRecord.TotalMembers, - Villages: tribe.TotalVillages - historyRecord.TotalVillages, - Points: tribe.Points - historyRecord.Points, - AllPoints: tribe.AllPoints - historyRecord.AllPoints, - Rank: (tribe.Rank - historyRecord.Rank) * -1, - Dominance: tribe.Dominance - historyRecord.Dominance, - CreateDate: historyRecord.CreateDate, - OpponentsDefeated: w.calculateODifference(tribe.OpponentsDefeated, historyRecord.OpponentsDefeated), - }) - } - } - - return todaysStats -} - -func (w *updateServerDataWorker) calculateDailyPlayerStats(players []*models.Player, - history []*models.PlayerHistory) []*models.DailyPlayerStats { - todaysStats := []*models.DailyPlayerStats{} - searchablePlayers := makePlayersSearchable(players) - - for _, historyRecord := range history { - if index := searchByID(searchablePlayers, historyRecord.PlayerID); index != -1 { - player := players[index] - todaysStats = append(todaysStats, &models.DailyPlayerStats{ - PlayerID: player.ID, - Villages: player.TotalVillages - historyRecord.TotalVillages, - Points: player.Points - historyRecord.Points, - Rank: (player.Rank - historyRecord.Rank) * -1, - CreateDate: historyRecord.CreateDate, - OpponentsDefeated: w.calculateODifference(player.OpponentsDefeated, historyRecord.OpponentsDefeated), - }) - } - } - - return todaysStats -} - -func (w *updateServerDataWorker) update() error { - pod, err := w.dataloader.LoadOD(false) - if err != nil { - return err - } - tod, err := w.dataloader.LoadOD(true) - if err != nil { - return err - } - - villages, err := w.dataloader.LoadVillages() - if err != nil { - return err - } - numberOfVillages := len(villages) - - tribes, err := w.loadTribes(tod, countPlayerVillages(villages)) - if err != nil { - return err - } - numberOfTribes := len(tribes) - - players, err := w.loadPlayers(pod) - if err != nil { - return err - } - numberOfPlayers := len(players) - - cfg, err := w.dataloader.GetConfig() - if err != nil { - return err - } - buildingCfg, err := w.dataloader.GetBuildingConfig() - if err != nil { - return err - } - unitCfg, err := w.dataloader.GetUnitConfig() - if err != nil { - return err - } - - tx, err := w.db.Begin() - if err != nil { - return err - } - defer tx.Close() - - if len(tribes) > 0 { - ids := []int{} - for _, tribe := range tribes { - ids = append(ids, tribe.ID) - } - - if _, err := tx.Model(&tribes). - OnConflict("(id) DO UPDATE"). - Set("name = EXCLUDED.name"). - Set("tag = EXCLUDED.tag"). - Set("total_members = EXCLUDED.total_members"). - Set("total_villages = EXCLUDED.total_villages"). - Set("points = EXCLUDED.points"). - Set("all_points = EXCLUDED.all_points"). - Set("rank = EXCLUDED.rank"). - Set("exists = EXCLUDED.exists"). - Set("dominance = EXCLUDED.dominance"). - Apply(appendODSetClauses). - Insert(); err != nil { - return errors.Wrap(err, "cannot insert tribes") - } - if _, err := tx.Model(&tribes). - Where("NOT (tribe.id = ANY (?))", pg.Array(ids)). - Set("exists = false"). - Update(); err != nil && err != pg.ErrNoRows { - return errors.Wrap(err, "cannot update nonexistent tribes") - } - - tribesHistory := []*models.TribeHistory{} - if err := w.db.Model(&tribesHistory). - DistinctOn("tribe_id"). - Column("*"). - Where("tribe_id = ANY (?)", pg.Array(ids)). - Order("tribe_id DESC", "create_date DESC"). - Select(); err != nil && err != pg.ErrNoRows { - return errors.Wrap(err, "cannot select tribe history records") - } - todaysTribeStats := w.calculateTodaysTribeStats(tribes, tribesHistory) - if len(todaysTribeStats) > 0 { - if _, err := tx. - Model(&todaysTribeStats). - OnConflict("ON CONSTRAINT daily_tribe_stats_tribe_id_create_date_key DO UPDATE"). - Set("members = EXCLUDED.members"). - Set("villages = EXCLUDED.villages"). - Set("points = EXCLUDED.points"). - Set("all_points = EXCLUDED.all_points"). - Set("rank = EXCLUDED.rank"). - Set("dominance = EXCLUDED.dominance"). - Apply(appendODSetClauses). - Insert(); err != nil { - return errors.Wrap(err, "cannot insert today's tribe stats") - } - } - } - - if len(players) > 0 { - ids := []int{} - for _, player := range players { - ids = append(ids, player.ID) - } - if _, err := tx.Model(&players). - OnConflict("(id) DO UPDATE"). - Set("name = EXCLUDED.name"). - Set("total_villages = EXCLUDED.total_villages"). - Set("points = EXCLUDED.points"). - Set("rank = EXCLUDED.rank"). - Set("exists = EXCLUDED.exists"). - Set("tribe_id = EXCLUDED.tribe_id"). - Set("daily_growth = EXCLUDED.daily_growth"). - Apply(appendODSetClauses). - Insert(); err != nil { - return errors.Wrap(err, "cannot insert players") - } - if _, err := tx.Model(&models.Player{}). - Where("NOT (player.id = ANY (?))", pg.Array(ids)). - Set("exists = false"). - Set("tribe_id = 0"). - Update(); err != nil && err != pg.ErrNoRows { - return errors.Wrap(err, "cannot update nonexistent players") - } - - playerHistory := []*models.PlayerHistory{} - if err := w.db.Model(&playerHistory). - DistinctOn("player_id"). - Column("*"). - Where("player_id = ANY (?)", pg.Array(ids)). - Order("player_id DESC", "create_date DESC").Select(); err != nil && err != pg.ErrNoRows { - return errors.Wrap(err, "cannot select player history records") - } - todaysPlayerStats := w.calculateDailyPlayerStats(players, playerHistory) - if len(todaysPlayerStats) > 0 { - if _, err := tx. - Model(&todaysPlayerStats). - OnConflict("ON CONSTRAINT daily_player_stats_player_id_create_date_key DO UPDATE"). - Set("villages = EXCLUDED.villages"). - Set("points = EXCLUDED.points"). - Set("rank = EXCLUDED.rank"). - Apply(appendODSetClauses). - Insert(); err != nil { - return errors.Wrap(err, "cannot insert today's player stats") - } - } - } - - if len(villages) > 0 { - if _, err := tx.Model(&models.Village{}). - Where("true"). - Delete(); err != nil && err != pg.ErrNoRows { - return errors.Wrap(err, "cannot delete villages") - } - if _, err := tx.Model(&villages). - OnConflict("(id) DO UPDATE"). - Set("name = EXCLUDED.name"). - Set("points = EXCLUDED.points"). - Set("x = EXCLUDED.x"). - Set("y = EXCLUDED.y"). - Set("bonus = EXCLUDED.bonus"). - Set("player_id = EXCLUDED.player_id"). - Insert(); err != nil { - return errors.Wrap(err, "cannot insert villages") - } - } - - if _, err := tx.Model(w.server). - Set("data_updated_at = ?", time.Now()). - Set("unit_config = ?", unitCfg). - Set("building_config = ?", buildingCfg). - Set("config = ?", cfg). - Set("number_of_players = ?", numberOfPlayers). - Set("number_of_tribes = ?", numberOfTribes). - Set("number_of_villages = ?", numberOfVillages). - Returning("*"). - WherePK(). - Update(); err != nil { - return errors.Wrap(err, "cannot update server") - } - - if err := tx.Commit(); err != nil { - return err - } - - return nil -} - -func appendODSetClauses(q *orm.Query) (*orm.Query, error) { - return q.Set("rank_att = EXCLUDED.rank_att"). - Set("score_att = EXCLUDED.score_att"). - Set("rank_def = EXCLUDED.rank_def"). - Set("score_def = EXCLUDED.score_def"). - Set("rank_sup = EXCLUDED.rank_sup"). - Set("score_sup = EXCLUDED.score_sup"). - Set("rank_total = EXCLUDED.rank_total"). - Set("score_total = EXCLUDED.score_total"), - nil -} From 30355785bc89610c16a566c2b2fed1b94f4a6257 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 18:37:55 +0200 Subject: [PATCH 09/31] don't wrap cron functions into utils.TrackExecutionTime --- cron/cron.go | 20 ++++++-------------- main.go | 4 ++-- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/cron/cron.go b/cron/cron.go index aae5120..21fd304 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -3,12 +3,10 @@ package cron import ( "fmt" - "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/shared/utils" - "github.com/go-pg/pg/v10" "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" + "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/cron/cron/queue" ) @@ -44,19 +42,13 @@ func Attach(c *cron.Cron, cfg Config) error { return err } - vacuumDatabase := utils.TrackExecutionTime(log, h.vacuumDatabase, "vacuumDatabase") - updateServerEnnoblements := utils.TrackExecutionTime(log, h.updateServerEnnoblements, "updateServerEnnoblements") var updateHistoryFuncs []func() var updateStatsFuncs []func() for _, version := range versions { - updateHistory := utils.TrackExecutionTime(log, - createFnWithTimezone(version.Timezone, h.updateHistory), - fmt.Sprintf("%s: updateHistory", version.Timezone)) + updateHistory := createFnWithTimezone(version.Timezone, h.updateHistory) updateHistoryFuncs = append(updateHistoryFuncs, updateHistory) - updateStats := utils.TrackExecutionTime(log, - createFnWithTimezone(version.Timezone, h.updateStats), - fmt.Sprintf("%s: updateStats", version.Timezone)) + updateStats := createFnWithTimezone(version.Timezone, h.updateStats) updateStatsFuncs = append(updateStatsFuncs, updateStats) if _, err := c.AddFunc(fmt.Sprintf("CRON_TZ=%s 30 1 * * *", version.Timezone), updateHistory); err != nil { @@ -69,15 +61,15 @@ func Attach(c *cron.Cron, cfg Config) error { if _, err := c.AddFunc("0 * * * *", h.updateServerData); err != nil { return err } - if _, err := c.AddFunc("20 1 * * *", vacuumDatabase); err != nil { + if _, err := c.AddFunc("20 1 * * *", h.vacuumDatabase); err != nil { return err } - if _, err := c.AddFunc("@every 1m", updateServerEnnoblements); err != nil { + if _, err := c.AddFunc("@every 1m", h.updateServerEnnoblements); err != nil { return err } if cfg.RunOnStartup { go func() { - h.updateServerData() + //h.updateServerData() //vacuumDatabase() //for _, fn := range updateHistoryFuncs { // go fn() diff --git a/main.go b/main.go index dc57079..d16cf96 100644 --- a/main.go +++ b/main.go @@ -35,13 +35,13 @@ func init() { func main() { client, err := initializeRedis() if err != nil { - logrus.Fatal(errors.Wrap(err, "Error establishing a redis connection")) + logrus.Fatal(errors.Wrap(err, "Couldn't connect to Redis")) } defer client.Close() conn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")}) if err != nil { - logrus.Fatal(errors.Wrap(err, "Error establishing a database connection")) + logrus.Fatal(errors.Wrap(err, "Couldn't connect to the db")) } defer conn.Close() logrus.Info("Connection with the database has been established") From f058facc321421e5c5ae4e7b4fd61dc2137764aa Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 18:46:31 +0200 Subject: [PATCH 10/31] add taskVacuum --- cron/tasks/task_vacuum.go | 31 +++++++++++++++++++++++++++++++ cron/tasks/tasks.go | 7 +++++++ 2 files changed, 38 insertions(+) create mode 100644 cron/tasks/task_vacuum.go diff --git a/cron/tasks/task_vacuum.go b/cron/tasks/task_vacuum.go new file mode 100644 index 0000000..d83d8bb --- /dev/null +++ b/cron/tasks/task_vacuum.go @@ -0,0 +1,31 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskVacuum struct { + *task +} + +func (t *taskVacuum) execute() error { + var servers []*models.Server + err := t.db. + Model(&servers). + Select() + if err != nil { + err = errors.Wrap(err, "taskVacuum.execute") + log.Errorln(err) + return err + } + log.Infof("Start database vacuuming...") + for _, server := range servers { + s := server + t.queue.Add(queue.MainQueue, Get(TaskNameVacuumServerDB).WithArgs(context.Background(), s)) + } + return nil +} diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 2b79c55..ebb46cb 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -13,6 +13,8 @@ const ( TaskNameLoadVersionsAndUpdateServerData = "loadVersions" TaskNameLoadServersAndUpdateData = "loadServers" TaskNameUpdateServerData = "updateServerData" + TaskNameVacuum = "vacuum" + TaskNameVacuumServerDB = "vacuumServerDB" defaultRetryLimit = 3 ) @@ -47,6 +49,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskUpdateServerData{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskNameVacuum, + RetryLimit: defaultRetryLimit, + Handler: (&taskVacuum{t}).execute, + }) return nil } From 1598b833bbd52851bb19baeff15bff3912c6c33c Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 18:59:16 +0200 Subject: [PATCH 11/31] add taskVacuumServerDB --- cron/cron.go | 2 +- cron/handler.go | 34 +--------------------------------- cron/tasks/tasks.go | 5 +++++ 3 files changed, 7 insertions(+), 34 deletions(-) diff --git a/cron/cron.go b/cron/cron.go index 21fd304..c3e3ec5 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -70,7 +70,7 @@ func Attach(c *cron.Cron, cfg Config) error { if cfg.RunOnStartup { go func() { //h.updateServerData() - //vacuumDatabase() + h.vacuumDatabase() //for _, fn := range updateHistoryFuncs { // go fn() //} diff --git a/cron/handler.go b/cron/handler.go index f03aae8..e7789d2 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -178,37 +178,5 @@ func (h *handler) updateStats(location *time.Location) { } func (h *handler) vacuumDatabase() { - servers := []*models.Server{} - err := h.db. - Model(&servers). - Select() - if err != nil { - log.Errorln(errors.Wrap(err, "vacuumDatabase")) - return - } - - var wg sync.WaitGroup - - for _, server := range servers { - h.pool.waitForWorker() - wg.Add(1) - worker := &vacuumServerDBWorker{ - db: h.db.WithParam("SERVER", pg.Safe(server.Key)), - } - go func(server *models.Server, worker *vacuumServerDBWorker) { - defer func() { - h.pool.releaseWorker() - wg.Done() - }() - log := log.WithField("serverKey", server.Key) - log.Infof("vacuumDatabase: %s: vacuuming database", server.Key) - if err := worker.vacuum(); err != nil { - log.Errorln("vacuumDatabase:", errors.Wrap(err, server.Key)) - return - } - log.Infof("vacuumDatabase: %s: database vacuumed", server.Key) - }(server, worker) - } - - wg.Wait() + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background())) } diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index ebb46cb..fbe0262 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -54,6 +54,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskVacuum{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskNameVacuumServerDB, + RetryLimit: defaultRetryLimit, + Handler: (&taskVacuumServerDB{t}).execute, + }) return nil } From 6994607e0c6600c7be8a2b744ea1ff6b6688afab Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:06:17 +0200 Subject: [PATCH 12/31] add taskUpdateEnnoblements --- cron/cron.go | 4 +- cron/handler.go | 42 ++--------------- cron/tasks/task_update_ennoblements.go | 38 ++++++++++++++++ .../task_vacuum_server_db.go} | 45 ++++++++++++++++--- cron/tasks/tasks.go | 11 ++++- 5 files changed, 90 insertions(+), 50 deletions(-) create mode 100644 cron/tasks/task_update_ennoblements.go rename cron/{vacuum_server_db_worker.go => tasks/task_vacuum_server_db.go} (58%) diff --git a/cron/cron.go b/cron/cron.go index c3e3ec5..ee4d901 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -64,13 +64,13 @@ func Attach(c *cron.Cron, cfg Config) error { if _, err := c.AddFunc("20 1 * * *", h.vacuumDatabase); err != nil { return err } - if _, err := c.AddFunc("@every 1m", h.updateServerEnnoblements); err != nil { + if _, err := c.AddFunc("@every 1m", h.updateEnnoblements); err != nil { return err } if cfg.RunOnStartup { go func() { //h.updateServerData() - h.vacuumDatabase() + //h.vacuumDatabase() //for _, fn := range updateHistoryFuncs { // go fn() //} diff --git a/cron/handler.go b/cron/handler.go index e7789d2..0025070 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -2,16 +2,13 @@ package cron import ( "context" - "fmt" "runtime" "sync" "time" - "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/shared/tw/dataloader" - "github.com/go-pg/pg/v10" "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/cron/cron/queue" "github.com/tribalwarshelp/cron/cron/tasks" @@ -40,41 +37,8 @@ func (h *handler) updateServerData() { h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) } -func (h *handler) updateServerEnnoblements() { - servers := []*models.Server{} - if err := h.db.Model(&servers).Relation("Version").Where("status = ?", models.ServerStatusOpen).Select(); err != nil { - log.Error(errors.Wrap(err, "updateServerEnnoblements: cannot load ennoblements")) - } - log. - WithField("numberOfServers", len(servers)). - Info("updateServerEnnoblements: servers loaded") - - var wg sync.WaitGroup - pool := newPool(h.maxConcurrentWorkers) - for _, server := range servers { - pool.waitForWorker() - wg.Add(1) - sh := &updateServerEnnoblementsWorker{ - db: h.db.WithParam("SERVER", pg.Safe(server.Key)), - server: server, - dataloader: dataloader.New(&dataloader.Config{ - BaseURL: fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), - }), - } - go func(worker *updateServerEnnoblementsWorker, server *models.Server) { - defer func() { - pool.releaseWorker() - wg.Done() - }() - log := log.WithField("serverKey", server.Key) - err := sh.update() - if err != nil { - log.Errorln("updateServerEnnoblements:", errors.Wrap(err, server.Key)) - return - } - }(sh, server) - } - wg.Wait() +func (h *handler) updateEnnoblements() { + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) } func (h *handler) updateHistory(location *time.Location) { diff --git a/cron/tasks/task_update_ennoblements.go b/cron/tasks/task_update_ennoblements.go new file mode 100644 index 0000000..4f212a6 --- /dev/null +++ b/cron/tasks/task_update_ennoblements.go @@ -0,0 +1,38 @@ +package tasks + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskUpdateEnnoblements struct { + *task +} + +func (t *taskUpdateEnnoblements) execute() error { + var servers []*models.Server + err := t.db. + Model(&servers). + Relation("Version"). + Where("status = ?", models.ServerStatusOpen). + Select() + if err != nil { + err = errors.Wrap(err, "taskUpdateEnnoblements.execute") + log.Errorln(err) + return err + } + log.Debug("Updating ennoblements...") + for _, server := range servers { + s := server + t.queue.Add( + queue.MainQueue, + Get(TaskNameVacuumServerDB). + WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s), + ) + } + return nil +} diff --git a/cron/vacuum_server_db_worker.go b/cron/tasks/task_vacuum_server_db.go similarity index 58% rename from cron/vacuum_server_db_worker.go rename to cron/tasks/task_vacuum_server_db.go index 07588ab..dbef830 100644 --- a/cron/vacuum_server_db_worker.go +++ b/cron/tasks/task_vacuum_server_db.go @@ -1,17 +1,48 @@ -package cron +package tasks import ( - "time" - "github.com/go-pg/pg/v10" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" + "time" ) const ( day = 24 * time.Hour ) +type taskVacuumServerDB struct { + *task +} + +func (t *taskVacuumServerDB) execute(server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + entry := log.WithField("key", server.Key) + entry.Infof("%s: vacumming the database...", server.Key) + err := (&vacuumServerDBWorker{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + }).vacuum() + if err != nil { + err = errors.Wrap(err, "taskVacuumServerDB.execute") + entry.Error(err) + return err + } + entry.Infof("%s: the database has been vacummed", server.Key) + + return nil +} + +func (t *taskVacuumServerDB) validatePayload(server *models.Server) error { + if server == nil { + return errors.Errorf("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + type vacuumServerDBWorker struct { db *pg.DB } @@ -31,7 +62,7 @@ func (w *vacuumServerDBWorker) vacuum() error { Where("player_id IN (Select id FROM players) OR player_history.create_date < ?", time.Now().Add(-1*day*180)). Delete() if err != nil { - return errors.Wrap(err, "cannot delete old player history records") + return errors.Wrap(err, "couldn't delete the old player history records") } _, err = tx.Model(&models.TribeHistory{}). @@ -39,7 +70,7 @@ func (w *vacuumServerDBWorker) vacuum() error { Where("tribe_id IN (Select id FROM tribes) OR tribe_history.create_date < ?", time.Now().Add(-1*day*180)). Delete() if err != nil { - return errors.Wrap(err, "cannot delete old tribe history records") + return errors.Wrap(err, "couldn't delete the old tribe history records") } _, err = tx.Model(&models.DailyPlayerStats{}). @@ -47,7 +78,7 @@ func (w *vacuumServerDBWorker) vacuum() error { Where("player_id IN (Select id FROM players) OR daily_player_stats.create_date < ?", time.Now().Add(-1*day*180)). Delete() if err != nil { - return errors.Wrap(err, "cannot delete old player stats records") + return errors.Wrap(err, "couldn't delete the old player stats records") } _, err = tx.Model(&models.DailyTribeStats{}). @@ -55,7 +86,7 @@ func (w *vacuumServerDBWorker) vacuum() error { Where("tribe_id IN (Select id FROM tribes) OR daily_tribe_stats.create_date < ?", time.Now().Add(-1*day*180)). Delete() if err != nil { - return errors.Wrap(err, "cannot delete old tribe stats records") + return errors.Wrap(err, "couldn't delete the old tribe stats records") } return tx.Commit() diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index fbe0262..17783c5 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -10,11 +10,13 @@ import ( ) const ( - TaskNameLoadVersionsAndUpdateServerData = "loadVersions" - TaskNameLoadServersAndUpdateData = "loadServers" + TaskNameLoadVersionsAndUpdateServerData = "loadVersionsAndUpdateServerData" + TaskNameLoadServersAndUpdateData = "loadServersAndUpdateData" TaskNameUpdateServerData = "updateServerData" TaskNameVacuum = "vacuum" TaskNameVacuumServerDB = "vacuumServerDB" + TaskUpdateEnnoblements = "updateEnnoblements" + TaskServerUpdateEnnoblements = "serverUpdateEnnoblements" defaultRetryLimit = 3 ) @@ -59,6 +61,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskVacuumServerDB{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateEnnoblements, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateEnnoblements{t}).execute, + }) return nil } From cc79a33e00d5c6de10b4ac6470c19fe79c9986cb Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:20:38 +0200 Subject: [PATCH 13/31] add taskUpdateServerEnnoblements --- cron/tasks/helpers.go | 7 ++ cron/tasks/task_update_ennoblements.go | 4 +- cron/tasks/task_update_server_data.go | 5 +- cron/tasks/task_update_server_ennoblements.go | 79 +++++++++++++++++++ cron/tasks/task_vacuum_server_db.go | 6 +- cron/tasks/tasks.go | 7 +- cron/update_server_ennoblements_worker.go | 44 ----------- 7 files changed, 98 insertions(+), 54 deletions(-) create mode 100644 cron/tasks/task_update_server_ennoblements.go delete mode 100644 cron/update_server_ennoblements_worker.go diff --git a/cron/tasks/helpers.go b/cron/tasks/helpers.go index d30542f..55a5963 100644 --- a/cron/tasks/helpers.go +++ b/cron/tasks/helpers.go @@ -1,6 +1,7 @@ package tasks import ( + "net/http" "time" "github.com/tribalwarshelp/shared/models" @@ -28,6 +29,12 @@ func calcPlayerDailyGrowth(diffInDays, points int) int { return 0 } +func newHTTPClient() *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + } +} + type tribesSearchableByID struct { tribes []*models.Tribe } diff --git a/cron/tasks/task_update_ennoblements.go b/cron/tasks/task_update_ennoblements.go index 4f212a6..4452d24 100644 --- a/cron/tasks/task_update_ennoblements.go +++ b/cron/tasks/task_update_ennoblements.go @@ -25,12 +25,12 @@ func (t *taskUpdateEnnoblements) execute() error { log.Errorln(err) return err } - log.Debug("Updating ennoblements...") + log.WithField("numberOfServers", len(servers)).Info("Update of the ennoblements has started...") for _, server := range servers { s := server t.queue.Add( queue.MainQueue, - Get(TaskNameVacuumServerDB). + Get(TaskUpdateServerEnnoblements). WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s), ) } diff --git a/cron/tasks/task_update_server_data.go b/cron/tasks/task_update_server_data.go index 427307a..107a73f 100644 --- a/cron/tasks/task_update_server_data.go +++ b/cron/tasks/task_update_server_data.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/shared/tw/dataloader" - "net/http" "time" ) @@ -26,9 +25,7 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error db: t.db.WithParam("SERVER", pg.Safe(server.Key)), dataloader: dataloader.New(&dataloader.Config{ BaseURL: url, - Client: &http.Client{ - Timeout: 10 * time.Second, - }, + Client: newHTTPClient(), }), server: server, }).update() diff --git a/cron/tasks/task_update_server_ennoblements.go b/cron/tasks/task_update_server_ennoblements.go new file mode 100644 index 0000000..2479341 --- /dev/null +++ b/cron/tasks/task_update_server_ennoblements.go @@ -0,0 +1,79 @@ +package tasks + +import ( + "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + "github.com/tribalwarshelp/shared/tw/dataloader" +) + +type taskUpdateServerEnnoblements struct { + *task +} + +func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + entry := log.WithField("key", server.Key) + entry.Debugf("%s: update of the ennoblements has started...", server.Key) + err := (&workerUpdateServerEnnoblements{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + dataloader: dataloader.New(&dataloader.Config{ + BaseURL: url, + Client: newHTTPClient(), + }), + }).update() + if err != nil { + err = errors.Wrap(err, "taskUpdateServerEnnoblements.execute") + entry.Error(err) + return err + } + entry.Debugf("%s: ennoblements has been updated", server.Key) + + return nil +} + +func (t *taskUpdateServerEnnoblements) validatePayload(server *models.Server) error { + if server == nil { + return errors.Errorf("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + +type workerUpdateServerEnnoblements struct { + db *pg.DB + dataloader dataloader.DataLoader +} + +func (w *workerUpdateServerEnnoblements) loadEnnoblements() ([]*models.Ennoblement, error) { + lastEnnoblement := &models.Ennoblement{} + if err := w.db. + Model(lastEnnoblement). + Limit(1). + Order("ennobled_at DESC"). + Select(); err != nil && err != pg.ErrNoRows { + return nil, errors.Wrapf(err, "couldn't load last ennoblement") + } + + return w.dataloader.LoadEnnoblements(&dataloader.LoadEnnoblementsConfig{ + EnnobledAtGT: lastEnnoblement.EnnobledAt, + }) +} + +func (w *workerUpdateServerEnnoblements) update() error { + ennoblements, err := w.loadEnnoblements() + if err != nil { + return err + } + + if len(ennoblements) > 0 { + if _, err := w.db.Model(&ennoblements).Insert(); err != nil { + return errors.Wrap(err, "couldn't insert ennoblements") + } + } + + return nil +} diff --git a/cron/tasks/task_vacuum_server_db.go b/cron/tasks/task_vacuum_server_db.go index dbef830..1a04e88 100644 --- a/cron/tasks/task_vacuum_server_db.go +++ b/cron/tasks/task_vacuum_server_db.go @@ -22,7 +22,7 @@ func (t *taskVacuumServerDB) execute(server *models.Server) error { } entry := log.WithField("key", server.Key) entry.Infof("%s: vacumming the database...", server.Key) - err := (&vacuumServerDBWorker{ + err := (&workerVacuumServerDB{ db: t.db.WithParam("SERVER", pg.Safe(server.Key)), }).vacuum() if err != nil { @@ -43,11 +43,11 @@ func (t *taskVacuumServerDB) validatePayload(server *models.Server) error { return nil } -type vacuumServerDBWorker struct { +type workerVacuumServerDB struct { db *pg.DB } -func (w *vacuumServerDBWorker) vacuum() error { +func (w *workerVacuumServerDB) vacuum() error { tx, err := w.db.Begin() if err != nil { return err diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 17783c5..234684e 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -16,7 +16,7 @@ const ( TaskNameVacuum = "vacuum" TaskNameVacuumServerDB = "vacuumServerDB" TaskUpdateEnnoblements = "updateEnnoblements" - TaskServerUpdateEnnoblements = "serverUpdateEnnoblements" + TaskUpdateServerEnnoblements = "updateServerEnnoblements" defaultRetryLimit = 3 ) @@ -66,6 +66,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskUpdateEnnoblements{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateServerEnnoblements, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateServerEnnoblements{t}).execute, + }) return nil } diff --git a/cron/update_server_ennoblements_worker.go b/cron/update_server_ennoblements_worker.go deleted file mode 100644 index 32984da..0000000 --- a/cron/update_server_ennoblements_worker.go +++ /dev/null @@ -1,44 +0,0 @@ -package cron - -import ( - "github.com/go-pg/pg/v10" - "github.com/pkg/errors" - "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/shared/tw/dataloader" -) - -type updateServerEnnoblementsWorker struct { - db *pg.DB - dataloader dataloader.DataLoader - server *models.Server -} - -func (w *updateServerEnnoblementsWorker) loadEnnoblements() ([]*models.Ennoblement, error) { - lastEnnoblement := &models.Ennoblement{} - if err := w.db. - Model(lastEnnoblement). - Limit(1). - Order("ennobled_at DESC"). - Select(); err != nil && err != pg.ErrNoRows { - return nil, errors.Wrapf(err, "cannot load last ennoblement") - } - - return w.dataloader.LoadEnnoblements(&dataloader.LoadEnnoblementsConfig{ - EnnobledAtGT: lastEnnoblement.EnnobledAt, - }) -} - -func (w *updateServerEnnoblementsWorker) update() error { - ennoblements, err := w.loadEnnoblements() - if err != nil { - return err - } - - if len(ennoblements) > 0 { - if _, err := w.db.Model(&ennoblements).Insert(); err != nil { - return errors.Wrap(err, "cannot insert ennoblements") - } - } - - return nil -} From 62f272217735e78748b2a58474b25082792a4215 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:21:13 +0200 Subject: [PATCH 14/31] remove sql_statements from the cron package --- cron/sql_statements.go | 306 ----------------------------------------- 1 file changed, 306 deletions(-) delete mode 100644 cron/sql_statements.go diff --git a/cron/sql_statements.go b/cron/sql_statements.go deleted file mode 100644 index b8237c0..0000000 --- a/cron/sql_statements.go +++ /dev/null @@ -1,306 +0,0 @@ -package cron - -const ( - allSpecialServersPGInsertStatements = ` - INSERT INTO public.special_servers (version_code, key) VALUES ('pl', 'pls1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('en', 'ens1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('uk', 'uks1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('uk', 'master') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('it', 'its1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('hu', 'hus1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('fr', 'frs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('us', 'uss1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('nl', 'nls1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('es', 'ess1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('ro', 'ros1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('gr', 'grs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('br', 'brs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('tr', 'trs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('cs', 'css1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('de', 'des1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('ru', 'rus1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('ch', 'chs1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('pt', 'pts1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - INSERT INTO public.special_servers (version_code, key) VALUES ('sk', 'sks1') ON CONFLICT ON CONSTRAINT special_servers_version_code_key_key DO NOTHING; - ` - allVersionsPGInsertStatements = ` - INSERT INTO public.versions (code, name, host, timezone) VALUES ('pl', 'Polska', 'plemiona.pl', 'Europe/Warsaw') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('uk', 'United Kingdom', 'tribalwars.co.uk', 'Europe/London') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('hu', 'Hungary', 'klanhaboru.hu', 'Europe/Budapest') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('it', 'Italy', 'tribals.it', 'Europe/Rome') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('fr', 'France', 'guerretribale.fr', 'Europe/Paris') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('us', 'United States', 'tribalwars.us', 'America/New_York') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('nl', 'The Netherlands', 'tribalwars.nl', 'Europe/Amsterdam') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('es', 'Spain', 'guerrastribales.es', 'Europe/Madrid') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('ro', 'Romania', 'triburile.ro', 'Europe/Bucharest') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('gr', 'Greece', 'fyletikesmaxes.gr', 'Europe/Athens') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('br', 'Brazil', 'tribalwars.com.br', 'America/Sao_Paulo') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('tr', 'Turkey', 'klanlar.org', 'Europe/Istanbul') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('cs', 'Czech Republic', 'divokekmeny.cz', 'Europe/Prague') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('ru', 'Russia', 'voyna-plemyon.ru', 'Europe/Moscow') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('ch', 'Switerzland', 'staemme.ch', 'Europe/Zurich') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('pt', 'Portugal', 'tribalwars.com.pt', 'Europe/Lisbon') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('en', 'International', 'tribalwars.net', 'Europe/London') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('de', 'Germany', 'die-staemme.de', 'Europe/Berlin') ON CONFLICT (code) DO NOTHING; - INSERT INTO public.versions (code, name, host, timezone) VALUES ('sk', 'Slovakia', 'divoke-kmene.sk', 'Europe/Bratislava') ON CONFLICT (code) DO NOTHING; - ` - - pgDropSchemaFunctions = ` - DO - $do$ - DECLARE - funcTableRecord RECORD; - BEGIN - FOR funcTableRecord IN SELECT routine_schema, routine_name from information_schema.routines where routine_type = 'FUNCTION' AND specific_schema = '?0' LOOP - EXECUTE 'DROP FUNCTION IF EXISTS ' || funcTableRecord.routine_schema || '.' || funcTableRecord.routine_name || ' CASCADE;'; - END LOOP; - END - $do$; - ` - - pgFunctions = ` - CREATE OR REPLACE FUNCTION check_daily_growth() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.exists = false THEN - NEW.daily_growth = 0; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION check_existence() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.exists = false AND OLD.exists = true THEN - NEW.deleted_at = now(); - END IF; - IF NEW.exists = true THEN - NEW.deleted_at = null; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION check_dominance() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.exists = false THEN - NEW.dominance = 0; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION update_most_points_most_villages_best_rank_last_activity() - RETURNS trigger AS - $BODY$ - BEGIN - IF TG_OP = 'INSERT' THEN - IF NEW.most_points IS null OR NEW.points > NEW.most_points THEN - NEW.most_points = NEW.points; - NEW.most_points_at = now(); - END IF; - IF NEW.most_villages IS null OR NEW.total_villages > NEW.most_villages THEN - NEW.most_villages = NEW.total_villages; - NEW.most_villages_at = now(); - END IF; - IF NEW.best_rank IS null OR NEW.rank < NEW.best_rank OR NEW.best_rank = 0 THEN - NEW.best_rank = NEW.rank; - NEW.best_rank_at = now(); - END IF; - END IF; - - IF TG_OP = 'UPDATE' THEN - IF NEW.most_points IS null OR NEW.points > OLD.most_points THEN - NEW.most_points = NEW.points; - NEW.most_points_at = now(); - END IF; - IF NEW.most_villages IS null OR NEW.total_villages > OLD.most_villages THEN - NEW.most_villages = NEW.total_villages; - NEW.most_villages_at = now(); - END IF; - IF NEW.best_rank IS null OR NEW.rank < OLD.best_rank OR OLD.best_rank = 0 THEN - NEW.best_rank = NEW.rank; - NEW.best_rank_at = now(); - END IF; - if TG_TABLE_NAME = 'players' THEN - IF NEW.points > OLD.points OR NEW.score_att > OLD.score_att THEN - NEW.last_activity_at = now(); - END IF; - END IF; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql; - ` - - serverPGFunctions = ` - CREATE OR REPLACE FUNCTION ?0.log_tribe_change() - RETURNS trigger AS - $BODY$ - BEGIN - IF TG_OP = 'INSERT' THEN - IF NEW.tribe_id <> 0 THEN - INSERT INTO ?0.tribe_changes(player_id,old_tribe_id,new_tribe_id,created_at) - VALUES(NEW.id,0,NEW.tribe_id,now()); - END IF; - END IF; - - IF TG_OP = 'UPDATE' THEN - IF NEW.tribe_id <> OLD.tribe_id THEN - INSERT INTO ?0.tribe_changes(player_id,old_tribe_id,new_tribe_id,created_at) - VALUES(OLD.id,OLD.tribe_id,NEW.tribe_id,now()); - END IF; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql VOLATILE; - - CREATE OR REPLACE FUNCTION ?0.log_player_name_change() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.name <> OLD.name AND old.exists = true THEN - INSERT INTO player_name_changes(version_code,player_id,old_name,new_name,change_date) - VALUES(?1,NEW.id,OLD.name,NEW.name,CURRENT_DATE) - ON CONFLICT DO NOTHING; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql VOLATILE; - - CREATE OR REPLACE FUNCTION ?0.get_old_and_new_owner_tribe_id() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.old_owner_id <> 0 THEN - SELECT tribe_id INTO NEW.old_owner_tribe_id - FROM ?0.players - WHERE id = NEW.old_owner_id; - END IF; - IF NEW.old_owner_tribe_id IS NULL THEN - NEW.old_owner_tribe_id = 0; - END IF; - IF NEW.new_owner_id <> 0 THEN - SELECT tribe_id INTO NEW.new_owner_tribe_id - FROM ?0.players - WHERE id = NEW.new_owner_id; - END IF; - IF NEW.new_owner_tribe_id IS NULL THEN - NEW.new_owner_tribe_id = 0; - END IF; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql VOLATILE; - - CREATE OR REPLACE FUNCTION ?0.insert_into_player_to_servers() - RETURNS trigger AS - $BODY$ - BEGIN - INSERT INTO player_to_servers(server_key,player_id) - VALUES('?0', NEW.id) - ON CONFLICT DO NOTHING; - - RETURN NEW; - END; - $BODY$ - LANGUAGE plpgsql; - ` - serverPGTriggers = ` - CREATE TRIGGER ?0_log_tribe_change_on_insert - AFTER INSERT - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE ?0.log_tribe_change(); - - CREATE TRIGGER ?0_log_tribe_change_on_update - AFTER UPDATE - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE ?0.log_tribe_change(); - - CREATE TRIGGER ?0_name_change - AFTER UPDATE - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE ?0.log_player_name_change(); - - CREATE TRIGGER ?0_check_daily_growth - BEFORE UPDATE - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE check_daily_growth(); - - CREATE TRIGGER ?0_check_player_existence - BEFORE UPDATE - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE check_existence(); - - CREATE TRIGGER ?0_check_tribe_existence - BEFORE UPDATE - ON ?0.tribes - FOR EACH ROW - EXECUTE PROCEDURE check_existence(); - - CREATE TRIGGER ?0_check_dominance - BEFORE UPDATE - ON ?0.tribes - FOR EACH ROW - EXECUTE PROCEDURE check_dominance(); - - CREATE TRIGGER ?0_update_ennoblement_old_and_new_owner_tribe_id - BEFORE INSERT - ON ?0.ennoblements - FOR EACH ROW - EXECUTE PROCEDURE ?0.get_old_and_new_owner_tribe_id(); - - CREATE TRIGGER ?0_insert_into_player_to_servers - AFTER INSERT - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE ?0.insert_into_player_to_servers(); - - CREATE TRIGGER ?0_update_most_points_most_villages_best_rank_last_activity - BEFORE INSERT OR UPDATE - ON ?0.players - FOR EACH ROW - EXECUTE PROCEDURE update_most_points_most_villages_best_rank_last_activity(); - - CREATE TRIGGER ?0_update_most_points_most_villages_best_rank_last_activity - BEFORE INSERT OR UPDATE - ON ?0.tribes - FOR EACH ROW - EXECUTE PROCEDURE update_most_points_most_villages_best_rank_last_activity(); - ` - - serverPGDefaultValues = ` - ALTER TABLE ?0.daily_player_stats ALTER COLUMN create_date set default CURRENT_DATE; - ALTER TABLE ?0.daily_tribe_stats ALTER COLUMN create_date set default CURRENT_DATE; - ALTER TABLE ?0.player_history ALTER COLUMN create_date set default CURRENT_DATE; - ALTER TABLE ?0.tribe_history ALTER COLUMN create_date set default CURRENT_DATE; - ALTER TABLE ?0.stats ALTER COLUMN create_date set default CURRENT_DATE; - ` - - pgDefaultValues = ` - ALTER TABLE player_name_changes ALTER COLUMN change_date set default CURRENT_DATE; - ` -) From 4f94e2f920eb55d19a8962d1c912ee7742178f3f Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:42:03 +0200 Subject: [PATCH 15/31] add taskUpdateStats and taskUpdateHistory --- cron/cron.go | 1 + cron/handler.go | 109 ++----------------------- cron/helpers.go | 12 +-- cron/tasks/task.go | 21 ++++- cron/tasks/task_update_ennoblements.go | 2 +- cron/tasks/task_update_history.go | 49 +++++++++++ cron/tasks/task_update_stats.go | 49 +++++++++++ cron/tasks/tasks.go | 14 ++++ 8 files changed, 141 insertions(+), 116 deletions(-) create mode 100644 cron/tasks/task_update_history.go create mode 100644 cron/tasks/task_update_stats.go diff --git a/cron/cron.go b/cron/cron.go index ee4d901..25be92e 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -69,6 +69,7 @@ func Attach(c *cron.Cron, cfg Config) error { } if cfg.RunOnStartup { go func() { + h.updateEnnoblements() //h.updateServerData() //h.vacuumDatabase() //for _, fn := range updateHistoryFuncs { diff --git a/cron/handler.go b/cron/handler.go index 0025070..79f055f 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -2,13 +2,8 @@ package cron import ( "context" - "runtime" - "sync" - "time" - "github.com/go-pg/pg/v10" - "github.com/pkg/errors" - "github.com/tribalwarshelp/shared/models" + "runtime" "github.com/tribalwarshelp/cron/cron/queue" "github.com/tribalwarshelp/cron/cron/tasks" @@ -38,107 +33,15 @@ func (h *handler) updateServerData() { } func (h *handler) updateEnnoblements() { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) + h.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) } -func (h *handler) updateHistory(location *time.Location) { - servers := []*models.Server{} - log := log.WithField("timezone", location.String()) - year, month, day := time.Now().In(location).Date() - t := time.Date(year, month, day, 1, 30, 0, 0, location) - err := h.db. - Model(&servers). - Where( - "status = ? AND (history_updated_at IS NULL OR history_updated_at < ?) AND timezone = ?", - models.ServerStatusOpen, - t, - location.String(), - ). - Relation("Version"). - Select() - if err != nil { - log.Errorln(errors.Wrap(err, "updateHistory")) - return - } - log. - WithField("numberOfServers", len(servers)). - Info("updateHistory: servers loaded") - - var wg sync.WaitGroup - - for _, server := range servers { - h.pool.waitForWorker() - wg.Add(1) - worker := &updateServerHistoryWorker{ - db: h.db.WithParam("SERVER", pg.Safe(server.Key)), - server: server, - location: location, - } - go func(server *models.Server, worker *updateServerHistoryWorker) { - defer func() { - h.pool.releaseWorker() - wg.Done() - }() - log := log.WithField("serverKey", server.Key) - log.Infof("updateHistory: %s: updating history", server.Key) - if err := worker.update(); err != nil { - log.Errorln("updateHistory:", errors.Wrap(err, server.Key)) - return - } - log.Infof("updateHistory: %s: history updated", server.Key) - }(server, worker) - } - - wg.Wait() +func (h *handler) updateHistory(timezone string) { + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone)) } -func (h *handler) updateStats(location *time.Location) { - servers := []*models.Server{} - log := log.WithField("timezone", location.String()) - year, month, day := time.Now().In(location).Date() - t := time.Date(year, month, day, 1, 45, 0, 0, location) - err := h.db. - Model(&servers). - Where( - "status = ? AND (stats_updated_at IS NULL OR stats_updated_at < ?) AND timezone = ?", - models.ServerStatusOpen, - t, - location.String(), - ). - Relation("Version"). - Select() - if err != nil { - log.Errorf(errors.Wrap(err, "updateServerStats").Error()) - return - } - log.WithField("numberOfServers", len(servers)).Info("updateServerStats: servers loaded") - - var wg sync.WaitGroup - - for _, server := range servers { - h.pool.waitForWorker() - wg.Add(1) - worker := &updateServerStatsWorker{ - db: h.db.WithParam("SERVER", pg.Safe(server.Key)), - server: server, - location: location, - } - go func(server *models.Server, worker *updateServerStatsWorker) { - defer func() { - h.pool.releaseWorker() - wg.Done() - }() - log := log.WithField("serverKey", server.Key) - log.Infof("updateServerStats: %s: updating stats", server.Key) - if err := worker.update(); err != nil { - log.Errorln("updateServerStats:", errors.Wrap(err, server.Key)) - return - } - log.Infof("updateServerStats: %s: stats updated", server.Key) - }(server, worker) - } - - wg.Wait() +func (h *handler) updateStats(timezone string) { + h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone)) } func (h *handler) vacuumDatabase() { diff --git a/cron/helpers.go b/cron/helpers.go index a700335..8512a35 100644 --- a/cron/helpers.go +++ b/cron/helpers.go @@ -1,15 +1,7 @@ package cron -import ( - "time" -) - -func createFnWithTimezone(timezone string, fn func(location *time.Location)) func() { - tz, err := time.LoadLocation(timezone) - if err != nil { - tz = time.UTC - } +func createFnWithTimezone(timezone string, fn func(timezone string)) func() { return func() { - fn(tz) + fn(timezone) } } diff --git a/cron/tasks/task.go b/cron/tasks/task.go index 03e6d84..86bc115 100644 --- a/cron/tasks/task.go +++ b/cron/tasks/task.go @@ -2,11 +2,28 @@ package tasks import ( "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "sync" + "time" "github.com/tribalwarshelp/cron/cron/queue" ) type task struct { - db *pg.DB - queue queue.Queue + db *pg.DB + queue queue.Queue + cachedLocations sync.Map +} + +func (t *task) loadLocation(timezone string) (*time.Location, error) { + val, ok := t.cachedLocations.Load(timezone) + if ok { + return val.(*time.Location), nil + } + location, err := time.LoadLocation(timezone) + if err != nil { + return nil, errors.Wrap(err, "task.loadLocation") + } + t.cachedLocations.Store(timezone, location) + return location, nil } diff --git a/cron/tasks/task_update_ennoblements.go b/cron/tasks/task_update_ennoblements.go index 4452d24..af3c3ea 100644 --- a/cron/tasks/task_update_ennoblements.go +++ b/cron/tasks/task_update_ennoblements.go @@ -29,7 +29,7 @@ func (t *taskUpdateEnnoblements) execute() error { for _, server := range servers { s := server t.queue.Add( - queue.MainQueue, + queue.EnnoblementsQueue, Get(TaskUpdateServerEnnoblements). WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s), ) diff --git a/cron/tasks/task_update_history.go b/cron/tasks/task_update_history.go new file mode 100644 index 0000000..84f4c4e --- /dev/null +++ b/cron/tasks/task_update_history.go @@ -0,0 +1,49 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + "time" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskUpdateHistory struct { + *task +} + +func (t *taskUpdateHistory) execute(timezone string) error { + location, err := t.loadLocation(timezone) + if err != nil { + err = errors.Wrap(err, "taskUpdateHistory.execute") + log.Error(err) + return err + } + year, month, day := time.Now().In(location).Date() + date := time.Date(year, month, day, 1, 30, 0, 0, location) + var servers []*models.Server + err = t.db. + Model(&servers). + Where( + "status = ? AND (history_updated_at IS NULL OR history_updated_at < ?) AND timezone = ?", + models.ServerStatusOpen, + date, + timezone, + ). + Relation("Version"). + Select() + if err != nil { + err = errors.Wrap(err, "taskUpdateHistory.execute") + log.Errorln(err) + return err + } + log. + WithField("numberOfServers", len(servers)). + Info("Update of the history has started") + for _, server := range servers { + s := server + t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), s)) + } + return nil +} diff --git a/cron/tasks/task_update_stats.go b/cron/tasks/task_update_stats.go new file mode 100644 index 0000000..17dd25c --- /dev/null +++ b/cron/tasks/task_update_stats.go @@ -0,0 +1,49 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + "time" + + "github.com/tribalwarshelp/cron/cron/queue" +) + +type taskUpdateStats struct { + *task +} + +func (t *taskUpdateStats) execute(timezone string) error { + location, err := t.loadLocation(timezone) + if err != nil { + err = errors.Wrap(err, "taskUpdateStats.execute") + log.Error(err) + return err + } + year, month, day := time.Now().In(location).Date() + date := time.Date(year, month, day, 1, 45, 0, 0, location) + var servers []*models.Server + err = t.db. + Model(&servers). + Where( + "status = ? AND (stats_updated_at IS NULL OR stats_updated_at < ?) AND timezone = ?", + models.ServerStatusOpen, + date, + location.String(), + ). + Relation("Version"). + Select() + if err != nil { + err = errors.Wrap(err, "taskUpdateStats.execute") + log.Errorln(err) + return err + } + log. + WithField("numberOfServers", len(servers)). + Info("Update of the stats has started") + for _, server := range servers { + s := server + t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), s)) + } + return nil +} diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 234684e..d97aab6 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -17,6 +17,10 @@ const ( TaskNameVacuumServerDB = "vacuumServerDB" TaskUpdateEnnoblements = "updateEnnoblements" TaskUpdateServerEnnoblements = "updateServerEnnoblements" + TaskUpdateHistory = "updateHistory" + TaskUpdateServerHistory = "updateServerHistory" + TaskUpdateStats = "updateStats" + TaskUpdateServerStats = "updateServerStats" defaultRetryLimit = 3 ) @@ -71,6 +75,16 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskUpdateServerEnnoblements{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateHistory, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateHistory{t}).execute, + }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateStats, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateStats{t}).execute, + }) return nil } From 7ec3669788f258017460580c564e64f3069e6fbf Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:45:09 +0200 Subject: [PATCH 16/31] cleanup --- cron/cron.go | 17 ++++------------- cron/handler.go | 19 +------------------ cron/pool.go | 22 ---------------------- 3 files changed, 5 insertions(+), 53 deletions(-) delete mode 100644 cron/pool.go diff --git a/cron/cron.go b/cron/cron.go index 25be92e..c13e7c5 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -5,19 +5,15 @@ import ( "github.com/go-pg/pg/v10" "github.com/robfig/cron/v3" - "github.com/sirupsen/logrus" "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/cron/cron/queue" ) -var log = logrus.WithField("package", "cron") - type Config struct { - DB *pg.DB - MaxConcurrentWorkers int - RunOnStartup bool - Queue queue.Queue + DB *pg.DB + RunOnStartup bool + Queue queue.Queue } func Attach(c *cron.Cron, cfg Config) error { @@ -29,12 +25,7 @@ func Attach(c *cron.Cron, cfg Config) error { } h := &handler{ - db: cfg.DB, - maxConcurrentWorkers: cfg.MaxConcurrentWorkers, - queue: cfg.Queue, - } - if err := h.init(); err != nil { - return err + queue: cfg.Queue, } var versions []*models.Version diff --git a/cron/handler.go b/cron/handler.go index 79f055f..d783b75 100644 --- a/cron/handler.go +++ b/cron/handler.go @@ -2,30 +2,13 @@ package cron import ( "context" - "github.com/go-pg/pg/v10" - "runtime" "github.com/tribalwarshelp/cron/cron/queue" "github.com/tribalwarshelp/cron/cron/tasks" ) type handler struct { - db *pg.DB - maxConcurrentWorkers int - pool *pool - queue queue.Queue -} - -func (h *handler) init() error { - if h.maxConcurrentWorkers <= 0 { - h.maxConcurrentWorkers = runtime.NumCPU() - } - - if h.pool == nil { - h.pool = newPool(h.maxConcurrentWorkers) - } - - return nil + queue queue.Queue } func (h *handler) updateServerData() { diff --git a/cron/pool.go b/cron/pool.go deleted file mode 100644 index b8534d3..0000000 --- a/cron/pool.go +++ /dev/null @@ -1,22 +0,0 @@ -package cron - -type pool struct { - workers chan bool -} - -func newPool(capacity int) *pool { - p := &pool{} - p.workers = make(chan bool, capacity) - for i := 0; i < capacity; i++ { - p.releaseWorker() - } - return p -} - -func (p *pool) releaseWorker() { - p.workers <- true -} - -func (p *pool) waitForWorker() bool { - return <-p.workers -} From 8f093978bd20b4c9db78e112794a90d69c1eebad Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 19:58:33 +0200 Subject: [PATCH 17/31] add taskUpdateServerHistory --- cron/cron.go | 7 +- cron/tasks/task_update_history.go | 9 +-- cron/tasks/task_update_server_ennoblements.go | 2 +- .../task_update_server_history.go} | 67 +++++++++++++++---- cron/tasks/task_update_stats.go | 2 +- cron/tasks/tasks.go | 5 ++ main.go | 7 +- 7 files changed, 71 insertions(+), 28 deletions(-) rename cron/{update_server_history_worker.go => tasks/task_update_server_history.go} (51%) diff --git a/cron/cron.go b/cron/cron.go index c13e7c5..04fabb7 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -60,12 +60,11 @@ func Attach(c *cron.Cron, cfg Config) error { } if cfg.RunOnStartup { go func() { - h.updateEnnoblements() //h.updateServerData() //h.vacuumDatabase() - //for _, fn := range updateHistoryFuncs { - // go fn() - //} + for _, fn := range updateHistoryFuncs { + go fn() + } //for _, fn := range updateStatsFuncs { // go fn() //} diff --git a/cron/tasks/task_update_history.go b/cron/tasks/task_update_history.go index 84f4c4e..02b2b32 100644 --- a/cron/tasks/task_update_history.go +++ b/cron/tasks/task_update_history.go @@ -14,10 +14,11 @@ type taskUpdateHistory struct { } func (t *taskUpdateHistory) execute(timezone string) error { + entry := log.WithField("timezone", timezone) location, err := t.loadLocation(timezone) if err != nil { err = errors.Wrap(err, "taskUpdateHistory.execute") - log.Error(err) + entry.Error(err) return err } year, month, day := time.Now().In(location).Date() @@ -35,15 +36,15 @@ func (t *taskUpdateHistory) execute(timezone string) error { Select() if err != nil { err = errors.Wrap(err, "taskUpdateHistory.execute") - log.Errorln(err) + entry.Errorln(err) return err } - log. + entry. WithField("numberOfServers", len(servers)). Info("Update of the history has started") for _, server := range servers { s := server - t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), s)) + t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), timezone, s)) } return nil } diff --git a/cron/tasks/task_update_server_ennoblements.go b/cron/tasks/task_update_server_ennoblements.go index 2479341..4a6e4f0 100644 --- a/cron/tasks/task_update_server_ennoblements.go +++ b/cron/tasks/task_update_server_ennoblements.go @@ -37,7 +37,7 @@ func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server func (t *taskUpdateServerEnnoblements) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil") + return errors.Errorf("taskUpdateServerEnnoblements.validatePayload: Expected *models.Server, got nil") } return nil diff --git a/cron/update_server_history_worker.go b/cron/tasks/task_update_server_history.go similarity index 51% rename from cron/update_server_history_worker.go rename to cron/tasks/task_update_server_history.go index 9bb9711..28a8859 100644 --- a/cron/update_server_history_worker.go +++ b/cron/tasks/task_update_server_history.go @@ -1,28 +1,67 @@ -package cron +package tasks import ( - "time" - "github.com/go-pg/pg/v10" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" + "time" ) -type updateServerHistoryWorker struct { +type taskUpdateServerHistory struct { + *task +} + +func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + location, err := t.loadLocation(timezone) + if err != nil { + err = errors.Wrap(err, "taskUpdateServerHistory.execute") + log.Error(err) + return err + } + entry := log.WithField("key", server.Key) + entry.Infof("%s: update of the history has started...", server.Key) + err = (&workerUpdateServerHistory{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + server: server, + location: location, + }).update() + if err != nil { + err = errors.Wrap(err, "taskUpdateServerHistory.execute") + entry.Error(err) + return err + } + entry.Infof("%s: history has been updated", server.Key) + + return nil +} + +func (t *taskUpdateServerHistory) validatePayload(server *models.Server) error { + if server == nil { + return errors.Errorf("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + +type workerUpdateServerHistory struct { db *pg.DB server *models.Server location *time.Location } -func (w *updateServerHistoryWorker) update() error { - players := []*models.Player{} +func (w *workerUpdateServerHistory) update() error { + var players []*models.Player if err := w.db.Model(&players).Where("exists = true").Select(); err != nil { - return errors.Wrap(err, "cannot load players") + return errors.Wrap(err, "couldn't load players") } now := time.Now().In(w.location) createDate := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - ph := []*models.PlayerHistory{} + var ph []*models.PlayerHistory for _, player := range players { ph = append(ph, &models.PlayerHistory{ OpponentsDefeated: player.OpponentsDefeated, @@ -35,11 +74,11 @@ func (w *updateServerHistoryWorker) update() error { }) } - tribes := []*models.Tribe{} + var tribes []*models.Tribe if err := w.db.Model(&tribes).Where("exists = true").Select(); err != nil { - return errors.Wrap(err, "cannot load tribes") + return errors.Wrap(err, "couldn't load tribes") } - th := []*models.TribeHistory{} + var th []*models.TribeHistory for _, tribe := range tribes { th = append(th, &models.TribeHistory{ OpponentsDefeated: tribe.OpponentsDefeated, @@ -62,13 +101,13 @@ func (w *updateServerHistoryWorker) update() error { if len(ph) > 0 { if _, err := w.db.Model(&ph).Insert(); err != nil { - return errors.Wrap(err, "cannot insert players history") + return errors.Wrap(err, "couldn't insert players history") } } if len(th) > 0 { if _, err := w.db.Model(&th).Insert(); err != nil { - return errors.Wrap(err, "cannot insert tribes history") + return errors.Wrap(err, "couldn't insert tribes history") } } @@ -77,7 +116,7 @@ func (w *updateServerHistoryWorker) update() error { WherePK(). Returning("*"). Update(); err != nil { - return errors.Wrap(err, "cannot update server") + return errors.Wrap(err, "couldn't update server") } diff --git a/cron/tasks/task_update_stats.go b/cron/tasks/task_update_stats.go index 17dd25c..7a251a3 100644 --- a/cron/tasks/task_update_stats.go +++ b/cron/tasks/task_update_stats.go @@ -43,7 +43,7 @@ func (t *taskUpdateStats) execute(timezone string) error { Info("Update of the stats has started") for _, server := range servers { s := server - t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), s)) + t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), timezone, s)) } return nil } diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index d97aab6..2d77f9c 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -80,6 +80,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskUpdateHistory{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateServerHistory, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateServerHistory{t}).execute, + }) taskq.RegisterTask(&taskq.TaskOptions{ Name: TaskUpdateStats, RetryLimit: defaultRetryLimit, diff --git a/main.go b/main.go index d16cf96..50b3cbd 100644 --- a/main.go +++ b/main.go @@ -65,10 +65,9 @@ func main() { cron.SkipIfStillRunning(cron.PrintfLogger(logrus.WithField("package", "cron"))), )) if err := twhelpcron.Attach(c, twhelpcron.Config{ - DB: conn, - MaxConcurrentWorkers: envutils.GetenvInt("MAX_CONCURRENT_WORKERS"), - RunOnStartup: envutils.GetenvBool("RUN_ON_STARTUP"), - Queue: queue, + DB: conn, + RunOnStartup: envutils.GetenvBool("RUN_ON_STARTUP"), + Queue: queue, }); err != nil { logrus.Fatal(err) } From 46ca716cd3c00d7382c49cf43f59ff6d8b8313b2 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 20:06:08 +0200 Subject: [PATCH 18/31] add taskUpdateServerStats --- cron/cron.go | 14 ++++++++++---- cron/helpers.go | 7 ------- cron/tasks/task_update_stats.go | 7 ++++--- cron/tasks/tasks.go | 5 +++++ 4 files changed, 19 insertions(+), 14 deletions(-) delete mode 100644 cron/helpers.go diff --git a/cron/cron.go b/cron/cron.go index 04fabb7..ec53077 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -62,14 +62,20 @@ func Attach(c *cron.Cron, cfg Config) error { go func() { //h.updateServerData() //h.vacuumDatabase() - for _, fn := range updateHistoryFuncs { - go fn() - } - //for _, fn := range updateStatsFuncs { + //for _, fn := range updateHistoryFuncs { // go fn() //} + for _, fn := range updateStatsFuncs { + go fn() + } }() } return nil } + +func createFnWithTimezone(timezone string, fn func(timezone string)) func() { + return func() { + fn(timezone) + } +} diff --git a/cron/helpers.go b/cron/helpers.go deleted file mode 100644 index 8512a35..0000000 --- a/cron/helpers.go +++ /dev/null @@ -1,7 +0,0 @@ -package cron - -func createFnWithTimezone(timezone string, fn func(timezone string)) func() { - return func() { - fn(timezone) - } -} diff --git a/cron/tasks/task_update_stats.go b/cron/tasks/task_update_stats.go index 7a251a3..5020443 100644 --- a/cron/tasks/task_update_stats.go +++ b/cron/tasks/task_update_stats.go @@ -14,10 +14,11 @@ type taskUpdateStats struct { } func (t *taskUpdateStats) execute(timezone string) error { + entry := log.WithField("timezone", timezone) location, err := t.loadLocation(timezone) if err != nil { err = errors.Wrap(err, "taskUpdateStats.execute") - log.Error(err) + entry.Error(err) return err } year, month, day := time.Now().In(location).Date() @@ -35,10 +36,10 @@ func (t *taskUpdateStats) execute(timezone string) error { Select() if err != nil { err = errors.Wrap(err, "taskUpdateStats.execute") - log.Errorln(err) + entry.Errorln(err) return err } - log. + entry. WithField("numberOfServers", len(servers)). Info("Update of the stats has started") for _, server := range servers { diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 2d77f9c..2ca4550 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -90,6 +90,11 @@ func RegisterTasks(cfg *Config) error { RetryLimit: defaultRetryLimit, Handler: (&taskUpdateStats{t}).execute, }) + taskq.RegisterTask(&taskq.TaskOptions{ + Name: TaskUpdateServerStats, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateServerStats{t}).execute, + }) return nil } From c6c028b0d5b351c20b481f7ce26930268fe5c245 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 20:14:21 +0200 Subject: [PATCH 19/31] uncomment a few lines in the file cron.go, refactor task initialization --- cron/cron.go | 10 +- .../task_update_server_stats.go} | 51 +++++++- cron/tasks/tasks.go | 109 +++++++++--------- 3 files changed, 104 insertions(+), 66 deletions(-) rename cron/{update_server_stats_worker.go => tasks/task_update_server_stats.go} (68%) diff --git a/cron/cron.go b/cron/cron.go index ec53077..b693373 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -60,11 +60,11 @@ func Attach(c *cron.Cron, cfg Config) error { } if cfg.RunOnStartup { go func() { - //h.updateServerData() - //h.vacuumDatabase() - //for _, fn := range updateHistoryFuncs { - // go fn() - //} + h.updateServerData() + h.vacuumDatabase() + for _, fn := range updateHistoryFuncs { + go fn() + } for _, fn := range updateStatsFuncs { go fn() } diff --git a/cron/update_server_stats_worker.go b/cron/tasks/task_update_server_stats.go similarity index 68% rename from cron/update_server_stats_worker.go rename to cron/tasks/task_update_server_stats.go index c97e4aa..797e17f 100644 --- a/cron/update_server_stats_worker.go +++ b/cron/tasks/task_update_server_stats.go @@ -1,20 +1,59 @@ -package cron +package tasks import ( - "time" - "github.com/go-pg/pg/v10" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" + "time" ) -type updateServerStatsWorker struct { +type taskUpdateServerStats struct { + *task +} + +func (t *taskUpdateServerStats) execute(timezone string, server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + location, err := t.loadLocation(timezone) + if err != nil { + err = errors.Wrap(err, "taskUpdateServerStats.execute") + log.Error(err) + return err + } + entry := log.WithField("key", server.Key) + entry.Infof("%s: update of the stats has started...", server.Key) + err = (&workerUpdateServerStats{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + server: server, + location: location, + }).update() + if err != nil { + err = errors.Wrap(err, "taskUpdateServerStats.execute") + entry.Error(err) + return err + } + entry.Infof("%s: stats have been updated", server.Key) + + return nil +} + +func (t *taskUpdateServerStats) validatePayload(server *models.Server) error { + if server == nil { + return errors.Errorf("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + +type workerUpdateServerStats struct { db *pg.DB server *models.Server location *time.Location } -func (w *updateServerStatsWorker) prepare() (*models.ServerStats, error) { +func (w *workerUpdateServerStats) prepare() (*models.ServerStats, error) { activePlayers, err := w.db.Model(&models.Player{}).Where("exists = true").Count() if err != nil { return nil, errors.Wrap(err, "cannot count active players") @@ -71,7 +110,7 @@ func (w *updateServerStatsWorker) prepare() (*models.ServerStats, error) { }, nil } -func (w *updateServerStatsWorker) update() error { +func (w *workerUpdateServerStats) update() error { stats, err := w.prepare() if err != nil { return err diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 2ca4550..724b898 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -40,61 +40,60 @@ func RegisterTasks(cfg *Config) error { db: cfg.DB, queue: cfg.Queue, } - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameLoadVersionsAndUpdateServerData, - RetryLimit: defaultRetryLimit, - Handler: (&taskLoadVersionsAndUpdateServerData{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameLoadServersAndUpdateData, - RetryLimit: defaultRetryLimit, - Handler: (&taskLoadServersAndUpdateData{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameUpdateServerData, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateServerData{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameVacuum, - RetryLimit: defaultRetryLimit, - Handler: (&taskVacuum{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskNameVacuumServerDB, - RetryLimit: defaultRetryLimit, - Handler: (&taskVacuumServerDB{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateEnnoblements, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateEnnoblements{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateServerEnnoblements, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateServerEnnoblements{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateHistory, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateHistory{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateServerHistory, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateServerHistory{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateStats, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateStats{t}).execute, - }) - taskq.RegisterTask(&taskq.TaskOptions{ - Name: TaskUpdateServerStats, - RetryLimit: defaultRetryLimit, - Handler: (&taskUpdateServerStats{t}).execute, - }) + options := []*taskq.TaskOptions{ + { + Name: TaskNameLoadVersionsAndUpdateServerData, + Handler: (&taskLoadVersionsAndUpdateServerData{t}).execute, + }, + { + Name: TaskNameLoadServersAndUpdateData, + Handler: (&taskLoadServersAndUpdateData{t}).execute, + }, + { + Name: TaskNameUpdateServerData, + Handler: (&taskUpdateServerData{t}).execute, + }, + { + Name: TaskNameVacuum, + Handler: (&taskVacuum{t}).execute, + }, + { + Name: TaskNameVacuumServerDB, + Handler: (&taskVacuumServerDB{t}).execute, + }, + { + Name: TaskUpdateEnnoblements, + Handler: (&taskUpdateEnnoblements{t}).execute, + }, + { + Name: TaskUpdateServerEnnoblements, + Handler: (&taskUpdateServerEnnoblements{t}).execute, + }, + { + Name: TaskUpdateHistory, + Handler: (&taskUpdateHistory{t}).execute, + }, + { + Name: TaskUpdateServerHistory, + RetryLimit: defaultRetryLimit, + Handler: (&taskUpdateServerHistory{t}).execute, + }, + { + Name: TaskUpdateStats, + Handler: (&taskUpdateStats{t}).execute, + }, + { + Name: TaskUpdateServerStats, + Handler: (&taskUpdateServerStats{t}).execute, + }, + } + for _, taskOptions := range options { + opts := taskOptions + if opts.RetryLimit == 0 { + opts.RetryLimit = defaultRetryLimit + } + taskq.RegisterTask(opts) + } return nil } From 35045af75ffdd37c896abf02d1d30f0f975c9f0d Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sun, 25 Apr 2021 20:24:26 +0200 Subject: [PATCH 20/31] refactor, update README.md, move queue.Config to the separate file --- README.md | 12 +++++-- cron/queue/config.go | 18 ++++++++++ cron/queue/queue.go | 35 ++++++------------- cron/tasks/helpers.go | 8 +++++ .../task_load_servers_and_update_data.go | 4 +-- cron/tasks/task_update_server_data.go | 9 ++--- cron/tasks/task_update_server_ennoblements.go | 7 ++-- main.go | 2 +- 8 files changed, 54 insertions(+), 41 deletions(-) create mode 100644 cron/queue/config.go diff --git a/README.md b/README.md index 2d9656e..3fded54 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# tribalwarshelp.com cron +# TWHelp cron Features: @@ -18,7 +18,15 @@ DB_PORT=5432 DB_HOST=your_db_host DB_PASSWORD=your_db_pass -MAX_CONCURRENT_WORKERS=1 #how many servers should update at the same time +REDIS_ADDR=redis_addr +REDIS_DB=redis_db +REDIS_USER=redis_user +REDIS_PASSWORD=redis_password + +RUN_ON_STARTUP=true|false +LOG_DB_QUERIES=true|false + +WORKER_LIMIT=1 #how many servers should update at the same time ``` ### Prerequisites diff --git a/cron/queue/config.go b/cron/queue/config.go new file mode 100644 index 0000000..52d1543 --- /dev/null +++ b/cron/queue/config.go @@ -0,0 +1,18 @@ +package queue + +import ( + "github.com/go-redis/redis/v8" + "github.com/pkg/errors" +) + +type Config struct { + Redis redis.UniversalClient + WorkerLimit int +} + +func validateConfig(cfg *Config) error { + if cfg == nil || cfg.Redis == nil { + return errors.New("cfg.Redis is required") + } + return nil +} diff --git a/cron/queue/queue.go b/cron/queue/queue.go index 75f32d2..4656537 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -17,11 +17,6 @@ const ( EnnoblementsQueue QueueName = "ennoblements" ) -type Config struct { - Redis redis.UniversalClient - WorkerLimit int -} - type Queue interface { Start(ctx context.Context) error Close() error @@ -69,6 +64,16 @@ func (q *queue) registerQueue(name QueueName, limit int) taskq.Queue { }) } +func (q *queue) getQueueByName(name QueueName) taskq.Queue { + switch name { + case MainQueue: + return q.mainQueue + case EnnoblementsQueue: + return q.ennoblementsQueue + } + return nil +} + func (q *queue) Start(ctx context.Context) error { if err := q.factory.StartConsumers(ctx); err != nil { return errors.Wrap(err, "Couldn't start the queue") @@ -93,23 +98,3 @@ func (q *queue) Add(name QueueName, msg *taskq.Message) error { } return nil } - -func (q *queue) getQueueByName(name QueueName) taskq.Queue { - switch name { - case MainQueue: - return q.mainQueue - case EnnoblementsQueue: - return q.ennoblementsQueue - } - return nil -} - -func validateConfig(cfg *Config) error { - if cfg == nil { - return errors.New("Config hasn't been provided") - } - if cfg.Redis == nil { - return errors.New("cfg.Redis is a required field") - } - return nil -} diff --git a/cron/tasks/helpers.go b/cron/tasks/helpers.go index 55a5963..89cf285 100644 --- a/cron/tasks/helpers.go +++ b/cron/tasks/helpers.go @@ -1,6 +1,7 @@ package tasks import ( + "github.com/tribalwarshelp/shared/tw/dataloader" "net/http" "time" @@ -35,6 +36,13 @@ func newHTTPClient() *http.Client { } } +func newDataloader(url string) dataloader.DataLoader { + return dataloader.New(&dataloader.Config{ + BaseURL: url, + Client: newHTTPClient(), + }) +} + type tribesSearchableByID struct { tribes []*models.Tribe } diff --git a/cron/tasks/task_load_servers_and_update_data.go b/cron/tasks/task_load_servers_and_update_data.go index 30462f2..8107a6a 100644 --- a/cron/tasks/task_load_servers_and_update_data.go +++ b/cron/tasks/task_load_servers_and_update_data.go @@ -103,11 +103,11 @@ func (t *taskLoadServersAndUpdateData) getServers(version *models.Version) (map[ bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the body", version.Host) + return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't read the response body", version.Host) } body, err := phpserialize.Decode(string(bodyBytes)) if err != nil { - return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the body into the go value", version.Host) + return nil, errors.Wrapf(err, "%s: taskLoadServersAndUpdateData.loadServers couldn't decode the response body into the go value", version.Host) } result := make(map[string]string) diff --git a/cron/tasks/task_update_server_data.go b/cron/tasks/task_update_server_data.go index 107a73f..9c7b074 100644 --- a/cron/tasks/task_update_server_data.go +++ b/cron/tasks/task_update_server_data.go @@ -22,12 +22,9 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error entry := log.WithField("key", server.Key) entry.Infof("%s: updating data...", server.Key) err := (&workerUpdateServerData{ - db: t.db.WithParam("SERVER", pg.Safe(server.Key)), - dataloader: dataloader.New(&dataloader.Config{ - BaseURL: url, - Client: newHTTPClient(), - }), - server: server, + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + dataloader: newDataloader(url), + server: server, }).update() if err != nil { err = errors.Wrap(err, "taskUpdateServerData.execute") diff --git a/cron/tasks/task_update_server_ennoblements.go b/cron/tasks/task_update_server_ennoblements.go index 4a6e4f0..d359976 100644 --- a/cron/tasks/task_update_server_ennoblements.go +++ b/cron/tasks/task_update_server_ennoblements.go @@ -19,11 +19,8 @@ func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server entry := log.WithField("key", server.Key) entry.Debugf("%s: update of the ennoblements has started...", server.Key) err := (&workerUpdateServerEnnoblements{ - db: t.db.WithParam("SERVER", pg.Safe(server.Key)), - dataloader: dataloader.New(&dataloader.Config{ - BaseURL: url, - Client: newHTTPClient(), - }), + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + dataloader: newDataloader(url), }).update() if err != nil { err = errors.Wrap(err, "taskUpdateServerEnnoblements.execute") diff --git a/main.go b/main.go index 50b3cbd..f46d489 100644 --- a/main.go +++ b/main.go @@ -47,7 +47,7 @@ func main() { logrus.Info("Connection with the database has been established") queue, err := queue.New(&queue.Config{ - WorkerLimit: envutils.GetenvInt("MAX_CONCURRENT_WORKERS"), + WorkerLimit: envutils.GetenvInt("WORKER_LIMIT"), Redis: client, }) if err != nil { From e29807e077092cb6407685926a55fbd141f2b955 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Tue, 27 Apr 2021 18:30:12 +0200 Subject: [PATCH 21/31] move task registration/queue creation logic to the cron package --- README.md | 2 +- cron/config.go | 26 ++++ cron/cron.go | 113 ++++++++++++++---- cron/handler.go | 32 ----- cron/queue/config.go | 2 +- cron/tasks/helpers.go | 7 +- .../task_load_servers_and_update_data.go | 8 +- main.go | 44 +++---- 8 files changed, 142 insertions(+), 92 deletions(-) create mode 100644 cron/config.go delete mode 100644 cron/handler.go diff --git a/README.md b/README.md index 3fded54..a1f0268 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ REDIS_DB=redis_db REDIS_USER=redis_user REDIS_PASSWORD=redis_password -RUN_ON_STARTUP=true|false +RUN_ON_INIT=true|false LOG_DB_QUERIES=true|false WORKER_LIMIT=1 #how many servers should update at the same time diff --git a/cron/config.go b/cron/config.go new file mode 100644 index 0000000..069724a --- /dev/null +++ b/cron/config.go @@ -0,0 +1,26 @@ +package cron + +import ( + "github.com/go-pg/pg/v10" + "github.com/go-redis/redis/v8" + "github.com/pkg/errors" + "github.com/robfig/cron/v3" +) + +type Config struct { + DB *pg.DB + Redis redis.UniversalClient + RunOnInit bool + Opts []cron.Option + WorkerLimit int +} + +func validateConfig(cfg *Config) error { + if cfg == nil || cfg.DB == nil { + return errors.New("validateConfig: cfg.DB is required") + } + if cfg.Redis == nil { + return errors.New("validateConfig: cfg.Redis is required") + } + return nil +} diff --git a/cron/cron.go b/cron/cron.go index b693373..8da6e1a 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -1,45 +1,58 @@ package cron import ( + "context" "fmt" - "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "github.com/robfig/cron/v3" "github.com/tribalwarshelp/shared/models" "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/cron/tasks" ) -type Config struct { - DB *pg.DB - RunOnStartup bool - Queue queue.Queue +type Cron struct { + *cron.Cron + queue queue.Queue + db *pg.DB + runOnInit bool } -func Attach(c *cron.Cron, cfg Config) error { - if cfg.DB == nil { - return fmt.Errorf("cfg.DB cannot be nil, expected *pg.DB") +func New(cfg *Config) (*Cron, error) { + if err := validateConfig(cfg); err != nil { + return nil, err } - if cfg.Queue == nil { - return fmt.Errorf("cfg.Queue cannot be nil, expected queue.Queue") - } - - h := &handler{ - queue: cfg.Queue, + q, err := initializeQueue(cfg) + if err != nil { + return nil, err } + c := &Cron{ + Cron: cron.New(cfg.Opts...), + queue: q, + db: cfg.DB, + runOnInit: cfg.RunOnInit, + } + if err := c.init(); err != nil { + return nil, err + } + return c, nil +} +func (c *Cron) init() error { var versions []*models.Version - if err := cfg.DB.Model(&versions).DistinctOn("timezone").Select(); err != nil { - return err + if err := c.db.Model(&versions).DistinctOn("timezone").Select(); err != nil { + return errors.Wrap(err, "Cron.init: couldn't load versions") } var updateHistoryFuncs []func() var updateStatsFuncs []func() for _, version := range versions { - updateHistory := createFnWithTimezone(version.Timezone, h.updateHistory) + updateHistory := createFnWithTimezone(version.Timezone, c.updateHistory) updateHistoryFuncs = append(updateHistoryFuncs, updateHistory) - updateStats := createFnWithTimezone(version.Timezone, h.updateStats) + updateStats := createFnWithTimezone(version.Timezone, c.updateStats) updateStatsFuncs = append(updateStatsFuncs, updateStats) if _, err := c.AddFunc(fmt.Sprintf("CRON_TZ=%s 30 1 * * *", version.Timezone), updateHistory); err != nil { @@ -49,19 +62,19 @@ func Attach(c *cron.Cron, cfg Config) error { return err } } - if _, err := c.AddFunc("0 * * * *", h.updateServerData); err != nil { + if _, err := c.AddFunc("0 * * * *", c.updateServerData); err != nil { return err } - if _, err := c.AddFunc("20 1 * * *", h.vacuumDatabase); err != nil { + if _, err := c.AddFunc("20 1 * * *", c.vacuumDatabase); err != nil { return err } - if _, err := c.AddFunc("@every 1m", h.updateEnnoblements); err != nil { + if _, err := c.AddFunc("@every 1m", c.updateEnnoblements); err != nil { return err } - if cfg.RunOnStartup { + if c.runOnInit { go func() { - h.updateServerData() - h.vacuumDatabase() + c.updateServerData() + c.vacuumDatabase() for _, fn := range updateHistoryFuncs { go fn() } @@ -70,10 +83,60 @@ func Attach(c *cron.Cron, cfg Config) error { } }() } - return nil } +func (c *Cron) Start(ctx context.Context) error { + if err := c.queue.Start(ctx); err != nil { + return errors.Wrap(err, "Cron.Start") + } + c.Cron.Start() + return nil +} + +func (c *Cron) Stop() error { + c.Cron.Stop() + if err := c.queue.Close(); err != nil { + return errors.Wrap(err, "Cron.Stop") + } + return nil +} + +func (c *Cron) updateServerData() { + c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) +} + +func (c *Cron) updateEnnoblements() { + c.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) +} + +func (c *Cron) updateHistory(timezone string) { + c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone)) +} + +func (c *Cron) updateStats(timezone string) { + c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone)) +} + +func (c *Cron) vacuumDatabase() { + c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background())) +} + +func initializeQueue(cfg *Config) (queue.Queue, error) { + q, err := queue.New(&queue.Config{ + WorkerLimit: cfg.WorkerLimit, + Redis: cfg.Redis, + }) + if err != nil { + return nil, errors.Wrap(err, "initializeQueue: Couldn't create the task q") + } + tasks.RegisterTasks(&tasks.Config{ + DB: cfg.DB, + Queue: q, + }) + return q, nil +} + func createFnWithTimezone(timezone string, fn func(timezone string)) func() { return func() { fn(timezone) diff --git a/cron/handler.go b/cron/handler.go deleted file mode 100644 index d783b75..0000000 --- a/cron/handler.go +++ /dev/null @@ -1,32 +0,0 @@ -package cron - -import ( - "context" - - "github.com/tribalwarshelp/cron/cron/queue" - "github.com/tribalwarshelp/cron/cron/tasks" -) - -type handler struct { - queue queue.Queue -} - -func (h *handler) updateServerData() { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) -} - -func (h *handler) updateEnnoblements() { - h.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) -} - -func (h *handler) updateHistory(timezone string) { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone)) -} - -func (h *handler) updateStats(timezone string) { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone)) -} - -func (h *handler) vacuumDatabase() { - h.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background())) -} diff --git a/cron/queue/config.go b/cron/queue/config.go index 52d1543..09e8f00 100644 --- a/cron/queue/config.go +++ b/cron/queue/config.go @@ -12,7 +12,7 @@ type Config struct { func validateConfig(cfg *Config) error { if cfg == nil || cfg.Redis == nil { - return errors.New("cfg.Redis is required") + return errors.New("validateConfig: cfg.Redis is required") } return nil } diff --git a/cron/tasks/helpers.go b/cron/tasks/helpers.go index 89cf285..4d3c437 100644 --- a/cron/tasks/helpers.go +++ b/cron/tasks/helpers.go @@ -19,8 +19,11 @@ func countPlayerVillages(villages []*models.Village) int { } func getDateDifferenceInDays(t1, t2 time.Time) int { - diff := t1.Sub(t2) - return int(diff.Hours() / 24) + hours := t1.Sub(t2).Hours() + if hours == 0 { + return 0 + } + return int(hours / 24) } func calcPlayerDailyGrowth(diffInDays, points int) int { diff --git a/cron/tasks/task_load_servers_and_update_data.go b/cron/tasks/task_load_servers_and_update_data.go index 8107a6a..ec7aed0 100644 --- a/cron/tasks/task_load_servers_and_update_data.go +++ b/cron/tasks/task_load_servers_and_update_data.go @@ -29,7 +29,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { return nil } entry := log.WithField("host", version.Host) - entry.Infof("%s: Loading servers", version.Host) + entry.Infof("taskLoadServersAndUpdateData.execute: %s: Loading servers", version.Host) data, err := t.getServers(version) if err != nil { log.Errorln(err) @@ -49,7 +49,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { Version: version, } if err := db.CreateSchema(t.db, server); err != nil { - logrus.Warn(errors.Wrapf(err, "%s: couldn't create the schema", server.Key)) + logrus.Warn(errors.Wrapf(err, "taskLoadServersAndUpdateData.execute: %s: couldn't create the schema", server.Key)) continue } servers = append(servers, server) @@ -64,7 +64,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { Returning("*"). Insert(); err != nil { err = errors.Wrap(err, "taskLoadServersAndUpdateData.execute: couldn't insert/update servers") - logrus.Fatal(err) + logrus.Error(err) return err } } @@ -74,7 +74,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { Where("key NOT IN (?) AND version_code = ?", pg.In(serverKeys), version.Code). Update(); err != nil { err = errors.Wrap(err, "taskLoadServersAndUpdateData.execute: couldn't update server statuses") - logrus.Fatal(err) + logrus.Error(err) return err } diff --git a/main.go b/main.go index f46d489..c50a321 100644 --- a/main.go +++ b/main.go @@ -13,8 +13,6 @@ import ( "github.com/tribalwarshelp/shared/mode" twhelpcron "github.com/tribalwarshelp/cron/cron" - "github.com/tribalwarshelp/cron/cron/queue" - "github.com/tribalwarshelp/cron/cron/tasks" "github.com/tribalwarshelp/cron/db" envutils "github.com/tribalwarshelp/cron/utils/env" @@ -33,46 +31,38 @@ func init() { } func main() { - client, err := initializeRedis() + redisClient, err := initializeRedis() if err != nil { logrus.Fatal(errors.Wrap(err, "Couldn't connect to Redis")) } - defer client.Close() + defer redisClient.Close() - conn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")}) + dbConn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")}) if err != nil { logrus.Fatal(errors.Wrap(err, "Couldn't connect to the db")) } - defer conn.Close() + defer dbConn.Close() logrus.Info("Connection with the database has been established") - queue, err := queue.New(&queue.Config{ + c, err := twhelpcron.New(&twhelpcron.Config{ + DB: dbConn, + RunOnInit: envutils.GetenvBool("RUN_ON_INIT"), + Redis: redisClient, WorkerLimit: envutils.GetenvInt("WORKER_LIMIT"), - Redis: client, + Opts: []cron.Option{ + cron.WithChain( + cron.SkipIfStillRunning( + cron.PrintfLogger(logrus.WithField("package", "cron")), + ), + ), + }, }) if err != nil { - logrus.Fatal(errors.Wrap(err, "Couldn't create the task queue")) - } - tasks.RegisterTasks(&tasks.Config{ - DB: conn, - Queue: queue, - }) - if err := queue.Start(context.Background()); err != nil { logrus.Fatal(err) } - - c := cron.New(cron.WithChain( - cron.SkipIfStillRunning(cron.PrintfLogger(logrus.WithField("package", "cron"))), - )) - if err := twhelpcron.Attach(c, twhelpcron.Config{ - DB: conn, - RunOnStartup: envutils.GetenvBool("RUN_ON_STARTUP"), - Queue: queue, - }); err != nil { + if err := c.Start(context.Background()); err != nil { logrus.Fatal(err) } - c.Start() - defer c.Stop() logrus.Info("Cron is running!") @@ -81,7 +71,7 @@ func main() { <-channel logrus.Info("shutting down") - if err := queue.Close(); err != nil { + if err := c.Stop(); err != nil { logrus.Fatal(err) } } From add1379e0b7417cc5479c7340808a30963855acc Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Tue, 27 Apr 2021 20:06:38 +0200 Subject: [PATCH 22/31] make messages more clear, fix most of the unhandled error --- README.md | 7 ++-- cron/cron.go | 44 ++++++++++++++++++++---- cron/queue/queue.go | 14 ++++---- cron/tasks/task_update_ennoblements.go | 14 ++++++-- cron/tasks/task_update_history.go | 13 +++++-- cron/tasks/task_update_server_data.go | 14 +++++--- cron/tasks/task_update_server_history.go | 10 ++++-- cron/tasks/task_update_server_stats.go | 10 ++++-- cron/tasks/task_update_stats.go | 12 +++++-- cron/tasks/task_vacuum.go | 14 ++++++-- cron/tasks/task_vacuum_server_db.go | 16 ++++++--- db/db.go | 12 +++++-- main.go | 35 +++++++++++++++---- 13 files changed, 165 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index a1f0268..d9e8d6d 100644 --- a/README.md +++ b/README.md @@ -26,17 +26,18 @@ REDIS_PASSWORD=redis_password RUN_ON_INIT=true|false LOG_DB_QUERIES=true|false -WORKER_LIMIT=1 #how many servers should update at the same time +WORKER_LIMIT=1 ``` ### Prerequisites 1. Golang -2. PostgreSQL database +2. PostgreSQL +3. Redis ### Installing 1. Clone this repo. 2. Navigate to the directory where you have cloned this repo. -3. Set the required env variables directly in your system or create .env.development file. +3. Set the required env variables directly in your system or create .env.local file. 4. go run main.go diff --git a/cron/cron.go b/cron/cron.go index 8da6e1a..b82cf04 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/go-pg/pg/v10" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/robfig/cron/v3" "github.com/tribalwarshelp/shared/models" @@ -103,23 +104,49 @@ func (c *Cron) Stop() error { } func (c *Cron) updateServerData() { - c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) + err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameLoadVersionsAndUpdateServerData).WithArgs(context.Background())) + if err != nil { + c.logError("Cron.updateServerData", tasks.TaskNameLoadVersionsAndUpdateServerData, err) + } } func (c *Cron) updateEnnoblements() { - c.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) + err := c.queue.Add(queue.EnnoblementsQueue, tasks.Get(tasks.TaskUpdateEnnoblements).WithArgs(context.Background())) + if err != nil { + c.logError("Cron.updateEnnoblements", tasks.TaskUpdateEnnoblements, err) + } } func (c *Cron) updateHistory(timezone string) { - c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone)) + err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateHistory).WithArgs(context.Background(), timezone)) + if err != nil { + c.logError("Cron.updateHistory", tasks.TaskUpdateHistory, err) + } } func (c *Cron) updateStats(timezone string) { - c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone)) + err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskUpdateStats).WithArgs(context.Background(), timezone)) + if err != nil { + c.logError("Cron.updateStats", tasks.TaskUpdateStats, err) + } } func (c *Cron) vacuumDatabase() { - c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background())) + err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameVacuum).WithArgs(context.Background())) + if err != nil { + c.logError("Cron.vacuumDatabase", tasks.TaskNameVacuum, err) + } +} + +func (c *Cron) logError(prefix string, taskName string, err error) { + logrus.Error( + errors.Wrapf( + err, + "%s: Couldn't add the task '%s' to the queue", + prefix, + taskName, + ), + ) } func initializeQueue(cfg *Config) (queue.Queue, error) { @@ -128,12 +155,15 @@ func initializeQueue(cfg *Config) (queue.Queue, error) { Redis: cfg.Redis, }) if err != nil { - return nil, errors.Wrap(err, "initializeQueue: Couldn't create the task q") + return nil, errors.Wrap(err, "initializeQueue: Couldn't create the task queue") } - tasks.RegisterTasks(&tasks.Config{ + err = tasks.RegisterTasks(&tasks.Config{ DB: cfg.DB, Queue: q, }) + if err != nil { + return nil, errors.Wrap(err, "initializeQueue: Couldn't create the task queue") + } return q, nil } diff --git a/cron/queue/queue.go b/cron/queue/queue.go index 4656537..197d076 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -10,17 +10,17 @@ import ( "github.com/vmihailenco/taskq/v3/redisq" ) -type QueueName string +type Name string const ( - MainQueue QueueName = "main" - EnnoblementsQueue QueueName = "ennoblements" + MainQueue Name = "main" + EnnoblementsQueue Name = "ennoblements" ) type Queue interface { Start(ctx context.Context) error Close() error - Add(name QueueName, msg *taskq.Message) error + Add(name Name, msg *taskq.Message) error } type queue struct { @@ -54,7 +54,7 @@ func (q *queue) init(cfg *Config) error { return nil } -func (q *queue) registerQueue(name QueueName, limit int) taskq.Queue { +func (q *queue) registerQueue(name Name, limit int) taskq.Queue { return q.factory.RegisterQueue(&taskq.QueueOptions{ Name: string(name), ReservationTimeout: time.Minute * 2, @@ -64,7 +64,7 @@ func (q *queue) registerQueue(name QueueName, limit int) taskq.Queue { }) } -func (q *queue) getQueueByName(name QueueName) taskq.Queue { +func (q *queue) getQueueByName(name Name) taskq.Queue { switch name { case MainQueue: return q.mainQueue @@ -88,7 +88,7 @@ func (q *queue) Close() error { return nil } -func (q *queue) Add(name QueueName, msg *taskq.Message) error { +func (q *queue) Add(name Name, msg *taskq.Message) error { queue := q.getQueueByName(name) if queue == nil { return errors.Errorf("Couldn't add the message to the queue: unknown queue name '%s'", name) diff --git a/cron/tasks/task_update_ennoblements.go b/cron/tasks/task_update_ennoblements.go index af3c3ea..8f08e91 100644 --- a/cron/tasks/task_update_ennoblements.go +++ b/cron/tasks/task_update_ennoblements.go @@ -25,14 +25,24 @@ func (t *taskUpdateEnnoblements) execute() error { log.Errorln(err) return err } - log.WithField("numberOfServers", len(servers)).Info("Update of the ennoblements has started...") + log.WithField("numberOfServers", len(servers)).Info("taskUpdateEnnoblements.execute: Update of the ennoblements has started...") for _, server := range servers { s := server - t.queue.Add( + err := t.queue.Add( queue.EnnoblementsQueue, Get(TaskUpdateServerEnnoblements). WithArgs(context.Background(), fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), s), ) + if err != nil { + log.Warn( + errors.Wrapf( + err, + "taskUpdateEnnoblements.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerEnnoblements, + ), + ) + } } return nil } diff --git a/cron/tasks/task_update_history.go b/cron/tasks/task_update_history.go index 02b2b32..c5e115f 100644 --- a/cron/tasks/task_update_history.go +++ b/cron/tasks/task_update_history.go @@ -41,10 +41,19 @@ func (t *taskUpdateHistory) execute(timezone string) error { } entry. WithField("numberOfServers", len(servers)). - Info("Update of the history has started") + Info("taskUpdateHistory.execute: Update of the history has started") for _, server := range servers { s := server - t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), timezone, s)) + err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), timezone, s)) + log.Warn( + errors.Wrapf( + err, + "taskUpdateHistory.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerHistory, + ), + ) + } return nil } diff --git a/cron/tasks/task_update_server_data.go b/cron/tasks/task_update_server_data.go index 9c7b074..04c8b17 100644 --- a/cron/tasks/task_update_server_data.go +++ b/cron/tasks/task_update_server_data.go @@ -20,7 +20,7 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error } now := time.Now() entry := log.WithField("key", server.Key) - entry.Infof("%s: updating data...", server.Key) + entry.Infof("taskUpdateServerData.execute: %s: Update of the server data has started...", server.Key) err := (&workerUpdateServerData{ db: t.db.WithParam("SERVER", pg.Safe(server.Key)), dataloader: newDataloader(url), @@ -37,13 +37,13 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error "duration": duration.Nanoseconds(), "durationPretty": duration.String(), }). - Infof("%s has been updated", server.Key) + Infof("taskUpdateServerData.execute: %s: data has been updated", server.Key) return nil } func (t *taskUpdateServerData) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskLoadServersAndUpdateData.validatePayload: Expected *models.Server, got nil") + return errors.Errorf("taskUpdateServerData.validatePayload: Expected *models.Server, got nil") } return nil @@ -59,7 +59,7 @@ func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeate var ennoblements = []*models.Ennoblement{} err := w.db.Model(&ennoblements).DistinctOn("new_owner_id").Order("new_owner_id ASC", "ennobled_at ASC").Select() if err != nil { - return nil, errors.Wrap(err, "loadPlayers: couldn't load ennoblements") + return nil, errors.Wrap(err, "workerUpdateServerData.loadPlayers: couldn't load ennoblements") } players, err := w.dataloader.LoadPlayers() @@ -210,7 +210,11 @@ func (w *workerUpdateServerData) update() error { if err != nil { return err } - defer tx.Close() + defer func(s *models.Server) { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key)) + } + }(w.server) if len(tribes) > 0 { ids := []int{} diff --git a/cron/tasks/task_update_server_history.go b/cron/tasks/task_update_server_history.go index 28a8859..df01bdc 100644 --- a/cron/tasks/task_update_server_history.go +++ b/cron/tasks/task_update_server_history.go @@ -23,7 +23,7 @@ func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server return err } entry := log.WithField("key", server.Key) - entry.Infof("%s: update of the history has started...", server.Key) + entry.Infof("taskUpdateServerHistory.execute: %s: update of the history has started...", server.Key) err = (&workerUpdateServerHistory{ db: t.db.WithParam("SERVER", pg.Safe(server.Key)), server: server, @@ -34,7 +34,7 @@ func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server entry.Error(err) return err } - entry.Infof("%s: history has been updated", server.Key) + entry.Infof("taskUpdateServerHistory.execute: %s: history has been updated", server.Key) return nil } @@ -97,7 +97,11 @@ func (w *workerUpdateServerHistory) update() error { if err != nil { return err } - defer tx.Close() + defer func(s *models.Server) { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key)) + } + }(w.server) if len(ph) > 0 { if _, err := w.db.Model(&ph).Insert(); err != nil { diff --git a/cron/tasks/task_update_server_stats.go b/cron/tasks/task_update_server_stats.go index 797e17f..2c77307 100644 --- a/cron/tasks/task_update_server_stats.go +++ b/cron/tasks/task_update_server_stats.go @@ -23,7 +23,7 @@ func (t *taskUpdateServerStats) execute(timezone string, server *models.Server) return err } entry := log.WithField("key", server.Key) - entry.Infof("%s: update of the stats has started...", server.Key) + entry.Infof("taskUpdateServerStats.execute: %s: update of the stats has started...", server.Key) err = (&workerUpdateServerStats{ db: t.db.WithParam("SERVER", pg.Safe(server.Key)), server: server, @@ -34,7 +34,7 @@ func (t *taskUpdateServerStats) execute(timezone string, server *models.Server) entry.Error(err) return err } - entry.Infof("%s: stats have been updated", server.Key) + entry.Infof("taskUpdateServerStats.execute: %s: stats have been updated", server.Key) return nil } @@ -120,7 +120,11 @@ func (w *workerUpdateServerStats) update() error { if err != nil { return err } - defer tx.Close() + defer func(s *models.Server) { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key)) + } + }(w.server) if _, err := tx.Model(stats).Insert(); err != nil { return errors.Wrap(err, "cannot insert server stats") diff --git a/cron/tasks/task_update_stats.go b/cron/tasks/task_update_stats.go index 5020443..5a4c43b 100644 --- a/cron/tasks/task_update_stats.go +++ b/cron/tasks/task_update_stats.go @@ -41,10 +41,18 @@ func (t *taskUpdateStats) execute(timezone string) error { } entry. WithField("numberOfServers", len(servers)). - Info("Update of the stats has started") + Info("taskUpdateStats.execute: Update of the stats has started") for _, server := range servers { s := server - t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), timezone, s)) + err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), timezone, s)) + log.Warn( + errors.Wrapf( + err, + "taskUpdateStats.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerStats, + ), + ) } return nil } diff --git a/cron/tasks/task_vacuum.go b/cron/tasks/task_vacuum.go index d83d8bb..0dda3e7 100644 --- a/cron/tasks/task_vacuum.go +++ b/cron/tasks/task_vacuum.go @@ -22,10 +22,20 @@ func (t *taskVacuum) execute() error { log.Errorln(err) return err } - log.Infof("Start database vacuuming...") + log.Infof("taskVacuum.execute: Start database vacumming...") for _, server := range servers { s := server - t.queue.Add(queue.MainQueue, Get(TaskNameVacuumServerDB).WithArgs(context.Background(), s)) + err := t.queue.Add(queue.MainQueue, Get(TaskNameVacuumServerDB).WithArgs(context.Background(), s)) + if err != nil { + log.Warn( + errors.Wrapf( + err, + "taskVacuum.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerEnnoblements, + ), + ) + } } return nil } diff --git a/cron/tasks/task_vacuum_server_db.go b/cron/tasks/task_vacuum_server_db.go index 1a04e88..719f2e8 100644 --- a/cron/tasks/task_vacuum_server_db.go +++ b/cron/tasks/task_vacuum_server_db.go @@ -21,16 +21,17 @@ func (t *taskVacuumServerDB) execute(server *models.Server) error { return nil } entry := log.WithField("key", server.Key) - entry.Infof("%s: vacumming the database...", server.Key) + entry.Infof("taskVacuumServerDB.execute: %s: vacumming the database...", server.Key) err := (&workerVacuumServerDB{ - db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + server: server, }).vacuum() if err != nil { err = errors.Wrap(err, "taskVacuumServerDB.execute") entry.Error(err) return err } - entry.Infof("%s: the database has been vacummed", server.Key) + entry.Infof("taskVacuumServerDB.execute: %s: the database has been vacummed", server.Key) return nil } @@ -44,7 +45,8 @@ func (t *taskVacuumServerDB) validatePayload(server *models.Server) error { } type workerVacuumServerDB struct { - db *pg.DB + db *pg.DB + server *models.Server } func (w *workerVacuumServerDB) vacuum() error { @@ -52,7 +54,11 @@ func (w *workerVacuumServerDB) vacuum() error { if err != nil { return err } - defer tx.Close() + defer func(s *models.Server) { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrapf(err, "%s: Couldn't rollback the transaction", s.Key)) + } + }(w.server) withNonExistentPlayers := w.db.Model(&models.Player{}).Column("id").Where("exists = false and NOW() - deleted_at > '14 days'") withNonExistentTribes := w.db.Model(&models.Tribe{}).Column("id").Where("exists = false and NOW() - deleted_at > '1 days'") diff --git a/db/db.go b/db/db.go index d946e48..59eb452 100644 --- a/db/db.go +++ b/db/db.go @@ -50,7 +50,11 @@ func prepareDB(db *pg.DB) error { if err != nil { return errors.Wrap(err, "Couldn't start a transaction") } - defer tx.Close() + defer func() { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrap(err, "prepareDB: Couldn't rollback the transaction")) + } + }() dbModels := []interface{}{ (*models.SpecialServer)(nil), @@ -140,7 +144,11 @@ func createSchema(db *pg.DB, server *models.Server, init bool) error { if err != nil { return errors.Wrap(err, "CreateSchema: couldn't start a transaction") } - defer tx.Close() + defer func() { + if err := tx.Close(); err != nil { + log.Warn(errors.Wrap(err, "createSchema: Couldn't rollback the transaction")) + } + }() if _, err := tx.Exec(fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", server.Key)); err != nil { return errors.Wrap(err, "CreateSchema: couldn't create the schema") diff --git a/main.go b/main.go index c50a321..2f355d6 100644 --- a/main.go +++ b/main.go @@ -21,12 +21,9 @@ import ( ) func init() { - os.Setenv("TZ", "UTC") - - if mode.Get() == mode.DevelopmentMode { - godotenv.Load(".env.local") + if err := setupENVs(); err != nil { + logrus.Fatal(err) } - setupLogger() } @@ -35,13 +32,21 @@ func main() { if err != nil { logrus.Fatal(errors.Wrap(err, "Couldn't connect to Redis")) } - defer redisClient.Close() + defer func() { + if err := redisClient.Close(); err != nil { + logrus.Warn(errors.Wrap(err, "Couldn't close the Redis connection")) + } + }() dbConn, err := db.New(&db.Config{LogQueries: envutils.GetenvBool("LOG_DB_QUERIES")}) if err != nil { logrus.Fatal(errors.Wrap(err, "Couldn't connect to the db")) } - defer dbConn.Close() + defer func() { + if err := dbConn.Close(); err != nil { + logrus.Warn(errors.Wrap(err, "Couldn't close the db connection")) + } + }() logrus.Info("Connection with the database has been established") c, err := twhelpcron.New(&twhelpcron.Config{ @@ -94,6 +99,22 @@ func setupLogger() { } } +func setupENVs() error { + err := os.Setenv("TZ", "UTC") + if err != nil { + return errors.Wrap(err, "setupENVs") + } + + if mode.Get() == mode.DevelopmentMode { + err := godotenv.Load(".env.local") + if err != nil { + return errors.Wrap(err, "setupENVs") + } + } + + return nil +} + func initializeRedis() (redis.UniversalClient, error) { client := redis.NewClient(&redis.Options{ Addr: envutils.GetenvString("REDIS_ADDR"), From ef73f63195ec76cc4090a3eeed577ca44b3cb945 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Tue, 27 Apr 2021 20:10:14 +0200 Subject: [PATCH 23/31] remove custom type 'Name' from the queue package --- cron/queue/queue.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cron/queue/queue.go b/cron/queue/queue.go index 197d076..20ac20f 100644 --- a/cron/queue/queue.go +++ b/cron/queue/queue.go @@ -10,17 +10,15 @@ import ( "github.com/vmihailenco/taskq/v3/redisq" ) -type Name string - const ( - MainQueue Name = "main" - EnnoblementsQueue Name = "ennoblements" + MainQueue = "main" + EnnoblementsQueue = "ennoblements" ) type Queue interface { Start(ctx context.Context) error Close() error - Add(name Name, msg *taskq.Message) error + Add(name string, msg *taskq.Message) error } type queue struct { @@ -54,9 +52,9 @@ func (q *queue) init(cfg *Config) error { return nil } -func (q *queue) registerQueue(name Name, limit int) taskq.Queue { +func (q *queue) registerQueue(name string, limit int) taskq.Queue { return q.factory.RegisterQueue(&taskq.QueueOptions{ - Name: string(name), + Name: name, ReservationTimeout: time.Minute * 2, Redis: q.redis, MinNumWorker: int32(limit), @@ -64,7 +62,7 @@ func (q *queue) registerQueue(name Name, limit int) taskq.Queue { }) } -func (q *queue) getQueueByName(name Name) taskq.Queue { +func (q *queue) getQueueByName(name string) taskq.Queue { switch name { case MainQueue: return q.mainQueue @@ -88,7 +86,7 @@ func (q *queue) Close() error { return nil } -func (q *queue) Add(name Name, msg *taskq.Message) error { +func (q *queue) Add(name string, msg *taskq.Message) error { queue := q.getQueueByName(name) if queue == nil { return errors.Errorf("Couldn't add the message to the queue: unknown queue name '%s'", name) From 352d26a61e5c89339e38e342ee46549bd8387d67 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Tue, 27 Apr 2021 20:18:06 +0200 Subject: [PATCH 24/31] add a new field to the Cron type - log --- cron/cron.go | 4 +++- cron/tasks/tasks.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cron/cron.go b/cron/cron.go index b82cf04..25fcb12 100644 --- a/cron/cron.go +++ b/cron/cron.go @@ -19,6 +19,7 @@ type Cron struct { queue queue.Queue db *pg.DB runOnInit bool + log logrus.FieldLogger } func New(cfg *Config) (*Cron, error) { @@ -34,6 +35,7 @@ func New(cfg *Config) (*Cron, error) { queue: q, db: cfg.DB, runOnInit: cfg.RunOnInit, + log: logrus.WithField("package", "cron"), } if err := c.init(); err != nil { return nil, err @@ -139,7 +141,7 @@ func (c *Cron) vacuumDatabase() { } func (c *Cron) logError(prefix string, taskName string, err error) { - logrus.Error( + c.log.Error( errors.Wrapf( err, "%s: Couldn't add the task '%s' to the queue", diff --git a/cron/tasks/tasks.go b/cron/tasks/tasks.go index 724b898..aa30eb4 100644 --- a/cron/tasks/tasks.go +++ b/cron/tasks/tasks.go @@ -24,7 +24,7 @@ const ( defaultRetryLimit = 3 ) -var log = logrus.WithField("package", "tasks") +var log = logrus.WithField("package", "cron/tasks") type Config struct { DB *pg.DB From d6a79d63fcc96e06cf6e2fc43d7a910277b06d0d Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Wed, 28 Apr 2021 19:51:48 +0200 Subject: [PATCH 25/31] don't inform about establishing the db connection --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 2f355d6..f1c5d8c 100644 --- a/main.go +++ b/main.go @@ -47,7 +47,6 @@ func main() { logrus.Warn(errors.Wrap(err, "Couldn't close the db connection")) } }() - logrus.Info("Connection with the database has been established") c, err := twhelpcron.New(&twhelpcron.Config{ DB: dbConn, From f70d608eab86c63011fec72cc3d40db4bad684da Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Wed, 28 Apr 2021 20:13:14 +0200 Subject: [PATCH 26/31] bump github.com/Kichiyaki/go-pg-logrus-query-logger/v10 --- db/db.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/db/db.go b/db/db.go index 59eb452..9fa5cf9 100644 --- a/db/db.go +++ b/db/db.go @@ -23,7 +23,7 @@ func New(cfg *Config) (*pg.DB, error) { if cfg != nil && cfg.LogQueries { db.AddQueryHook(gopglogrusquerylogger.QueryLogger{ - Entry: log, + Log: log, MaxQueryLength: 5000, }) } diff --git a/go.mod b/go.mod index ef15513..f10e40d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tribalwarshelp/cron go 1.16 require ( - github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 + github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9 github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 github.com/go-pg/pg/v10 v10.9.1 github.com/go-redis/redis/v8 v8.8.2 diff --git a/go.sum b/go.sum index 65be078..c8aa19f 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 h1:7IdSzhdupqm4AC3UDH9b5gdCDE2SlX6qkVC0zwqAuLA= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= +github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9 h1:S/08K0AD4bXYeSPJKei8ZbumDy1JNARZsgYbNZgr9Tk= +github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83/go.mod h1:+iGkf5HfOVeRVd9K7qQDucIl+/Kt3MyenMa90b/O/c4= github.com/aws/aws-sdk-go v1.35.28 h1:S2LuRnfC8X05zgZLC8gy/Sb82TGv2Cpytzbzz7tkeHc= From cb6e110e399db43ed28ed4709e282c2dda769489 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sat, 1 May 2021 07:24:02 +0200 Subject: [PATCH 27/31] move most of the code to the 'internal' directory --- {cron => internal/cron}/config.go | 0 {cron => internal/cron}/cron.go | 4 +- {cron => internal/cron}/queue/config.go | 0 {cron => internal/cron}/queue/queue.go | 0 {cron => internal/cron}/tasks/helpers.go | 0 {cron => internal/cron}/tasks/task.go | 2 +- .../task_load_servers_and_update_data.go | 4 +- ...sk_load_versions_and_update_server_data.go | 2 +- .../cron}/tasks/task_update_ennoblements.go | 2 +- .../cron}/tasks/task_update_history.go | 2 +- .../cron}/tasks/task_update_server_data.go | 0 .../tasks/task_update_server_ennoblements.go | 0 .../cron}/tasks/task_update_server_history.go | 0 .../cron}/tasks/task_update_server_stats.go | 0 .../cron}/tasks/task_update_stats.go | 2 +- {cron => internal/cron}/tasks/task_vacuum.go | 2 +- .../cron}/tasks/task_vacuum_server_db.go | 0 {cron => internal/cron}/tasks/tasks.go | 2 +- {db => internal/db}/db.go | 2 +- {db => internal/db}/sql_statements.go | 0 {utils => internal/utils}/env/env.go | 0 main.go | 38 +++++++++---------- 22 files changed, 31 insertions(+), 31 deletions(-) rename {cron => internal/cron}/config.go (100%) rename {cron => internal/cron}/cron.go (97%) rename {cron => internal/cron}/queue/config.go (100%) rename {cron => internal/cron}/queue/queue.go (100%) rename {cron => internal/cron}/tasks/helpers.go (100%) rename {cron => internal/cron}/tasks/task.go (91%) rename {cron => internal/cron}/tasks/task_load_servers_and_update_data.go (97%) rename {cron => internal/cron}/tasks/task_load_versions_and_update_server_data.go (93%) rename {cron => internal/cron}/tasks/task_update_ennoblements.go (95%) rename {cron => internal/cron}/tasks/task_update_history.go (96%) rename {cron => internal/cron}/tasks/task_update_server_data.go (100%) rename {cron => internal/cron}/tasks/task_update_server_ennoblements.go (100%) rename {cron => internal/cron}/tasks/task_update_server_history.go (100%) rename {cron => internal/cron}/tasks/task_update_server_stats.go (100%) rename {cron => internal/cron}/tasks/task_update_stats.go (96%) rename {cron => internal/cron}/tasks/task_vacuum.go (93%) rename {cron => internal/cron}/tasks/task_vacuum_server_db.go (100%) rename {cron => internal/cron}/tasks/tasks.go (98%) rename {db => internal/db}/db.go (98%) rename {db => internal/db}/sql_statements.go (100%) rename {utils => internal/utils}/env/env.go (100%) diff --git a/cron/config.go b/internal/cron/config.go similarity index 100% rename from cron/config.go rename to internal/cron/config.go diff --git a/cron/cron.go b/internal/cron/cron.go similarity index 97% rename from cron/cron.go rename to internal/cron/cron.go index 25fcb12..49b84df 100644 --- a/cron/cron.go +++ b/internal/cron/cron.go @@ -10,8 +10,8 @@ import ( "github.com/robfig/cron/v3" "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/cron/cron/queue" - "github.com/tribalwarshelp/cron/cron/tasks" + "github.com/tribalwarshelp/cron/internal/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/tasks" ) type Cron struct { diff --git a/cron/queue/config.go b/internal/cron/queue/config.go similarity index 100% rename from cron/queue/config.go rename to internal/cron/queue/config.go diff --git a/cron/queue/queue.go b/internal/cron/queue/queue.go similarity index 100% rename from cron/queue/queue.go rename to internal/cron/queue/queue.go diff --git a/cron/tasks/helpers.go b/internal/cron/tasks/helpers.go similarity index 100% rename from cron/tasks/helpers.go rename to internal/cron/tasks/helpers.go diff --git a/cron/tasks/task.go b/internal/cron/tasks/task.go similarity index 91% rename from cron/tasks/task.go rename to internal/cron/tasks/task.go index 86bc115..0d52cce 100644 --- a/cron/tasks/task.go +++ b/internal/cron/tasks/task.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type task struct { diff --git a/cron/tasks/task_load_servers_and_update_data.go b/internal/cron/tasks/task_load_servers_and_update_data.go similarity index 97% rename from cron/tasks/task_load_servers_and_update_data.go rename to internal/cron/tasks/task_load_servers_and_update_data.go index ec7aed0..a1500ad 100644 --- a/cron/tasks/task_load_servers_and_update_data.go +++ b/internal/cron/tasks/task_load_servers_and_update_data.go @@ -11,8 +11,8 @@ import ( "io/ioutil" "net/http" - "github.com/tribalwarshelp/cron/cron/queue" - "github.com/tribalwarshelp/cron/db" + "github.com/tribalwarshelp/cron/internal/cron/queue" + "github.com/tribalwarshelp/cron/internal/db" ) const ( diff --git a/cron/tasks/task_load_versions_and_update_server_data.go b/internal/cron/tasks/task_load_versions_and_update_server_data.go similarity index 93% rename from cron/tasks/task_load_versions_and_update_server_data.go rename to internal/cron/tasks/task_load_versions_and_update_server_data.go index 847efd5..0b86027 100644 --- a/cron/tasks/task_load_versions_and_update_server_data.go +++ b/internal/cron/tasks/task_load_versions_and_update_server_data.go @@ -5,7 +5,7 @@ import ( "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type taskLoadVersionsAndUpdateServerData struct { diff --git a/cron/tasks/task_update_ennoblements.go b/internal/cron/tasks/task_update_ennoblements.go similarity index 95% rename from cron/tasks/task_update_ennoblements.go rename to internal/cron/tasks/task_update_ennoblements.go index 8f08e91..00c57f9 100644 --- a/cron/tasks/task_update_ennoblements.go +++ b/internal/cron/tasks/task_update_ennoblements.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type taskUpdateEnnoblements struct { diff --git a/cron/tasks/task_update_history.go b/internal/cron/tasks/task_update_history.go similarity index 96% rename from cron/tasks/task_update_history.go rename to internal/cron/tasks/task_update_history.go index c5e115f..fef2520 100644 --- a/cron/tasks/task_update_history.go +++ b/internal/cron/tasks/task_update_history.go @@ -6,7 +6,7 @@ import ( "github.com/tribalwarshelp/shared/models" "time" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type taskUpdateHistory struct { diff --git a/cron/tasks/task_update_server_data.go b/internal/cron/tasks/task_update_server_data.go similarity index 100% rename from cron/tasks/task_update_server_data.go rename to internal/cron/tasks/task_update_server_data.go diff --git a/cron/tasks/task_update_server_ennoblements.go b/internal/cron/tasks/task_update_server_ennoblements.go similarity index 100% rename from cron/tasks/task_update_server_ennoblements.go rename to internal/cron/tasks/task_update_server_ennoblements.go diff --git a/cron/tasks/task_update_server_history.go b/internal/cron/tasks/task_update_server_history.go similarity index 100% rename from cron/tasks/task_update_server_history.go rename to internal/cron/tasks/task_update_server_history.go diff --git a/cron/tasks/task_update_server_stats.go b/internal/cron/tasks/task_update_server_stats.go similarity index 100% rename from cron/tasks/task_update_server_stats.go rename to internal/cron/tasks/task_update_server_stats.go diff --git a/cron/tasks/task_update_stats.go b/internal/cron/tasks/task_update_stats.go similarity index 96% rename from cron/tasks/task_update_stats.go rename to internal/cron/tasks/task_update_stats.go index 5a4c43b..edfb5ea 100644 --- a/cron/tasks/task_update_stats.go +++ b/internal/cron/tasks/task_update_stats.go @@ -6,7 +6,7 @@ import ( "github.com/tribalwarshelp/shared/models" "time" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type taskUpdateStats struct { diff --git a/cron/tasks/task_vacuum.go b/internal/cron/tasks/task_vacuum.go similarity index 93% rename from cron/tasks/task_vacuum.go rename to internal/cron/tasks/task_vacuum.go index 0dda3e7..d92291a 100644 --- a/cron/tasks/task_vacuum.go +++ b/internal/cron/tasks/task_vacuum.go @@ -5,7 +5,7 @@ import ( "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) type taskVacuum struct { diff --git a/cron/tasks/task_vacuum_server_db.go b/internal/cron/tasks/task_vacuum_server_db.go similarity index 100% rename from cron/tasks/task_vacuum_server_db.go rename to internal/cron/tasks/task_vacuum_server_db.go diff --git a/cron/tasks/tasks.go b/internal/cron/tasks/tasks.go similarity index 98% rename from cron/tasks/tasks.go rename to internal/cron/tasks/tasks.go index aa30eb4..60655a7 100644 --- a/cron/tasks/tasks.go +++ b/internal/cron/tasks/tasks.go @@ -6,7 +6,7 @@ import ( "github.com/sirupsen/logrus" "github.com/vmihailenco/taskq/v3" - "github.com/tribalwarshelp/cron/cron/queue" + "github.com/tribalwarshelp/cron/internal/cron/queue" ) const ( diff --git a/db/db.go b/internal/db/db.go similarity index 98% rename from db/db.go rename to internal/db/db.go index 9fa5cf9..546fc07 100644 --- a/db/db.go +++ b/internal/db/db.go @@ -9,7 +9,7 @@ import ( "github.com/sirupsen/logrus" "github.com/tribalwarshelp/shared/models" - envutils "github.com/tribalwarshelp/cron/utils/env" + envutils "github.com/tribalwarshelp/cron/internal/utils/env" ) var log = logrus.WithField("package", "db") diff --git a/db/sql_statements.go b/internal/db/sql_statements.go similarity index 100% rename from db/sql_statements.go rename to internal/db/sql_statements.go diff --git a/utils/env/env.go b/internal/utils/env/env.go similarity index 100% rename from utils/env/env.go rename to internal/utils/env/env.go diff --git a/main.go b/main.go index f1c5d8c..19b603a 100644 --- a/main.go +++ b/main.go @@ -12,9 +12,9 @@ import ( "github.com/sirupsen/logrus" "github.com/tribalwarshelp/shared/mode" - twhelpcron "github.com/tribalwarshelp/cron/cron" - "github.com/tribalwarshelp/cron/db" - envutils "github.com/tribalwarshelp/cron/utils/env" + twhelpcron "github.com/tribalwarshelp/cron/internal/cron" + "github.com/tribalwarshelp/cron/internal/db" + envutils "github.com/tribalwarshelp/cron/internal/utils/env" "github.com/joho/godotenv" "github.com/robfig/cron/v3" @@ -80,6 +80,22 @@ func main() { } } +func setupENVs() error { + err := os.Setenv("TZ", "UTC") + if err != nil { + return errors.Wrap(err, "setupENVs") + } + + if mode.Get() == mode.DevelopmentMode { + err := godotenv.Load(".env.local") + if err != nil { + return errors.Wrap(err, "setupENVs") + } + } + + return nil +} + func setupLogger() { if mode.Get() == mode.DevelopmentMode { logrus.SetLevel(logrus.DebugLevel) @@ -98,22 +114,6 @@ func setupLogger() { } } -func setupENVs() error { - err := os.Setenv("TZ", "UTC") - if err != nil { - return errors.Wrap(err, "setupENVs") - } - - if mode.Get() == mode.DevelopmentMode { - err := godotenv.Load(".env.local") - if err != nil { - return errors.Wrap(err, "setupENVs") - } - } - - return nil -} - func initializeRedis() (redis.UniversalClient, error) { client := redis.NewClient(&redis.Options{ Addr: envutils.GetenvString("REDIS_ADDR"), From 97e638697c0b0805d0a31ea436f584fe206f5dd1 Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sat, 1 May 2021 07:43:34 +0200 Subject: [PATCH 28/31] add a new task - taskDeleteNonExistentVillages --- .../task_delete_non_existent_villages.go | 45 +++++++++++++++++++ internal/cron/tasks/task_update_history.go | 19 ++++---- .../tasks/task_update_server_ennoblements.go | 2 +- .../cron/tasks/task_update_server_stats.go | 4 +- internal/cron/tasks/task_update_stats.go | 18 ++++---- internal/cron/tasks/tasks.go | 6 +++ 6 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 internal/cron/tasks/task_delete_non_existent_villages.go diff --git a/internal/cron/tasks/task_delete_non_existent_villages.go b/internal/cron/tasks/task_delete_non_existent_villages.go new file mode 100644 index 0000000..997eadf --- /dev/null +++ b/internal/cron/tasks/task_delete_non_existent_villages.go @@ -0,0 +1,45 @@ +package tasks + +import ( + "context" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + + "github.com/tribalwarshelp/cron/internal/cron/queue" +) + +type taskDeleteNonExistentVillages struct { + *task +} + +func (t *taskDeleteNonExistentVillages) execute() error { + var servers []*models.Server + err := t.db. + Model(&servers). + Where("status = ?", models.ServerStatusOpen). + Relation("Version"). + Select() + if err != nil { + err = errors.Wrap(err, "taskDeleteNonExistentVillages.execute") + log.Errorln(err) + return err + } + log. + WithField("numberOfServers", len(servers)). + Info("taskDeleteNonExistentVillages.execute: Servers have been loaded and added to the queue") + for _, server := range servers { + s := server + err := t.queue.Add(queue.MainQueue, Get(TaskNameServerDeleteNonExistentVillages).WithArgs(context.Background(), s)) + if err != nil { + log.Warn( + errors.Wrapf( + err, + "taskDeleteNonExistentVillages.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskNameServerDeleteNonExistentVillages, + ), + ) + } + } + return nil +} diff --git a/internal/cron/tasks/task_update_history.go b/internal/cron/tasks/task_update_history.go index fef2520..ca305c5 100644 --- a/internal/cron/tasks/task_update_history.go +++ b/internal/cron/tasks/task_update_history.go @@ -45,15 +45,16 @@ func (t *taskUpdateHistory) execute(timezone string) error { for _, server := range servers { s := server err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerHistory).WithArgs(context.Background(), timezone, s)) - log.Warn( - errors.Wrapf( - err, - "taskUpdateHistory.execute: %s: Couldn't add the task '%s' for this server", - server.Key, - TaskUpdateServerHistory, - ), - ) - + if err != nil { + log.Warn( + errors.Wrapf( + err, + "taskUpdateHistory.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerHistory, + ), + ) + } } return nil } diff --git a/internal/cron/tasks/task_update_server_ennoblements.go b/internal/cron/tasks/task_update_server_ennoblements.go index d359976..addd275 100644 --- a/internal/cron/tasks/task_update_server_ennoblements.go +++ b/internal/cron/tasks/task_update_server_ennoblements.go @@ -27,7 +27,7 @@ func (t *taskUpdateServerEnnoblements) execute(url string, server *models.Server entry.Error(err) return err } - entry.Debugf("%s: ennoblements has been updated", server.Key) + entry.Debugf("%s: ennoblements have been updated", server.Key) return nil } diff --git a/internal/cron/tasks/task_update_server_stats.go b/internal/cron/tasks/task_update_server_stats.go index 2c77307..180268b 100644 --- a/internal/cron/tasks/task_update_server_stats.go +++ b/internal/cron/tasks/task_update_server_stats.go @@ -127,7 +127,7 @@ func (w *workerUpdateServerStats) update() error { }(w.server) if _, err := tx.Model(stats).Insert(); err != nil { - return errors.Wrap(err, "cannot insert server stats") + return errors.Wrap(err, "couldn't insert server stats") } _, err = tx.Model(w.server). @@ -136,7 +136,7 @@ func (w *workerUpdateServerStats) update() error { Returning("*"). Update() if err != nil { - return errors.Wrap(err, "cannot update server") + return errors.Wrap(err, "couldn't update the server") } return tx.Commit() diff --git a/internal/cron/tasks/task_update_stats.go b/internal/cron/tasks/task_update_stats.go index edfb5ea..1b7aad3 100644 --- a/internal/cron/tasks/task_update_stats.go +++ b/internal/cron/tasks/task_update_stats.go @@ -45,14 +45,16 @@ func (t *taskUpdateStats) execute(timezone string) error { for _, server := range servers { s := server err := t.queue.Add(queue.MainQueue, Get(TaskUpdateServerStats).WithArgs(context.Background(), timezone, s)) - log.Warn( - errors.Wrapf( - err, - "taskUpdateStats.execute: %s: Couldn't add the task '%s' for this server", - server.Key, - TaskUpdateServerStats, - ), - ) + if err != nil { + log.Warn( + errors.Wrapf( + err, + "taskUpdateStats.execute: %s: Couldn't add the task '%s' for this server", + server.Key, + TaskUpdateServerStats, + ), + ) + } } return nil } diff --git a/internal/cron/tasks/tasks.go b/internal/cron/tasks/tasks.go index 60655a7..c2b1e8d 100644 --- a/internal/cron/tasks/tasks.go +++ b/internal/cron/tasks/tasks.go @@ -21,6 +21,8 @@ const ( TaskUpdateServerHistory = "updateServerHistory" TaskUpdateStats = "updateStats" TaskUpdateServerStats = "updateServerStats" + TaskNameDeleteNonExistentVillages = "deleteNonExistentVillages" + TaskNameServerDeleteNonExistentVillages = "serverDeleteNonExistentVillages" defaultRetryLimit = 3 ) @@ -86,6 +88,10 @@ func RegisterTasks(cfg *Config) error { Name: TaskUpdateServerStats, Handler: (&taskUpdateServerStats{t}).execute, }, + { + Name: TaskNameDeleteNonExistentVillages, + Handler: (&taskDeleteNonExistentVillages{t}).execute, + }, } for _, taskOptions := range options { opts := taskOptions From 01299b184115769aead5a10debf0d0022550f16a Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sat, 1 May 2021 08:27:20 +0200 Subject: [PATCH 29/31] add a new task - taskServerDeleteNonExistentVillages --- internal/cron/cron.go | 14 +++- internal/cron/tasks/helpers.go | 54 ++++++++----- .../task_delete_non_existent_villages.go | 12 ++- .../task_load_servers_and_update_data.go | 2 +- ...ask_server_delete_non_existent_villages.go | 77 +++++++++++++++++++ .../cron/tasks/task_update_server_data.go | 33 ++++---- .../cron/tasks/task_update_server_history.go | 2 +- .../cron/tasks/task_update_server_stats.go | 2 +- internal/cron/tasks/task_vacuum_server_db.go | 2 +- internal/cron/tasks/tasks.go | 4 + 10 files changed, 159 insertions(+), 43 deletions(-) create mode 100644 internal/cron/tasks/task_server_delete_non_existent_villages.go diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 49b84df..1f89068 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -71,6 +71,9 @@ func (c *Cron) init() error { if _, err := c.AddFunc("20 1 * * *", c.vacuumDatabase); err != nil { return err } + if _, err := c.AddFunc("10 1 * * *", c.deleteNonExistentVillages); err != nil { + return err + } if _, err := c.AddFunc("@every 1m", c.updateEnnoblements); err != nil { return err } @@ -79,10 +82,10 @@ func (c *Cron) init() error { c.updateServerData() c.vacuumDatabase() for _, fn := range updateHistoryFuncs { - go fn() + fn() } for _, fn := range updateStatsFuncs { - go fn() + fn() } }() } @@ -140,6 +143,13 @@ func (c *Cron) vacuumDatabase() { } } +func (c *Cron) deleteNonExistentVillages() { + err := c.queue.Add(queue.MainQueue, tasks.Get(tasks.TaskNameDeleteNonExistentVillages).WithArgs(context.Background())) + if err != nil { + c.logError("Cron.deleteNonExistentVillages", tasks.TaskNameDeleteNonExistentVillages, err) + } +} + func (c *Cron) logError(prefix string, taskName string, err error) { c.log.Error( errors.Wrapf( diff --git a/internal/cron/tasks/helpers.go b/internal/cron/tasks/helpers.go index 4d3c437..38efb20 100644 --- a/internal/cron/tasks/helpers.go +++ b/internal/cron/tasks/helpers.go @@ -46,45 +46,57 @@ func newDataloader(url string) dataloader.DataLoader { }) } -type tribesSearchableByID struct { - tribes []*models.Tribe -} - -func (searchable tribesSearchableByID) GetID(index int) int { - return searchable.tribes[index].ID -} - -func (searchable tribesSearchableByID) Len() int { - return len(searchable.tribes) -} - type playersSearchableByID struct { players []*models.Player } -func (searchable playersSearchableByID) GetID(index int) int { +func (searchable playersSearchableByID) getID(index int) int { return searchable.players[index].ID } -func (searchable playersSearchableByID) Len() int { +func (searchable playersSearchableByID) len() int { return len(searchable.players) } +type tribesSearchableByID struct { + tribes []*models.Tribe +} + +func (searchable tribesSearchableByID) getID(index int) int { + return searchable.tribes[index].ID +} + +func (searchable tribesSearchableByID) len() int { + return len(searchable.tribes) +} + +type villagesSearchableByID struct { + villages []*models.Village +} + +func (searchable villagesSearchableByID) getID(index int) int { + return searchable.villages[index].ID +} + +func (searchable villagesSearchableByID) len() int { + return len(searchable.villages) +} + type ennoblementsSearchableByNewOwnerID struct { ennoblements []*models.Ennoblement } -func (searchable ennoblementsSearchableByNewOwnerID) GetID(index int) int { +func (searchable ennoblementsSearchableByNewOwnerID) getID(index int) int { return searchable.ennoblements[index].NewOwnerID } -func (searchable ennoblementsSearchableByNewOwnerID) Len() int { +func (searchable ennoblementsSearchableByNewOwnerID) len() int { return len(searchable.ennoblements) } type searchableByID interface { - GetID(index int) int - Len() int + getID(index int) int + len() int } func makePlayersSearchable(players []*models.Player) searchableByID { @@ -101,19 +113,19 @@ func makeTribesSearchable(tribes []*models.Tribe) searchableByID { func searchByID(haystack searchableByID, id int) int { low := 0 - high := haystack.Len() - 1 + high := haystack.len() - 1 for low <= high { median := (low + high) / 2 - if haystack.GetID(median) < id { + if haystack.getID(median) < id { low = median + 1 } else { high = median - 1 } } - if low == haystack.Len() || haystack.GetID(low) != id { + if low == haystack.len() || haystack.getID(low) != id { return -1 } diff --git a/internal/cron/tasks/task_delete_non_existent_villages.go b/internal/cron/tasks/task_delete_non_existent_villages.go index 997eadf..79b8cfa 100644 --- a/internal/cron/tasks/task_delete_non_existent_villages.go +++ b/internal/cron/tasks/task_delete_non_existent_villages.go @@ -2,6 +2,7 @@ package tasks import ( "context" + "fmt" "github.com/pkg/errors" "github.com/tribalwarshelp/shared/models" @@ -16,6 +17,7 @@ func (t *taskDeleteNonExistentVillages) execute() error { var servers []*models.Server err := t.db. Model(&servers). + Relation("Version"). Where("status = ?", models.ServerStatusOpen). Relation("Version"). Select() @@ -29,7 +31,15 @@ func (t *taskDeleteNonExistentVillages) execute() error { Info("taskDeleteNonExistentVillages.execute: Servers have been loaded and added to the queue") for _, server := range servers { s := server - err := t.queue.Add(queue.MainQueue, Get(TaskNameServerDeleteNonExistentVillages).WithArgs(context.Background(), s)) + err := t.queue.Add( + queue.MainQueue, + Get(TaskNameServerDeleteNonExistentVillages). + WithArgs( + context.Background(), + fmt.Sprintf("https://%s.%s", server.Key, server.Version.Host), + s, + ), + ) if err != nil { log.Warn( errors.Wrapf( diff --git a/internal/cron/tasks/task_load_servers_and_update_data.go b/internal/cron/tasks/task_load_servers_and_update_data.go index a1500ad..28acc18 100644 --- a/internal/cron/tasks/task_load_servers_and_update_data.go +++ b/internal/cron/tasks/task_load_servers_and_update_data.go @@ -89,7 +89,7 @@ func (t *taskLoadServersAndUpdateData) execute(version *models.Version) error { func (t *taskLoadServersAndUpdateData) validatePayload(version *models.Version) error { if version == nil { - return errors.Errorf("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil") + return errors.New("taskLoadServersAndUpdateData.validatePayload: Expected *models.Version, got nil") } return nil } diff --git a/internal/cron/tasks/task_server_delete_non_existent_villages.go b/internal/cron/tasks/task_server_delete_non_existent_villages.go new file mode 100644 index 0000000..72c07ad --- /dev/null +++ b/internal/cron/tasks/task_server_delete_non_existent_villages.go @@ -0,0 +1,77 @@ +package tasks + +import ( + "github.com/go-pg/pg/v10" + "github.com/pkg/errors" + "github.com/tribalwarshelp/shared/models" + "github.com/tribalwarshelp/shared/tw/dataloader" +) + +type taskServerDeleteNonExistentVillages struct { + *task +} + +func (t *taskServerDeleteNonExistentVillages) execute(url string, server *models.Server) error { + if err := t.validatePayload(server); err != nil { + log.Debug(err) + return nil + } + entry := log.WithField("key", server.Key) + entry.Infof("taskServerDeleteNonExistentVillages.execute: %s: Deleting non-existent villages...", server.Key) + err := (&workerDeleteNonExistentVillages{ + db: t.db.WithParam("SERVER", pg.Safe(server.Key)), + dataloader: newDataloader(url), + server: server, + }).delete() + if err != nil { + err = errors.Wrap(err, "taskServerDeleteNonExistentVillages.execute") + entry.Error(err) + return err + } + entry.Infof("taskServerDeleteNonExistentVillages.execute: %s: Non-existent villages have been deleted", server.Key) + return nil +} + +func (t *taskServerDeleteNonExistentVillages) validatePayload(server *models.Server) error { + if server == nil { + return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil") + } + + return nil +} + +type workerDeleteNonExistentVillages struct { + db *pg.DB + dataloader dataloader.DataLoader + server *models.Server +} + +func (w *workerDeleteNonExistentVillages) delete() error { + var villagesFromDB []*models.Village + if err := w.db.Model(&villagesFromDB).Column("id").Select(); err != nil { + return errors.Wrap(err, "workerDeleteNonExistentVillages.delete") + } + villages, err := w.dataloader.LoadVillages() + if err != nil { + return errors.Wrap(err, "workerDeleteNonExistentVillages.delete") + } + var idsToDelete []int + searchableByVillageID := &villagesSearchableByID{villages} + for _, village := range villagesFromDB { + index := searchByID(searchableByVillageID, village.ID) + if index < 0 { + idsToDelete = append(idsToDelete, village.ID) + } + } + + totalDeleted := 0 + if len(idsToDelete) > 0 { + result, err := w.db.Model(&models.Village{}).Where("id = ANY(?)", pg.Array(idsToDelete)).Delete() + if err != nil { + return errors.Wrap(err, "workerDeleteNonExistentVillages.delete") + } + totalDeleted = result.RowsAffected() + } + log.Debugf("%s: deleted %d villages", w.server.Key, totalDeleted) + return nil +} diff --git a/internal/cron/tasks/task_update_server_data.go b/internal/cron/tasks/task_update_server_data.go index 04c8b17..c7aa313 100644 --- a/internal/cron/tasks/task_update_server_data.go +++ b/internal/cron/tasks/task_update_server_data.go @@ -43,7 +43,7 @@ func (t *taskUpdateServerData) execute(url string, server *models.Server) error func (t *taskUpdateServerData) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskUpdateServerData.validatePayload: Expected *models.Server, got nil") + return errors.New("taskUpdateServerData.validatePayload: Expected *models.Server, got nil") } return nil @@ -56,7 +56,7 @@ type workerUpdateServerData struct { } func (w *workerUpdateServerData) loadPlayers(od map[int]*models.OpponentsDefeated) ([]*models.Player, error) { - var ennoblements = []*models.Ennoblement{} + var ennoblements []*models.Ennoblement err := w.db.Model(&ennoblements).DistinctOn("new_owner_id").Order("new_owner_id ASC", "ennobled_at ASC").Select() if err != nil { return nil, errors.Wrap(err, "workerUpdateServerData.loadPlayers: couldn't load ennoblements") @@ -145,7 +145,7 @@ func (w *workerUpdateServerData) calculateTodaysTribeStats( func (w *workerUpdateServerData) calculateDailyPlayerStats(players []*models.Player, history []*models.PlayerHistory) []*models.DailyPlayerStats { - todaysStats := []*models.DailyPlayerStats{} + var todaysStats []*models.DailyPlayerStats searchablePlayers := makePlayersSearchable(players) for _, historyRecord := range history { @@ -168,42 +168,45 @@ func (w *workerUpdateServerData) calculateDailyPlayerStats(players []*models.Pla func (w *workerUpdateServerData) update() error { pod, err := w.dataloader.LoadOD(false) if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } + tod, err := w.dataloader.LoadOD(true) if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } villages, err := w.dataloader.LoadVillages() if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } numberOfVillages := len(villages) tribes, err := w.loadTribes(tod, countPlayerVillages(villages)) if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } numberOfTribes := len(tribes) players, err := w.loadPlayers(pod) if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } numberOfPlayers := len(players) cfg, err := w.dataloader.GetConfig() if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } + buildingCfg, err := w.dataloader.GetBuildingConfig() if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } + unitCfg, err := w.dataloader.GetUnitConfig() if err != nil { - return err + return errors.Wrap(err, "workerUpdateServerData.update") } tx, err := w.db.Begin() @@ -217,7 +220,7 @@ func (w *workerUpdateServerData) update() error { }(w.server) if len(tribes) > 0 { - ids := []int{} + var ids []int for _, tribe := range tribes { ids = append(ids, tribe.ID) } @@ -244,7 +247,7 @@ func (w *workerUpdateServerData) update() error { return errors.Wrap(err, "couldn't update non-existent tribes") } - tribesHistory := []*models.TribeHistory{} + var tribesHistory []*models.TribeHistory if err := w.db.Model(&tribesHistory). DistinctOn("tribe_id"). Column("*"). @@ -272,7 +275,7 @@ func (w *workerUpdateServerData) update() error { } if len(players) > 0 { - ids := []int{} + var ids []int for _, player := range players { ids = append(ids, player.ID) } @@ -297,7 +300,7 @@ func (w *workerUpdateServerData) update() error { return errors.Wrap(err, "couldn't update non-existent players") } - playerHistory := []*models.PlayerHistory{} + var playerHistory []*models.PlayerHistory if err := w.db.Model(&playerHistory). DistinctOn("player_id"). Column("*"). diff --git a/internal/cron/tasks/task_update_server_history.go b/internal/cron/tasks/task_update_server_history.go index df01bdc..4d16b17 100644 --- a/internal/cron/tasks/task_update_server_history.go +++ b/internal/cron/tasks/task_update_server_history.go @@ -41,7 +41,7 @@ func (t *taskUpdateServerHistory) execute(timezone string, server *models.Server func (t *taskUpdateServerHistory) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil") + return errors.New("taskUpdateServerHistory.validatePayload: Expected *models.Server, got nil") } return nil diff --git a/internal/cron/tasks/task_update_server_stats.go b/internal/cron/tasks/task_update_server_stats.go index 180268b..eee020f 100644 --- a/internal/cron/tasks/task_update_server_stats.go +++ b/internal/cron/tasks/task_update_server_stats.go @@ -41,7 +41,7 @@ func (t *taskUpdateServerStats) execute(timezone string, server *models.Server) func (t *taskUpdateServerStats) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil") + return errors.New("taskUpdateServerStats.validatePayload: Expected *models.Server, got nil") } return nil diff --git a/internal/cron/tasks/task_vacuum_server_db.go b/internal/cron/tasks/task_vacuum_server_db.go index 719f2e8..752cd34 100644 --- a/internal/cron/tasks/task_vacuum_server_db.go +++ b/internal/cron/tasks/task_vacuum_server_db.go @@ -38,7 +38,7 @@ func (t *taskVacuumServerDB) execute(server *models.Server) error { func (t *taskVacuumServerDB) validatePayload(server *models.Server) error { if server == nil { - return errors.Errorf("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil") + return errors.New("taskVacuumServerDB.validatePayload: Expected *models.Server, got nil") } return nil diff --git a/internal/cron/tasks/tasks.go b/internal/cron/tasks/tasks.go index c2b1e8d..a9c8d29 100644 --- a/internal/cron/tasks/tasks.go +++ b/internal/cron/tasks/tasks.go @@ -92,6 +92,10 @@ func RegisterTasks(cfg *Config) error { Name: TaskNameDeleteNonExistentVillages, Handler: (&taskDeleteNonExistentVillages{t}).execute, }, + { + Name: TaskNameServerDeleteNonExistentVillages, + Handler: (&taskServerDeleteNonExistentVillages{t}).execute, + }, } for _, taskOptions := range options { opts := taskOptions From 125c597a5681d0a79c77a0a5e63a30b26458b6ef Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sat, 1 May 2021 08:28:36 +0200 Subject: [PATCH 30/31] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index c8aa19f..8661edc 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7 h1:7IdSzhdupqm4AC3UDH9b5gdCDE2SlX6qkVC0zwqAuLA= -github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210423175217-c83fa01c60d7/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9 h1:S/08K0AD4bXYeSPJKei8ZbumDy1JNARZsgYbNZgr9Tk= github.com/Kichiyaki/go-pg-logrus-query-logger/v10 v10.0.0-20210428180109-fb97298564d9/go.mod h1:ADHVWnGlWcRn1aGthuh7I1Lrn6zzsjkVJju151dXyDw= github.com/Kichiyaki/go-php-serialize v0.0.0-20200601110855-47b6982acf83 h1:Oa8Bk4LNcknxw50gZOlvPwEreOlAbOnu7V82lUYNbOM= From fa23651644a01dabde061a1a8a4a4fdda357915c Mon Sep 17 00:00:00 2001 From: Kichiyaki Date: Sat, 1 May 2021 08:29:32 +0200 Subject: [PATCH 31/31] delete dev.sh --- dev.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 dev.sh diff --git a/dev.sh b/dev.sh deleted file mode 100755 index 16b66ca..0000000 --- a/dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -export MODE=development -go run main.go \ No newline at end of file