diff --git a/src/scraper/.DS_Store b/src/scraper/.DS_Store new file mode 100644 index 0000000..1b83a3b Binary files /dev/null and b/src/scraper/.DS_Store differ diff --git a/src/scraper/creatures.go b/src/scraper/creatures.go index e28801e..bd1fd72 100644 --- a/src/scraper/creatures.go +++ b/src/scraper/creatures.go @@ -2,9 +2,10 @@ package main import ( "fmt" - "github.com/PuerkitoBio/goquery" "log" "strings" + + "github.com/PuerkitoBio/goquery" ) func (d *Data) getCreatureDetails(i int, id string) { @@ -33,13 +34,34 @@ func (d *Data) getCreatureDetails(i int, id string) { } } - // Get the the creature's lore + // Get the the creature's lore and image URL if selection.AttrOr("id", "") == "ctl00_MainContent_DetailedOutput" { lore := selection.Text() lore = lore[:strings.Index(lore, "Creature ")] d.Creatures[i].Lore = lore + + /* + Get the creature's thumbnail image if it is in the expected node, + imediately after the name of the creature h, + Not all creatures has it. + TODO check if it is in the same positions for all creatures. + + Image URL is relative to domain: https://2e.aonprd.com/ + Es: https://2e.aonprd.com/Images/Monsters/Serpentfolk_AapophSerpentfolk.png + */ + + selection.Children().Each(func(j int, s *goquery.Selection) { + if s.HasClass("title") { + + if s.Next().Children().AttrOr("class", "") == "thumbnail" { + imageURL, _ := s.Next().Children().Attr("src") + d.Creatures[i].ImgURL = imageURL + } + } + }) + } - // TODO Get the creature's image + }) } diff --git a/src/scraper/fetch.go b/src/scraper/fetch.go index 927e458..4e40fdb 100644 --- a/src/scraper/fetch.go +++ b/src/scraper/fetch.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/PuerkitoBio/goquery" "io/ioutil" "log" "net/http" "strconv" "strings" + + "github.com/PuerkitoBio/goquery" ) func getAONCreatures(url string) string { @@ -65,4 +66,4 @@ func (d *Data) parseAONTable(data string) { // Drop first entry which is the table header row d.Creatures = d.Creatures[1:] -} \ No newline at end of file +} diff --git a/src/scraper/go.mod b/src/scraper/go.mod index 6e664ca..67a155c 100644 --- a/src/scraper/go.mod +++ b/src/scraper/go.mod @@ -3,8 +3,15 @@ module pf2-encounter go 1.15 require ( - github.com/PuerkitoBio/goquery v1.6.0 + github.com/PuerkitoBio/goquery v1.6.1 + github.com/andybalholm/cascadia v1.2.0 // indirect github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e github.com/davecgh/go-spew v1.1.1 github.com/helloeave/json v1.15.3 + github.com/yuin/goldmark v1.3.7 // indirect + golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect + golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect + golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea // indirect + golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect + golang.org/x/tools v0.1.2 // indirect ) diff --git a/src/scraper/go.sum b/src/scraper/go.sum index b90eacd..b51abf5 100644 --- a/src/scraper/go.sum +++ b/src/scraper/go.sum @@ -1,16 +1,54 @@ github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= +github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e h1:V9a67dfYqPLAvzk5hMQOXYJlZ4SLIXgyKIE+ZiHzgGQ= github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/helloeave/json v1.15.3 h1:roUxUEGhsSvhuhi80c4qmLiW633d5uf0mkzUGzBMfX8= github.com/helloeave/json v1.15.3/go.mod h1:uTHhuUsgnrpm9cc7Gi3tfIUwgf1dq/7+uLfpUFLBFEQ= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.3.7 h1:NSaHgaeJFCtWXCBkBKXw0rhgMuJ0VoE9FB5mWldcrQ4= +github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea h1:+WiDlPBBaO+h9vPNZi8uJ3k4BkKQB7Iow3aqwHVA5hI= +golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/src/scraper/main.go b/src/scraper/main.go index 3990a12..229c4ce 100644 --- a/src/scraper/main.go +++ b/src/scraper/main.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/davecgh/go-spew/spew" - "github.com/helloeave/json" "io" "io/ioutil" "log" "os" + + "github.com/davecgh/go-spew/spew" + "github.com/helloeave/json" ) // baseURL for creatures pages @@ -29,6 +30,7 @@ type Creature struct { Rarity string `json:"rarity"` Id string `json:"id"` Lore string `json:"lore"` + ImgURL string `json:image_url` } type metadata struct { @@ -75,7 +77,7 @@ func main() { // Pretty print the result, DEBUG only if os.Getenv("DEBUG") == "1" { - fmt.Printf("Found %i creatures\n", len(d.Creatures)) + fmt.Printf("Found %d creatures\n", len(d.Creatures)) // Just spew a bunch of them, we don't need to check them all! spew.Dump(d.Creatures[:2]) } diff --git a/src/scraper/metadata.go b/src/scraper/metadata.go index 5c960d4..5ccdf85 100644 --- a/src/scraper/metadata.go +++ b/src/scraper/metadata.go @@ -51,5 +51,4 @@ func (d *Data) FillMetadata() { return d.Metadata.Sizes[i] < d.Metadata.Sizes[j] }) - } diff --git a/src/webui/package.json b/src/webui/package.json index 556c01b..5c9cc58 100644 --- a/src/webui/package.json +++ b/src/webui/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@quasar/extras": "^1.0.0", - "axios": "^0.18.1", + "axios": "^0.21.1", "core-js": "^3.6.5", "quasar": "^1.0.0", "vue-tour": "^1.5.0" diff --git a/src/webui/yarn.lock b/src/webui/yarn.lock index 839709c..bf26250 100644 --- a/src/webui/yarn.lock +++ b/src/webui/yarn.lock @@ -1823,13 +1823,12 @@ autoprefixer@9.8.6: postcss "^7.0.32" postcss-value-parser "^4.1.0" -axios@^0.18.1: - version "0.18.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3" - integrity sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g== +axios@^0.21.1: + version "0.21.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8" + integrity sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA== dependencies: - follow-redirects "1.5.10" - is-buffer "^2.0.2" + follow-redirects "^1.10.0" babel-eslint@^10.0.1: version "10.1.0" @@ -3021,13 +3020,6 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8: dependencies: ms "2.0.0" -debug@=3.1.0, debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - debug@^3.1.0, debug@^3.1.1, debug@^3.2.5: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" @@ -3042,6 +3034,13 @@ debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: dependencies: ms "2.1.2" +debug@~3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== + dependencies: + ms "2.0.0" + decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -4019,14 +4018,7 @@ flush-write-stream@^1.0.0: inherits "^2.0.3" readable-stream "^2.3.6" -follow-redirects@1.5.10: - version "1.5.10" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" - integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== - dependencies: - debug "=3.1.0" - -follow-redirects@^1.0.0: +follow-redirects@^1.0.0, follow-redirects@^1.10.0: version "1.13.1" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.1.tgz#5f69b813376cee4fd0474a3aba835df04ab763b7" integrity sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg== @@ -4813,11 +4805,6 @@ is-buffer@^1.1.5: resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== -is-buffer@^2.0.2: - version "2.0.5" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" - integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== - is-callable@^1.1.4, is-callable@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.2.tgz#c7c6715cd22d4ddb48d3e19970223aceabb080d9"