Add data-fetch tool used to fetch unknown spell metadata from wowhead <3

package.json

@@ -24,7 +24,8 @@
     "test": "react-scripts test",
     "eject": "react-scripts eject",
     "predeploy": "yarn run build",
-    "deploy": "gh-pages -d build"
+    "deploy": "gh-pages -d build",
+    "fetch-spells": "ts-node --files --project src/tools/tsconfig.json src/tools/data-fetch.ts"
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -42,7 +43,12 @@
     ]
   },
   "devDependencies": {
+    "@types/cheerio": "^0.22.12",
+    "@types/node-fetch": "^2.5.0",
     "@welldone-software/why-did-you-render": "^3.2.1",
-    "gh-pages": "^2.0.1"
+    "cheerio": "^1.0.0-rc.3",
+    "gh-pages": "^2.0.1",
+    "node-fetch": "^2.6.0",
+    "ts-node": "^8.3.0"
   }
 }

src/components/Calculator.tsx

@@ -9,7 +9,7 @@ import {
 import { talentsBySpec } from '../data/talents'
 import { classByName } from '../data/classes'
 import { History } from 'history'
-import { debugPrintKnown } from '../lib/debug'
+// import { debugPrintKnown } from '../lib/debug'
 import { Link } from 'react-router-dom';
 
 interface Props {
@@ -59,7 +59,7 @@ export class Calculator extends React.PureComponent<Props> {
     this.setState({ knownTalents: newKnownTalents })
 
     // Debug
-    debugPrintKnown(newKnownTalents)
+    // debugPrintKnown(newKnownTalents)
   }
 
   render() {

src/lib/tree.ts

@@ -204,7 +204,6 @@ export function decodeKnownTalents(pointString: string, className: string): Map<
   for (let i = 0; i < parts.length; i++) {
     const specId = specs[i]
     const specPointStr = parts[i]
-    console.log(specPointStr, { specId })
     const talents = talentsBySpecArray[specId].sort(SORT_TALENTS)
 
     for (let y = 0; y < specPointStr.length; y++) {

src/tools/data-fetch.ts

@@ -0,0 +1,65 @@
+// import '../src/types'
+import { talentsById } from '../data/talents'
+import fetch from 'node-fetch'
+import fs from 'fs'
+import cheerio from 'cheerio'
+import spells from '../data/spells.json'
+
+const getRankFromTooltip = (spellId: number, tooltip: string): number | null => {
+  const matches = tooltip.match(/Rank (\d)/)
+  if (!matches) {
+    console.warn(`Spell ${spellId}: No rank found in tooltip: \n\t`, tooltip)
+    return null
+  }
+  return parseInt(matches[1], 10)
+}
+
+const getUnknownSpellIds = (): number[] => {
+  const allSpellIds = Object.values(talentsById).reduce((reduction, talent: TalentData) => {
+    reduction = [...reduction, ...talent.ranks]
+    return reduction
+  }, [])
+
+  return allSpellIds.filter(spellId => !spells[spellId.toString()])
+}
+
+export const fetchSpellData = async () => {
+  const unknownSpellIds = getUnknownSpellIds()
+  console.log(`Fetching spell data for ${unknownSpellIds.length} unknown spells`)
+
+  const promises = unknownSpellIds.slice(0, 20).map(async (spellId): Promise<boolean> => {
+    if (!spells[spellId.toString()]) {
+      console.log(`Loading data for unknown spell: ${spellId}`)
+
+      const res = await fetch(`https://classic.wowhead.com/tooltip/spell/${spellId}`)
+      if (res.status !== 200) {
+        console.error(`Could not find data for spell: ${spellId}`)
+        return null
+      }
+
+      const json: any = await res.json()
+      const $ = cheerio.load(json.tooltip)
+
+      const spell: SpellData = {
+        name: json.name,
+        icon: json.icon,
+        rank: getRankFromTooltip(spellId, json.tooltip),
+        description: $('.q').text()
+      }
+
+      spells[spellId.toString()] = spell 
+      return true
+    }
+  })
+
+  await Promise.all(promises)
+
+  fs.writeFileSync('./src/data/spells.json', JSON.stringify(spells, null, 2))
+
+  if (getUnknownSpellIds().length > 0) {
+    console.log('--- We have more unknown spells: ', getUnknownSpellIds().length)
+    fetchSpellData()
+  }
+}
+
+fetchSpellData()

src/tools/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "preserve"
+  },
+  "include": [
+    "../types.d.ts"
+  ],
+  "exclude": [
+    "../src/lib/foo.ts"
+  ]
+}

src/types.d.ts

@@ -5,6 +5,14 @@ interface TalentTree {
   talents: Talent[]
 }
 
+interface SpellData {
+  id?: number
+  name: string
+  rank: number
+  icon: string
+  description: string
+}
+
 interface ClassData {
   id: number
   name: string
@@ -41,4 +49,4 @@ interface Talent {
   prerequisite?: [number, number] | number // [row, column] OR index 
 }
 
-type TalentClickHandler = (specId: number, talentId: number, modifier: 1 | -1) => void
+type TalentClickHandler = (specId: number, talentId: number, modifier: 1 | -1) => void

yarn.lock

@@ -1271,6 +1271,13 @@
   dependencies:
     "@babel/types" "^7.3.0"
 
+"@types/cheerio@^0.22.12":
+  version "0.22.12"
+  resolved "http://npm.soundtrackyourbrand.com/@types%2fcheerio/-/cheerio-0.22.12.tgz#93c050401d4935a5376e8b352965f7458bed5340"
+  integrity sha512-aczowyAJNfrkBV+HS8DyAA87OnvkqGrrOmm5s7V6Jbgimzv/1ZoAy91cLJX8GQrUS60KufD7EIzA2LbK8HV4hg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/history@*":
   version "4.7.2"
   resolved "http://npm.soundtrackyourbrand.com/@types%2fhistory/-/history-4.7.2.tgz#0e670ea254d559241b6eeb3894f8754991e73220"
@@ -1308,6 +1315,18 @@
   dependencies:
     "@types/jest-diff" "*"
 
+"@types/node-fetch@^2.5.0":
+  version "2.5.0"
+  resolved "http://npm.soundtrackyourbrand.com/@types%2fnode-fetch/-/node-fetch-2.5.0.tgz#1c55616a4591bdd15a389fbd0da4a55b9502add5"
+  integrity sha512-TLFRywthBgL68auWj+ziWu+vnmmcHCDFC/sqCOQf1xTz4hRq8cu79z8CtHU9lncExGBsB8fXA4TiLDLt6xvMzw==
+  dependencies:
+    "@types/node" "*"
+
+"@types/node@*":
+  version "12.6.8"
+  resolved "http://npm.soundtrackyourbrand.com/@types%2fnode/-/node-12.6.8.tgz#e469b4bf9d1c9832aee4907ba8a051494357c12c"
+  integrity sha512-aX+gFgA5GHcDi89KG5keey2zf0WfZk/HAQotEamsK2kbey+8yGKcson0hbK8E+v0NArlCJQCqMP161YhV6ZXLg==
+
 "@types/node@12.6.2":
   version "12.6.2"
   resolved "http://npm.soundtrackyourbrand.com/@types%2fnode/-/node-12.6.2.tgz#a5ccec6abb6060d5f20d256fb03ed743e9774999"
@@ -1709,6 +1728,11 @@ are-we-there-yet@~1.1.2:
     delegates "^1.0.0"
     readable-stream "^2.0.6"
 
+arg@^4.1.0:
+  version "4.1.1"
+  resolved "http://npm.soundtrackyourbrand.com/arg/-/arg-4.1.1.tgz#485f8e7c390ce4c5f78257dbea80d4be11feda4c"
+  integrity sha512-SlmP3fEA88MBv0PypnXZ8ZfJhwmDeIE3SP71j37AiXQBXYosPV0x6uISAaHYSlSVhmHOVkomen0tbGk6Anlebw==
+
 argparse@^1.0.7:
   version "1.0.10"
   resolved "http://npm.soundtrackyourbrand.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -2495,6 +2519,18 @@ chardet@^0.7.0:
   resolved "http://npm.soundtrackyourbrand.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
   integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
 
+cheerio@^1.0.0-rc.3:
+  version "1.0.0-rc.3"
+  resolved "http://npm.soundtrackyourbrand.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6"
+  integrity sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==
+  dependencies:
+    css-select "~1.2.0"
+    dom-serializer "~0.1.1"
+    entities "~1.1.1"
+    htmlparser2 "^3.9.1"
+    lodash "^4.15.0"
+    parse5 "^3.0.1"
+
 chokidar@^2.0.0, chokidar@^2.0.2, chokidar@^2.0.4:
   version "2.1.6"
   resolved "http://npm.soundtrackyourbrand.com/chokidar/-/chokidar-2.1.6.tgz#b6cad653a929e244ce8a834244164d241fa954c5"
@@ -2985,7 +3021,7 @@ css-select-base-adapter@^0.1.1:
   resolved "http://npm.soundtrackyourbrand.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz#3b2ff4972cc362ab88561507a95408a1432135d7"
   integrity sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w==
 
-css-select@^1.1.0:
+css-select@^1.1.0, css-select@~1.2.0:
   version "1.2.0"
   resolved "http://npm.soundtrackyourbrand.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
   integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
@@ -3334,6 +3370,11 @@ diff-sequences@^24.3.0:
   resolved "http://npm.soundtrackyourbrand.com/diff-sequences/-/diff-sequences-24.3.0.tgz#0f20e8a1df1abddaf4d9c226680952e64118b975"
   integrity sha512-xLqpez+Zj9GKSnPWS0WZw1igGocZ+uua8+y+5dDNTT934N3QuY1sp2LkHzwiaYQGz60hMq0pjAshdeXm5VUOEw==
 
+diff@^4.0.1:
+  version "4.0.1"
+  resolved "http://npm.soundtrackyourbrand.com/diff/-/diff-4.0.1.tgz#0c667cb467ebbb5cea7f14f135cc2dba7780a8ff"
+  integrity sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==
+
 diffie-hellman@^5.0.0:
   version "5.0.3"
   resolved "http://npm.soundtrackyourbrand.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
@@ -3400,7 +3441,7 @@ dom-converter@^0.2:
   dependencies:
     utila "~0.4"
 
-dom-serializer@0:
+dom-serializer@0, dom-serializer@~0.1.1:
   version "0.1.1"
   resolved "http://npm.soundtrackyourbrand.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
   integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
@@ -3547,7 +3588,7 @@ enhanced-resolve@^4.1.0:
     memory-fs "^0.4.0"
     tapable "^1.0.0"
 
-entities@^1.1.1:
+entities@^1.1.1, entities@~1.1.1:
   version "1.1.2"
   resolved "http://npm.soundtrackyourbrand.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
   integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
@@ -4713,7 +4754,7 @@ html-webpack-plugin@4.0.0-beta.5:
     tapable "^1.1.0"
     util.promisify "1.0.0"
 
-htmlparser2@^3.3.0:
+htmlparser2@^3.3.0, htmlparser2@^3.9.1:
   version "3.10.1"
   resolved "http://npm.soundtrackyourbrand.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
   integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
@@ -6167,7 +6208,7 @@ lodash.uniq@^4.5.0:
   resolved "http://npm.soundtrackyourbrand.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
   integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
 
-"lodash@>=3.5 <5", lodash@^4, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.5, lodash@~4.17.10:
+"lodash@>=3.5 <5", lodash@^4, lodash@^4.0.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.5, lodash@~4.17.10:
   version "4.17.15"
   resolved "http://npm.soundtrackyourbrand.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
   integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
@@ -6220,6 +6261,11 @@ make-dir@^2.0.0, make-dir@^2.1.0:
     pify "^4.0.1"
     semver "^5.6.0"
 
+make-error@^1.1.1:
+  version "1.3.5"
+  resolved "http://npm.soundtrackyourbrand.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"
+  integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==
+
 makeerror@1.0.x:
   version "1.0.11"
   resolved "http://npm.soundtrackyourbrand.com/makeerror/-/makeerror-1.0.11.tgz#e01a5c9109f2af79660e4e8b9587790184f5a96c"
@@ -6615,6 +6661,11 @@ no-case@^2.2.0:
   dependencies:
     lower-case "^1.1.1"
 
+node-fetch@^2.6.0:
+  version "2.6.0"
+  resolved "http://npm.soundtrackyourbrand.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
 node-forge@0.7.5:
   version "0.7.5"
   resolved "http://npm.soundtrackyourbrand.com/node-forge/-/node-forge-0.7.5.tgz#6c152c345ce11c52f465c2abd957e8639cd674df"
@@ -7178,6 +7229,13 @@ parse5@5.1.0:
   resolved "http://npm.soundtrackyourbrand.com/parse5/-/parse5-5.1.0.tgz#c59341c9723f414c452975564c7c00a68d58acd2"
   integrity sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==
 
+parse5@^3.0.1:
+  version "3.0.3"
+  resolved "http://npm.soundtrackyourbrand.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
+  integrity sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==
+  dependencies:
+    "@types/node" "*"
+
 parseurl@~1.3.2, parseurl@~1.3.3:
   version "1.3.3"
   resolved "http://npm.soundtrackyourbrand.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@@ -9813,6 +9871,17 @@ trim-right@^1.0.1:
   dependencies:
     glob "^7.1.2"
 
+ts-node@^8.3.0:
+  version "8.3.0"
+  resolved "http://npm.soundtrackyourbrand.com/ts-node/-/ts-node-8.3.0.tgz#e4059618411371924a1fb5f3b125915f324efb57"
+  integrity sha512-dyNS/RqyVTDcmNM4NIBAeDMpsAdaQ+ojdf0GOLqE6nwJOgzEkdRNzJywhDfwnuvB10oa6NLVG1rUJQCpRN7qoQ==
+  dependencies:
+    arg "^4.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    source-map-support "^0.5.6"
+    yn "^3.0.0"
+
 ts-pnp@1.1.2, ts-pnp@^1.0.0:
   version "1.1.2"
   resolved "http://npm.soundtrackyourbrand.com/ts-pnp/-/ts-pnp-1.1.2.tgz#be8e4bfce5d00f0f58e0666a82260c34a57af552"
@@ -10635,3 +10704,8 @@ yargs@^7.0.0:
     which-module "^1.0.0"
     y18n "^3.2.1"
     yargs-parser "^5.0.0"
+
+yn@^3.0.0:
+  version "3.1.0"
+  resolved "http://npm.soundtrackyourbrand.com/yn/-/yn-3.1.0.tgz#fcbe2db63610361afcc5eb9e0ac91e976d046114"
+  integrity sha512-kKfnnYkbTfrAdd0xICNFw7Atm8nKpLcLv9AZGEt+kczL/WQVai4e2V6ZN8U/O+iI6WrNuJjNNOyu4zfhl9D3Hg==