Add CN wiki data scrap script

package-lock.json

@@ -794,6 +794,16 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz",
       "integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw=="
     },
+    "@types/node-fetch": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.1.tgz",
+      "integrity": "sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      }
+    },
     "@types/nprogress": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/@types/nprogress/-/nprogress-0.2.0.tgz",
@@ -1107,6 +1117,12 @@
         "lodash": "^4.17.14"
       }
     },
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
+      "dev": true
+    },
     "available-typed-arrays": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
@@ -1421,6 +1437,15 @@
       "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz",
       "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g=="
     },
+    "combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dev": true,
+      "requires": {
+        "delayed-stream": "~1.0.0"
+      }
+    },
     "commondir": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
@@ -1650,6 +1675,12 @@
         "object-keys": "^1.0.12"
       }
     },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
+      "dev": true
+    },
     "depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -2512,6 +2543,17 @@
       "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
       "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
     },
+    "form-data": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "dev": true,
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      }
+    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -3356,6 +3398,21 @@
       "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
       "dev": true
     },
+    "mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "dev": true
+    },
+    "mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dev": true,
+      "requires": {
+        "mime-db": "1.52.0"
+      }
+    },
     "minimalistic-assert": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",

package.json

@@ -18,6 +18,7 @@
     "theme-ui": "^0.12.1"
   },
   "devDependencies": {
+    "@types/node-fetch": "^2.6.1",
     "@types/nprogress": "^0.2.0",
     "@types/ramda": "^0.27.60",
     "@types/react": "^17.0.37",

src/scripts/downloadValkyrieData.ts

@@ -0,0 +1,61 @@
+import path from 'path'
+import cheerio from 'cheerio'
+import fetch from 'node-fetch'
+import { existsSync, readFileSync, writeFileSync } from 'fs'
+
+async function run() {
+  const contentNumber = process.argv[2]
+
+  const htmlFilePathname = path.join(
+    process.cwd(),
+    `dump/mihoyo-cn-wiki-${contentNumber}.html`
+  )
+  const url = getUrl(contentNumber)
+  const data = await readOrFetch(htmlFilePathname, url)
+
+  const $ = cheerio.load(data)
+
+  const categories = $('.obc-tmpl-valkyrie__skill-item:nth-child(2n+1)')
+
+  const skills: { title: string; description: string }[] = []
+  categories.each((index, element) => {
+    const $el = cheerio.load(element)
+    const title = $el('.obc-tmpl-valkyrie__skill-item-title').text()
+    const description = $el('.obc-tmpl-valkyrie__skill-item-desc').text()
+    skills.push({ title, description })
+  })
+
+  const avatarImageSrc = $('.obc-tmpl-character__avatar > img').attr('src')
+
+  writeFileSync(
+    `dump/valkyrie-${contentNumber}.md`,
+    [
+      `![](${avatarImageSrc})`,
+      '',
+      ...skills.map(({ title, description }) => {
+        return [`# ${title}`, '', description.trim(), ''].join('\n')
+      }),
+    ].join('\n')
+  )
+}
+
+function getUrl(contentNumber: string) {
+  return `https://bbs.mihoyo.com/bh3/wiki/content/${contentNumber}/detail`
+}
+
+run()
+  .then(() => process.exit())
+  .catch((error) => {
+    console.error(error)
+    process.exit(1)
+  })
+
+async function readOrFetch(pathname: string, url: string) {
+  if (existsSync(pathname)) {
+    return readFileSync(pathname).toString()
+  }
+  const res = await fetch(url)
+  const data = await res.text()
+  writeFileSync(pathname, data)
+  return data
+}