From 622940be68ba09b97e1fbaa605c2d4e520f01c57 Mon Sep 17 00:00:00 2001 From: Sakura Knoll Date: Thu, 28 Apr 2022 21:11:51 +0900 Subject: [PATCH] Add CN wiki data scrap script --- package-lock.json | 57 +++++++++++++++++++++++++++ package.json | 1 + src/scripts/downloadValkyrieData.ts | 61 +++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 src/scripts/downloadValkyrieData.ts diff --git a/package-lock.json b/package-lock.json index 6458383a..97a0350b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -794,6 +794,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.12.tgz", "integrity": "sha512-+2Iggwg7PxoO5Kyhvsq9VarmPbIelXP070HMImEpbtGCoyWNINQj4wzjbQCXzdHTRXnqufutJb5KAURZANNBAw==" }, + "@types/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/nprogress": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@types/nprogress/-/nprogress-0.2.0.tgz", @@ -1107,6 +1117,12 @@ "lodash": "^4.17.14" } }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, "available-typed-arrays": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", @@ -1421,6 +1437,15 @@ "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==" }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -1650,6 +1675,12 @@ "object-keys": "^1.0.12" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -2512,6 +2543,17 @@ "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -3356,6 +3398,21 @@ "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", "dev": true }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "requires": { + "mime-db": "1.52.0" + } + }, "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", diff --git a/package.json b/package.json index 1b293d68..5792bd2a 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "theme-ui": "^0.12.1" }, "devDependencies": { + "@types/node-fetch": "^2.6.1", "@types/nprogress": "^0.2.0", "@types/ramda": "^0.27.60", "@types/react": "^17.0.37", diff --git a/src/scripts/downloadValkyrieData.ts b/src/scripts/downloadValkyrieData.ts new file mode 100644 index 00000000..8de2a7a9 --- /dev/null +++ b/src/scripts/downloadValkyrieData.ts @@ -0,0 +1,61 @@ +import path from 'path' +import cheerio from 'cheerio' +import fetch from 'node-fetch' +import { existsSync, readFileSync, writeFileSync } from 'fs' + +async function run() { + const contentNumber = process.argv[2] + + const htmlFilePathname = path.join( + process.cwd(), + `dump/mihoyo-cn-wiki-${contentNumber}.html` + ) + const url = getUrl(contentNumber) + const data = await readOrFetch(htmlFilePathname, url) + + const $ = cheerio.load(data) + + const categories = $('.obc-tmpl-valkyrie__skill-item:nth-child(2n+1)') + + const skills: { title: string; description: string }[] = [] + categories.each((index, element) => { + const $el = cheerio.load(element) + const title = $el('.obc-tmpl-valkyrie__skill-item-title').text() + const description = $el('.obc-tmpl-valkyrie__skill-item-desc').text() + skills.push({ title, description }) + }) + + const avatarImageSrc = $('.obc-tmpl-character__avatar > img').attr('src') + + writeFileSync( + `dump/valkyrie-${contentNumber}.md`, + [ + `![](${avatarImageSrc})`, + '', + ...skills.map(({ title, description }) => { + return [`# ${title}`, '', description.trim(), ''].join('\n') + }), + ].join('\n') + ) +} + +function getUrl(contentNumber: string) { + return `https://bbs.mihoyo.com/bh3/wiki/content/${contentNumber}/detail` +} + +run() + .then(() => process.exit()) + .catch((error) => { + console.error(error) + process.exit(1) + }) + +async function readOrFetch(pathname: string, url: string) { + if (existsSync(pathname)) { + return readFileSync(pathname).toString() + } + const res = await fetch(url) + const data = await res.text() + writeFileSync(pathname, data) + return data +}