diff --git a/web/src/robots.js b/web/src/robots.js new file mode 100644 index 000000000..c76140386 --- /dev/null +++ b/web/src/robots.js @@ -0,0 +1,12 @@ +const fs = require('fs'); +const path = require('path'); + +let robots; +async function getRobots(ctx) { + if (!robots) { + robots = fs.readFileSync(path.join(__dirname, '/../dist/public/robots.txt'), 'utf8'); + } + return robots; +} + +module.exports = { getRobots }; diff --git a/web/webpack.config.js b/web/webpack.config.js index 33ac21b47..8381d51b5 100644 --- a/web/webpack.config.js +++ b/web/webpack.config.js @@ -61,6 +61,15 @@ if (fs.existsSync(CUSTOM_OG_PATH)) { }); } +const ROBOTS_TXT_PATH = `${CUSTOM_ROOT}/robots.txt`; +if (fs.existsSync(ROBOTS_TXT_PATH)) { + copyWebpackCommands.push({ + from: ROBOTS_TXT_PATH, + to: `${DIST_ROOT}/robots.txt`, + force: true, + }); +} + let plugins = [ new WriteFilePlugin(), new CopyWebpackPlugin(copyWebpackCommands),