From 550d285a8b663d0a480642f3f38ff1347857cc04 Mon Sep 17 00:00:00 2001 From: Joe Carstairs Date: Tue, 24 Mar 2026 22:02:07 +0000 Subject: [PATCH] adds extended-glob loader --- website/src/loaders/extended-glob.d.ts | 18 ++ website/src/loaders/extended-glob.js | 306 +++++++++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 website/src/loaders/extended-glob.d.ts create mode 100644 website/src/loaders/extended-glob.js diff --git a/website/src/loaders/extended-glob.d.ts b/website/src/loaders/extended-glob.d.ts new file mode 100644 index 0000000..87a4814 --- /dev/null +++ b/website/src/loaders/extended-glob.d.ts @@ -0,0 +1,18 @@ +/** + * This file is edited from astro/loaders/glob.d.ts + */ + +import type { glob, Loader } from "astro/loaders"; +type GlobOptions = Parameters[0]; +/** + * Loads multiple entries, using a glob pattern to match files. + * @param pattern A glob pattern to match files, relative to the content directory. + * @param ignore Glob patterns to exclude from the matched results. + */ +export declare function extendedGlob( + globOptions: GlobOptions & { + postprocessSlug?: (slug: string) => string; + ignore?: string[]; + }, +): Loader; +export {}; diff --git a/website/src/loaders/extended-glob.js b/website/src/loaders/extended-glob.js new file mode 100644 index 0000000..53e36b6 --- /dev/null +++ b/website/src/loaders/extended-glob.js @@ -0,0 +1,306 @@ +/** + * This file is edited from astro/loaders/glob.js + */ + +import { existsSync, promises as fs } from "node:fs"; +import { relative } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import pLimit from "p-limit"; +import colors from "piccolore"; +import picomatch from "picomatch"; +import { glob as tinyglobby } from "tinyglobby"; +import { + getContentEntryIdAndSlug, + posixRelative, +} from "../../node_modules/astro/dist/content/utils.js"; +function generateIdDefault({ entry, base, data }, postprocessSlug, isLegacy) { + if (data.slug) { + return data.slug; + } + const entryURL = new URL(encodeURI(entry), base); + if (isLegacy) { + const { id } = getContentEntryIdAndSlug({ + entry: entryURL, + contentDir: base, + collection: "", + }); + return id; + } + const { slug } = getContentEntryIdAndSlug({ + entry: entryURL, + contentDir: base, + collection: "", + }); + if (postprocessSlug) { + return postprocessSlug(slug); + } + return slug; +} +function checkPrefix(pattern, prefix) { + if (Array.isArray(pattern)) { + return pattern.some((p) => p.startsWith(prefix)); + } + return pattern.startsWith(prefix); +} +const secretLegacyFlag = /* @__PURE__ */ Symbol("astro.legacy-glob"); +function extendedGlob(globOptions) { + if (checkPrefix(globOptions.pattern, "../")) { + throw new Error( + "Glob patterns cannot start with `../`. Set the `base` option to a parent directory instead.", + ); + } + if (checkPrefix(globOptions.pattern, "/")) { + throw new Error( + "Glob patterns cannot start with `/`. Set the `base` option to a parent directory or use a relative path instead.", + ); + } + const isLegacy = !!globOptions[secretLegacyFlag]; + const generateId = + globOptions?.generateId ?? + ((opts) => generateIdDefault(opts, globOptions?.postprocessSlug, isLegacy)); + const fileToIdMap = /* @__PURE__ */ new Map(); + const ignore = globOptions.ignore || []; + return { + name: "extended-glob-loader", + load: async ({ + config, + collection, + logger, + watcher, + parseData, + store, + generateDigest, + entryTypes, + }) => { + const renderFunctionByContentType = /* @__PURE__ */ new WeakMap(); + const untouchedEntries = new Set(store.keys()); + async function syncData(entry, base, entryType, oldId) { + if (!entryType) { + logger.warn(`No entry type found for ${entry}`); + return; + } + const fileUrl = new URL(encodeURI(entry), base); + const contents = await fs.readFile(fileUrl, "utf-8").catch((err) => { + logger.error(`Error reading ${entry}: ${err.message}`); + return; + }); + if (!contents && contents !== "") { + logger.warn(`No contents found for ${entry}`); + return; + } + const { body, data } = await entryType.getEntryInfo({ + contents, + fileUrl, + }); + const id = generateId({ entry, base, data }); + if (oldId && oldId !== id) { + store.delete(oldId); + } + untouchedEntries.delete(id); + const existingEntry = store.get(id); + const digest = generateDigest(contents); + const filePath2 = fileURLToPath(fileUrl); + if ( + existingEntry && + existingEntry.digest === digest && + existingEntry.filePath + ) { + if (existingEntry.deferredRender) { + store.addModuleImport(existingEntry.filePath); + } + if (existingEntry.assetImports?.length) { + store.addAssetImports( + existingEntry.assetImports, + existingEntry.filePath, + ); + } + fileToIdMap.set(filePath2, id); + return; + } + const relativePath2 = posixRelative( + fileURLToPath(config.root), + filePath2, + ); + const parsedData = await parseData({ + id, + data, + filePath: filePath2, + }); + if ( + existingEntry && + existingEntry.filePath && + existingEntry.filePath !== relativePath2 + ) { + const oldFilePath = new URL(existingEntry.filePath, config.root); + if (existsSync(oldFilePath)) { + logger.warn( + `Duplicate id "${id}" found in ${filePath2}. Later items with the same id will overwrite earlier ones.`, + ); + } + } + if (entryType.getRenderFunction) { + let render = renderFunctionByContentType.get(entryType); + if (!render) { + render = await entryType.getRenderFunction(config); + renderFunctionByContentType.set(entryType, render); + } + let rendered = void 0; + try { + rendered = await render?.({ + id, + data, + body, + filePath: filePath2, + digest, + }); + } catch (error) { + logger.error(`Error rendering ${entry}: ${error.message}`); + } + store.set({ + id, + data: parsedData, + body: globOptions.retainBody === false ? void 0 : body, + filePath: relativePath2, + digest, + rendered, + assetImports: rendered?.metadata?.imagePaths, + }); + } else if ("contentModuleTypes" in entryType) { + store.set({ + id, + data: parsedData, + body: globOptions.retainBody === false ? void 0 : body, + filePath: relativePath2, + digest, + deferredRender: true, + }); + } else { + store.set({ + id, + data: parsedData, + body: globOptions.retainBody === false ? void 0 : body, + filePath: relativePath2, + digest, + }); + } + fileToIdMap.set(filePath2, id); + } + let baseDir; + if (isLegacy && !globOptions.base) { + baseDir = new URL(`./src/content/${collection}`, config.root); + } else { + baseDir = globOptions.base + ? new URL(globOptions.base, config.root) + : config.root; + } + if (!baseDir.pathname.endsWith("/")) { + baseDir.pathname = `${baseDir.pathname}/`; + } + const filePath = fileURLToPath(baseDir); + const relativePath = relative(fileURLToPath(config.root), filePath); + const exists = existsSync(baseDir); + if (!exists) { + logger.warn( + `The base directory "${fileURLToPath(baseDir)}" does not exist.`, + ); + } + const files = await tinyglobby(globOptions.pattern, { + cwd: fileURLToPath(baseDir), + expandDirectories: false, + ignore, + }); + if (exists && files.length === 0) { + logger.warn( + `No files found matching "${globOptions.pattern}" in directory "${relativePath}"`, + ); + return; + } + function configForFile(file) { + const ext = file.split(".").at(-1); + if (!ext) { + logger.warn(`No extension found for ${file}`); + return; + } + return entryTypes.get(`.${ext}`); + } + const limit = pLimit(10); + const skippedFiles = []; + const contentDir = new URL("content/", config.srcDir); + const configFiles = new Set( + ["config.js", "config.ts", "config.mjs"].map( + (file) => new URL(file, contentDir).href, + ), + ); + function isConfigFile(file) { + const fileUrl = new URL(file, baseDir); + return configFiles.has(fileUrl.href); + } + await Promise.all( + files.map((entry) => { + if (isConfigFile(entry)) { + return; + } + return limit(async () => { + const entryType = configForFile(entry); + await syncData(entry, baseDir, entryType); + }); + }), + ); + const skipCount = skippedFiles.length; + if (skipCount > 0) { + const patternList = Array.isArray(globOptions.pattern) + ? globOptions.pattern.join(", ") + : globOptions.pattern; + logger.warn( + `The glob() loader cannot be used for files in ${colors.bold("src/content")} when legacy mode is enabled.`, + ); + if (skipCount > 10) { + logger.warn( + `Skipped ${colors.green(skippedFiles.length)} files that matched ${colors.green(patternList)}.`, + ); + } else { + logger.warn( + `Skipped the following files that matched ${colors.green(patternList)}:`, + ); + skippedFiles.forEach((file) => + logger.warn(`\u2022 ${colors.green(file)}`), + ); + } + } + untouchedEntries.forEach((id) => store.delete(id)); + if (!watcher) { + return; + } + watcher.add(filePath); + const matchesGlob = (entry) => + !entry.startsWith("../") && + picomatch.isMatch(entry, globOptions.pattern); + const basePath = fileURLToPath(baseDir); + async function onChange(changedPath) { + const entry = posixRelative(basePath, changedPath); + if (!matchesGlob(entry)) { + return; + } + const entryType = configForFile(changedPath); + const baseUrl = pathToFileURL(basePath); + const oldId = fileToIdMap.get(changedPath); + await syncData(entry, baseUrl, entryType, oldId); + logger.info(`Reloaded data from ${colors.green(entry)}`); + } + watcher.on("change", onChange); + watcher.on("add", onChange); + watcher.on("unlink", async (deletedPath) => { + const entry = posixRelative(basePath, deletedPath); + if (!matchesGlob(entry)) { + return; + } + const id = fileToIdMap.get(deletedPath); + if (id) { + store.delete(id); + fileToIdMap.delete(deletedPath); + } + }); + }, + }; +} +export { extendedGlob, secretLegacyFlag };