deploy: v0.12 fix 存档key名

This commit is contained in:
2026-06-24 03:09:02 +00:00
parent 7a08d49fe3
commit e0a5281119
58743 changed files with 7297269 additions and 39505 deletions
@@ -0,0 +1,38 @@
import CatalogManager from './catalog/CatalogManager.js';
import MessageExtractor from './extractor/MessageExtractor.js';
class ExtractionCompiler {
constructor(config, opts = {}) {
const extractor = opts.extractor ?? new MessageExtractor(opts);
this.manager = new CatalogManager(config, {
...opts,
extractor
});
this[Symbol.dispose] = this[Symbol.dispose].bind(this);
this.installExitHandlers();
}
async extractAll() {
// We can't rely on all files being compiled (e.g. due to persistent
// caching), so loading the messages initially is necessary.
await this.manager.loadMessages();
await this.manager.save();
}
[Symbol.dispose]() {
this.uninstallExitHandlers();
this.manager[Symbol.dispose]();
}
installExitHandlers() {
const cleanup = this[Symbol.dispose];
process.on('exit', cleanup);
process.on('SIGINT', cleanup);
process.on('SIGTERM', cleanup);
}
uninstallExitHandlers() {
const cleanup = this[Symbol.dispose];
process.off('exit', cleanup);
process.off('SIGINT', cleanup);
process.off('SIGTERM', cleanup);
}
}
export { ExtractionCompiler as default };
@@ -0,0 +1,84 @@
import fs$1 from 'fs';
import fs from 'fs/promises';
import path from 'path';
class CatalogLocales {
onChangeCallbacks = (() => new Set())();
constructor(params) {
this.messagesDir = params.messagesDir;
this.sourceLocale = params.sourceLocale;
this.extension = params.extension;
this.locales = params.locales;
}
async getTargetLocales() {
if (this.targetLocales) {
return this.targetLocales;
}
if (this.locales === 'infer') {
this.targetLocales = await this.readTargetLocales();
} else {
this.targetLocales = this.locales.filter(locale => locale !== this.sourceLocale);
}
return this.targetLocales;
}
async readTargetLocales() {
try {
const files = await fs.readdir(this.messagesDir);
return files.filter(file => file.endsWith(this.extension)).map(file => path.basename(file, this.extension)).filter(locale => locale !== this.sourceLocale);
} catch {
return [];
}
}
subscribeLocalesChange(callback) {
this.onChangeCallbacks.add(callback);
if (this.locales === 'infer' && !this.watcher) {
void this.startWatcher();
}
}
unsubscribeLocalesChange(callback) {
this.onChangeCallbacks.delete(callback);
if (this.onChangeCallbacks.size === 0) {
this.stopWatcher();
}
}
async startWatcher() {
if (this.watcher) {
return;
}
await fs.mkdir(this.messagesDir, {
recursive: true
});
this.watcher = fs$1.watch(this.messagesDir, {
persistent: false,
recursive: false
}, (event, filename) => {
const isCatalogFile = filename != null && filename.endsWith(this.extension) && !filename.includes(path.sep);
if (isCatalogFile) {
void this.onChange();
}
});
}
stopWatcher() {
if (this.watcher) {
this.watcher.close();
this.watcher = undefined;
}
}
async onChange() {
const oldLocales = new Set(this.targetLocales || []);
this.targetLocales = await this.readTargetLocales();
const newLocalesSet = new Set(this.targetLocales);
const added = this.targetLocales.filter(locale => !oldLocales.has(locale));
const removed = Array.from(oldLocales).filter(locale => !newLocalesSet.has(locale));
if (added.length > 0 || removed.length > 0) {
for (const callback of this.onChangeCallbacks) {
callback({
added,
removed
});
}
}
}
}
export { CatalogLocales as default };
@@ -0,0 +1,370 @@
import fs from 'fs/promises';
import path from 'path';
import { resolveCodec, getFormatExtension } from '../format/index.js';
import SourceFileScanner from '../source/SourceFileScanner.js';
import SourceFileWatcher from '../source/SourceFileWatcher.js';
import { getDefaultProjectRoot, compareReferences } from '../utils.js';
import CatalogLocales from './CatalogLocales.js';
import CatalogPersister from './CatalogPersister.js';
import SaveScheduler from './SaveScheduler.js';
class CatalogManager {
/**
* The source of truth for which messages are used.
* NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync.
*/
messagesByFile = (() => new Map())();
/**
* Fast lookup for messages by ID across all files,
* contains the same messages as `messagesByFile`.
* NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync.
*/
messagesById = (() => new Map())();
/**
* This potentially also includes outdated ones that were initially available,
* but are not used anymore. This allows to restore them if they are used again.
**/
translationsByTargetLocale = (() => new Map())();
lastWriteByLocale = (() => new Map())();
// Cached instances
// Resolves when all catalogs are loaded
// Resolves when the initial project scan and processing is complete
constructor(config, opts) {
this.config = config;
this.saveScheduler = new SaveScheduler(50);
this.projectRoot = opts.projectRoot ?? getDefaultProjectRoot();
this.isDevelopment = opts.isDevelopment ?? false;
this.extractor = opts.extractor;
if (this.isDevelopment) {
// We kick this off as early as possible, so we get notified about changes
// that happen during the initial project scan (while awaiting it to
// complete though)
this.sourceWatcher = new SourceFileWatcher(this.getSrcPaths(), this.handleFileEvents.bind(this));
void this.sourceWatcher.start();
}
}
async getCodec() {
if (!this.codec) {
this.codec = await resolveCodec(this.config.messages.format, this.projectRoot);
}
return this.codec;
}
async getPersister() {
if (this.persister) {
return this.persister;
} else {
this.persister = new CatalogPersister({
messagesPath: this.config.messages.path,
codec: await this.getCodec(),
extension: getFormatExtension(this.config.messages.format)
});
return this.persister;
}
}
getCatalogLocales() {
if (this.catalogLocales) {
return this.catalogLocales;
} else {
const messagesDir = path.join(this.projectRoot, this.config.messages.path);
this.catalogLocales = new CatalogLocales({
messagesDir,
sourceLocale: this.config.sourceLocale,
extension: getFormatExtension(this.config.messages.format),
locales: this.config.messages.locales
});
return this.catalogLocales;
}
}
async getTargetLocales() {
return this.getCatalogLocales().getTargetLocales();
}
getSrcPaths() {
return (Array.isArray(this.config.srcPath) ? this.config.srcPath : [this.config.srcPath]).map(srcPath => path.join(this.projectRoot, srcPath));
}
async loadMessages() {
const sourceDiskMessages = await this.loadSourceMessages();
this.loadCatalogsPromise = this.loadTargetMessages();
await this.loadCatalogsPromise;
this.scanCompletePromise = (async () => {
const sourceFiles = await SourceFileScanner.getSourceFiles(this.getSrcPaths());
await Promise.all(Array.from(sourceFiles).map(async filePath => this.processFile(filePath)));
this.mergeSourceDiskMetadata(sourceDiskMessages);
})();
await this.scanCompletePromise;
if (this.isDevelopment) {
const catalogLocales = this.getCatalogLocales();
catalogLocales.subscribeLocalesChange(this.onLocalesChange);
}
}
async loadSourceMessages() {
// Load source catalog to hydrate metadata (e.g. flags) later without
// treating catalog entries as source of truth.
const diskMessages = await this.loadLocaleMessages(this.config.sourceLocale);
const byId = new Map();
for (const diskMessage of diskMessages) {
byId.set(diskMessage.id, diskMessage);
}
return byId;
}
async loadLocaleMessages(locale) {
const persister = await this.getPersister();
const messages = await persister.read(locale);
const fileTime = await persister.getLastModified(locale);
this.lastWriteByLocale.set(locale, fileTime);
return messages;
}
async loadTargetMessages() {
const targetLocales = await this.getTargetLocales();
await Promise.all(targetLocales.map(locale => this.reloadLocaleCatalog(locale)));
}
async reloadLocaleCatalog(locale) {
const diskMessages = await this.loadLocaleMessages(locale);
if (locale === this.config.sourceLocale) {
// For source: Merge additional properties like flags
for (const diskMessage of diskMessages) {
const prev = this.messagesById.get(diskMessage.id);
if (prev) {
// Mutate the existing object instead of creating a copy
// to keep messagesById and messagesByFile in sync.
// Unknown properties (like flags): disk wins
// Known properties: existing (from extraction) wins
for (const key of Object.keys(diskMessage)) {
if (!['id', 'message', 'description', 'references'].includes(key)) {
// For unknown properties (like flags), disk wins
prev[key] = diskMessage[key];
}
}
}
}
} else {
// For target: disk wins completely, BUT preserve existing translations
// if we read empty (likely a write in progress by an external tool
// that causes the file to temporarily be empty)
const existingTranslations = this.translationsByTargetLocale.get(locale);
const hasExistingTranslations = existingTranslations && existingTranslations.size > 0;
if (diskMessages.length > 0) {
// We got content from disk, replace with it
const translations = new Map();
for (const message of diskMessages) {
translations.set(message.id, message);
}
this.translationsByTargetLocale.set(locale, translations);
} else if (hasExistingTranslations) ; else {
// We read empty and have no existing translations
const translations = new Map();
this.translationsByTargetLocale.set(locale, translations);
}
}
}
mergeSourceDiskMetadata(diskMessages) {
for (const [id, diskMessage] of diskMessages) {
const existing = this.messagesById.get(id);
if (!existing) continue;
// Mutate the existing object instead of creating a copy.
// This keeps `messagesById` and `messagesByFile` in sync since
// they reference the same object instance.
for (const key of Object.keys(diskMessage)) {
if (existing[key] == null) {
existing[key] = diskMessage[key];
}
}
}
}
async processFile(absoluteFilePath) {
let messages = [];
try {
const content = await fs.readFile(absoluteFilePath, 'utf8');
let extraction;
try {
extraction = await this.extractor.extract(absoluteFilePath, content);
} catch {
return false;
}
messages = extraction.messages;
} catch (err) {
if (err.code !== 'ENOENT') {
throw err;
}
// ENOENT -> treat as no messages
}
const prevFileMessages = this.messagesByFile.get(absoluteFilePath);
const relativeFilePath = path.relative(this.projectRoot, absoluteFilePath);
// Init with all previous ones
const idsToRemove = Array.from(prevFileMessages?.keys() ?? []);
// Replace existing messages with new ones
const fileMessages = new Map();
for (let message of messages) {
const prevMessage = this.messagesById.get(message.id);
// Merge with previous message if it exists
if (prevMessage) {
message = {
...message
};
if (message.references) {
message.references = this.mergeReferences(prevMessage.references ?? [], relativeFilePath, message.references);
}
// Merge other properties like description, or unknown
// attributes like flags that are opaque to us
for (const key of Object.keys(prevMessage)) {
if (message[key] == null) {
message[key] = prevMessage[key];
}
}
}
this.messagesById.set(message.id, message);
fileMessages.set(message.id, message);
// This message continues to exist in this file
const index = idsToRemove.indexOf(message.id);
if (index !== -1) idsToRemove.splice(index, 1);
}
// Clean up removed messages from `messagesById`
idsToRemove.forEach(id => {
const message = this.messagesById.get(id);
if (!message) return;
const hasOtherReferences = message.references?.some(ref => ref.path !== relativeFilePath);
if (!hasOtherReferences) {
// No other references, delete the message entirely
this.messagesById.delete(id);
} else {
// Message is used elsewhere, remove this file from references
// Mutate the existing object to keep `messagesById` and `messagesByFile` in sync
message.references = message.references?.filter(ref => ref.path !== relativeFilePath);
}
});
// Update the stored messages
if (messages.length > 0) {
this.messagesByFile.set(absoluteFilePath, fileMessages);
} else {
this.messagesByFile.delete(absoluteFilePath);
}
const changed = this.haveMessagesChangedForFile(prevFileMessages, fileMessages);
return changed;
}
mergeReferences(existing, currentFilePath, currentFileRefs) {
// Keep refs from other files, replace all refs from the current file
const otherFileRefs = existing.filter(ref => ref.path !== currentFilePath);
const merged = [...otherFileRefs, ...currentFileRefs];
return merged.sort(compareReferences);
}
haveMessagesChangedForFile(beforeMessages, afterMessages) {
// If one exists and the other doesn't, there's a change
if (!beforeMessages) {
return afterMessages.size > 0;
}
// Different sizes means changes
if (beforeMessages.size !== afterMessages.size) {
return true;
}
// Check differences in beforeMessages vs afterMessages
for (const [id, msg1] of beforeMessages) {
const msg2 = afterMessages.get(id);
if (!msg2 || !this.areMessagesEqual(msg1, msg2)) {
return true; // Early exit on first difference
}
}
return false;
}
areMessagesEqual(msg1, msg2) {
// Note: We intentionally don't compare references here.
// References are aggregated metadata from multiple files and comparing
// them would cause false positives due to parallel extraction order.
return msg1.id === msg2.id && msg1.message === msg2.message && msg1.description === msg2.description;
}
async save() {
return this.saveScheduler.schedule(() => this.saveImpl());
}
async saveImpl() {
await this.saveLocale(this.config.sourceLocale);
const targetLocales = await this.getTargetLocales();
await Promise.all(targetLocales.map(locale => this.saveLocale(locale)));
}
async saveLocale(locale) {
await this.loadCatalogsPromise;
const messages = Array.from(this.messagesById.values());
const persister = await this.getPersister();
const isSourceLocale = locale === this.config.sourceLocale;
// Check if file was modified externally (poll-at-save is cheaper than
// watchers here since stat() is fast and avoids continuous overhead)
const lastWriteTime = this.lastWriteByLocale.get(locale);
const currentFileTime = await persister.getLastModified(locale);
if (currentFileTime && lastWriteTime && currentFileTime > lastWriteTime) {
await this.reloadLocaleCatalog(locale);
}
const localeMessages = isSourceLocale ? this.messagesById : this.translationsByTargetLocale.get(locale);
const messagesToPersist = messages.map(message => {
const localeMessage = localeMessages?.get(message.id);
return {
...localeMessage,
id: message.id,
description: message.description,
references: message.references,
message: isSourceLocale ? message.message : localeMessage?.message ?? ''
};
});
await persister.write(messagesToPersist, {
locale,
sourceMessagesById: this.messagesById
});
// Update timestamps
const newTime = await persister.getLastModified(locale);
this.lastWriteByLocale.set(locale, newTime);
}
onLocalesChange = async params => {
// Chain to existing promise
this.loadCatalogsPromise = Promise.all([this.loadCatalogsPromise, ...params.added.map(locale => this.reloadLocaleCatalog(locale))]);
for (const locale of params.added) {
await this.saveLocale(locale);
}
for (const locale of params.removed) {
this.translationsByTargetLocale.delete(locale);
this.lastWriteByLocale.delete(locale);
}
};
async handleFileEvents(events) {
if (this.loadCatalogsPromise) {
await this.loadCatalogsPromise;
}
// Wait for initial scan to complete to avoid race conditions
if (this.scanCompletePromise) {
await this.scanCompletePromise;
}
let changed = false;
const expandedEvents = await this.sourceWatcher.expandDirectoryDeleteEvents(events, Array.from(this.messagesByFile.keys()));
for (const event of expandedEvents) {
const hasChanged = await this.processFile(event.path);
changed ||= hasChanged;
}
if (changed) {
await this.save();
}
}
[Symbol.dispose]() {
this.sourceWatcher?.stop();
this.sourceWatcher = undefined;
this.saveScheduler[Symbol.dispose]();
if (this.catalogLocales && this.isDevelopment) {
this.catalogLocales.unsubscribeLocalesChange(this.onLocalesChange);
}
}
}
export { CatalogManager as default };
@@ -0,0 +1,63 @@
import fs from 'fs/promises';
import path from 'path';
class CatalogPersister {
constructor(params) {
this.messagesPath = params.messagesPath;
this.codec = params.codec;
this.extension = params.extension;
}
getFileName(locale) {
return locale + this.extension;
}
getFilePath(locale) {
return path.join(this.messagesPath, this.getFileName(locale));
}
async read(locale) {
const filePath = this.getFilePath(locale);
let content;
try {
content = await fs.readFile(filePath, 'utf8');
} catch (error) {
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
return [];
}
throw new Error(`Error while reading ${this.getFileName(locale)}:\n> ${error}`, {
cause: error
});
}
try {
return this.codec.decode(content, {
locale
});
} catch (error) {
throw new Error(`Error while decoding ${this.getFileName(locale)}:\n> ${error}`, {
cause: error
});
}
}
async write(messages, context) {
const filePath = this.getFilePath(context.locale);
const content = this.codec.encode(messages, context);
try {
const outputDir = path.dirname(filePath);
await fs.mkdir(outputDir, {
recursive: true
});
await fs.writeFile(filePath, content);
} catch (error) {
console.error(`❌ Failed to write catalog: ${error}`);
}
}
async getLastModified(locale) {
const filePath = this.getFilePath(locale);
try {
const stats = await fs.stat(filePath);
return stats.mtime;
} catch {
return undefined;
}
}
}
export { CatalogPersister as default };
@@ -0,0 +1,85 @@
/**
* De-duplicates excessive save invocations,
* while keeping a single one instant.
*/
class SaveScheduler {
isSaving = false;
pendingResolvers = [];
constructor(delayMs = 50) {
this.delayMs = delayMs;
}
async schedule(saveTask) {
return new Promise((resolve, reject) => {
this.pendingResolvers.push({
resolve,
reject
});
this.nextSaveTask = saveTask;
if (!this.isSaving && !this.saveTimeout) {
// Not currently saving and no scheduled save, save immediately
this.executeSave();
} else if (this.saveTimeout) {
// A save is already scheduled, reschedule to debounce
this.scheduleSave();
}
// If isSaving is true and no timeout is scheduled, the current save
// will check for pending resolvers when it completes and schedule
// another save if needed (see finally block in executeSave)
});
}
scheduleSave() {
if (this.saveTimeout) {
clearTimeout(this.saveTimeout);
}
this.saveTimeout = setTimeout(() => {
this.saveTimeout = undefined;
this.executeSave();
}, this.delayMs);
}
async executeSave() {
if (this.isSaving) {
return;
}
const saveTask = this.nextSaveTask;
if (!saveTask) {
return;
}
// Capture current pending resolvers for this save
const resolversForThisSave = this.pendingResolvers;
this.pendingResolvers = [];
this.nextSaveTask = undefined;
this.isSaving = true;
try {
const result = await saveTask();
// Resolve only the promises that were pending when this save started
resolversForThisSave.forEach(({
resolve
}) => resolve(result));
} catch (error) {
// Reject only the promises that were pending when this save started
resolversForThisSave.forEach(({
reject
}) => reject(error));
} finally {
this.isSaving = false;
// If new saves were requested during this save, schedule another
if (this.pendingResolvers.length > 0) {
this.scheduleSave();
}
}
}
[Symbol.dispose]() {
if (this.saveTimeout) {
clearTimeout(this.saveTimeout);
this.saveTimeout = undefined;
}
this.pendingResolvers = [];
this.nextSaveTask = undefined;
this.isSaving = false;
}
}
export { SaveScheduler as default };
+35
View File
@@ -0,0 +1,35 @@
import path from 'path';
import { getFormatExtension, resolveCodec } from './format/index.js';
let cachedCodec = null;
async function getCodec(options, projectRoot) {
if (!cachedCodec) {
cachedCodec = await resolveCodec(options.messages.format, projectRoot);
}
return cachedCodec;
}
/**
* Parses and optimizes catalog files.
*
* Note that if we use a dynamic import like `import(`${locale}.json`)`, then
* the loader will optimistically run for all candidates in this folder (both
* during dev as well as at build time).
*/
function catalogLoader(source) {
const options = this.getOptions();
const callback = this.async();
const extension = getFormatExtension(options.messages.format);
getCodec(options, this.rootContext).then(codec => {
const locale = path.basename(this.resourcePath, extension);
const jsonString = codec.toJSONString(source, {
locale
});
// https://v8.dev/blog/cost-of-javascript-2019#json
const result = `export default JSON.parse(${JSON.stringify(jsonString)});`;
callback(null, result);
}).catch(callback);
}
export { catalogLoader as default };
@@ -0,0 +1,15 @@
import ExtractionCompiler from './ExtractionCompiler.js';
import MessageExtractor from './extractor/MessageExtractor.js';
import { getDefaultProjectRoot } from './utils.js';
async function extractMessages(params) {
const compiler = new ExtractionCompiler(params, {
extractor: new MessageExtractor({
isDevelopment: false,
projectRoot: getDefaultProjectRoot()
})
});
await compiler.extractAll();
}
export { extractMessages as default };
@@ -0,0 +1,26 @@
import MessageExtractor from './extractor/MessageExtractor.js';
// Module-level extractor instance for transformation caching.
// Note: Next.js/Turbopack may create multiple loader instances, but each
// only handles file transformation. The ExtractionCompiler (which manages
// catalogs) is initialized separately in createNextIntlPlugin.
let extractor;
function extractionLoader(source) {
const callback = this.async();
const projectRoot = this.rootContext;
// Avoid rollup's `replace` plugin to compile this away
const isDevelopment = process.env['NODE_ENV'.trim()] === 'development';
if (!extractor) {
extractor = new MessageExtractor({
isDevelopment,
projectRoot,
sourceMap: this.sourceMap
});
}
extractor.extract(this.resourcePath, source).then(result => {
callback(null, result.code, result.map);
}).catch(callback);
}
export { extractionLoader as default };
@@ -0,0 +1,30 @@
class LRUCache {
constructor(maxSize) {
this.maxSize = maxSize;
this.cache = new Map();
}
set(key, value) {
const isNewKey = !this.cache.has(key);
if (isNewKey && this.cache.size >= this.maxSize) {
const lruKey = this.cache.keys().next().value;
if (lruKey !== undefined) {
this.cache.delete(lruKey);
}
}
this.cache.set(key, {
key,
value
});
}
get(key) {
const item = this.cache.get(key);
if (item) {
this.cache.delete(key);
this.cache.set(key, item);
return item.value;
}
return undefined;
}
}
export { LRUCache as default };
@@ -0,0 +1,66 @@
import { createRequire } from 'module';
import path from 'path';
import { transform } from '@swc/core';
import { getDefaultProjectRoot } from '../utils.js';
import LRUCache from './LRUCache.js';
const require = createRequire(import.meta.url);
class MessageExtractor {
compileCache = (() => new LRUCache(750))();
constructor(opts) {
this.isDevelopment = opts.isDevelopment ?? false;
this.projectRoot = opts.projectRoot ?? getDefaultProjectRoot();
this.sourceMap = opts.sourceMap ?? false;
}
async extract(absoluteFilePath, source) {
const cacheKey = [source, absoluteFilePath].join('!');
const cached = this.compileCache.get(cacheKey);
if (cached) return cached;
// Shortcut parsing if hook is not used. The Turbopack integration already
// pre-filters this, but for webpack this feature doesn't exist, so we need
// to do it here.
if (!source.includes('useExtracted') && !source.includes('getExtracted')) {
return {
messages: [],
code: source
};
}
const filePath = path.relative(this.projectRoot, absoluteFilePath);
const result = await transform(source, {
jsc: {
target: 'esnext',
parser: {
syntax: 'typescript',
tsx: true,
decorators: true
},
experimental: {
cacheRoot: 'node_modules/.cache/swc',
disableBuiltinTransformsForInternalTesting: true,
disableAllLints: true,
plugins: [[require.resolve('next-intl-swc-plugin-extractor'), {
isDevelopment: this.isDevelopment,
filePath
}]]
}
},
sourceMaps: this.sourceMap,
sourceFileName: filePath,
filename: filePath
});
// TODO: Improve the typing of @swc/core
const output = result.output;
const messages = JSON.parse(JSON.parse(output).results);
const extractionResult = {
code: result.code,
map: result.map,
messages
};
this.compileCache.set(cacheKey, extractionResult);
return extractionResult;
}
}
export { MessageExtractor as default };
@@ -0,0 +1,5 @@
function defineCodec(factory) {
return factory;
}
export { defineCodec };
@@ -0,0 +1,40 @@
import { getSortedMessages, setNestedProperty } from '../../utils.js';
import { defineCodec } from '../ExtractorCodec.js';
var JSONCodec = defineCodec(() => ({
decode(source) {
const json = JSON.parse(source);
const messages = [];
traverseMessages(json, (message, id) => {
messages.push({
id,
message
});
});
return messages;
},
encode(messages) {
const root = {};
for (const message of getSortedMessages(messages)) {
setNestedProperty(root, message.id, message.message);
}
return JSON.stringify(root, null, 2) + '\n';
},
toJSONString(source) {
return source;
}
}));
function traverseMessages(obj, callback, path = '') {
const NAMESPACE_SEPARATOR = '.';
for (const key of Object.keys(obj)) {
const newPath = path ? path + NAMESPACE_SEPARATOR + key : key;
const value = obj[key];
if (typeof value === 'string') {
callback(value, newPath);
} else if (typeof value === 'object') {
traverseMessages(value, callback, newPath);
}
}
}
export { JSONCodec as default };
@@ -0,0 +1,93 @@
import POParser from 'po-parser';
import { setNestedProperty, getSortedMessages } from '../../utils.js';
import { defineCodec } from '../ExtractorCodec.js';
var POCodec = defineCodec(() => {
// See also https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html
const DEFAULT_METADATA = {
// Recommended by spec
'Content-Type': 'text/plain; charset=utf-8',
'Content-Transfer-Encoding': '8bit',
// Otherwise other tools might set this
'X-Generator': 'next-intl',
// Crowdin defaults to using msgid as source key
'X-Crowdin-SourceKey': 'msgstr'
};
// Move all parts before the last dot to msgctxt
const NAMESPACE_SEPARATOR = '.';
// Metadata is stored so it can be retained when writing
const metadataByLocale = new Map();
return {
decode(content, context) {
const catalog = POParser.parse(content);
if (catalog.meta) {
metadataByLocale.set(context.locale, catalog.meta);
}
const messages = catalog.messages || [];
return messages.map(msg => {
const {
extractedComments,
msgctxt,
msgid,
msgstr,
...rest
} = msg;
if (extractedComments && extractedComments.length > 1) {
throw new Error(`Multiple extracted comments are not supported. Found ${extractedComments.length} comments for msgid "${msgid}".`);
}
return {
...rest,
id: msgctxt ? [msgctxt, msgid].join(NAMESPACE_SEPARATOR) : msgid,
message: msgstr,
...(extractedComments && extractedComments.length > 0 && {
description: extractedComments[0]
})
};
});
},
encode(messages, context) {
const encodedMessages = getSortedMessages(messages).map(msg => {
const {
description,
id,
message,
...rest
} = msg;
const lastDotIndex = id.lastIndexOf(NAMESPACE_SEPARATOR);
const hasNamespace = id.includes(NAMESPACE_SEPARATOR);
const msgid = hasNamespace ? id.slice(lastDotIndex + NAMESPACE_SEPARATOR.length) : id;
return {
msgid,
msgstr: message,
...(description && {
extractedComments: [description]
}),
...(hasNamespace && {
msgctxt: id.slice(0, lastDotIndex)
}),
...rest
};
});
return POParser.serialize({
meta: {
Language: context.locale,
...DEFAULT_METADATA,
...metadataByLocale.get(context.locale)
},
messages: encodedMessages
});
},
toJSONString(source, context) {
const parsed = this.decode(source, context);
const messagesObject = {};
for (const message of parsed) {
setNestedProperty(messagesObject, message.id, message.message);
}
return JSON.stringify(messagesObject);
}
};
});
export { POCodec as default };
+44
View File
@@ -0,0 +1,44 @@
import path from 'path';
import { throwError } from '../../plugin/utils.js';
const formats = {
json: {
codec: () => import('./codecs/JSONCodec.js'),
extension: '.json'
},
po: {
codec: () => import('./codecs/POCodec.js'),
extension: '.po'
}
};
function isBuiltInFormat(format) {
return typeof format === 'string' && format in formats;
}
function getFormatExtension(format) {
if (isBuiltInFormat(format)) {
return formats[format].extension;
} else {
return format.extension;
}
}
async function resolveCodec(format, projectRoot) {
if (isBuiltInFormat(format)) {
const factory = (await formats[format].codec()).default;
return factory();
} else {
const resolvedPath = path.isAbsolute(format.codec) ? format.codec : path.resolve(projectRoot, format.codec);
let module;
try {
module = await import(resolvedPath);
} catch (error) {
throwError(`Could not load codec from "${resolvedPath}".\n${error}`);
}
const factory = module.default;
if (!factory || typeof factory !== 'function') {
throwError(`Codec at "${resolvedPath}" must have a default export returned from \`defineCodec\`.`);
}
return factory();
}
}
export { formats as default, getFormatExtension, resolveCodec };
@@ -0,0 +1,29 @@
import path from 'path';
class SourceFileFilter {
static EXTENSIONS = ['ts', 'tsx', 'js', 'jsx'];
// Will not be entered, except if explicitly asked for
// TODO: At some point we should infer these from .gitignore
static IGNORED_DIRECTORIES = ['node_modules', '.next', '.git'];
static isSourceFile(filePath) {
const ext = path.extname(filePath);
return SourceFileFilter.EXTENSIONS.map(cur => '.' + cur).includes(ext);
}
static shouldEnterDirectory(dirPath, srcPaths) {
const dirName = path.basename(dirPath);
if (SourceFileFilter.IGNORED_DIRECTORIES.includes(dirName)) {
return SourceFileFilter.isIgnoredDirectoryExplicitlyIncluded(dirPath, srcPaths);
}
return true;
}
static isIgnoredDirectoryExplicitlyIncluded(ignoredDirPath, srcPaths) {
return srcPaths.some(srcPath => SourceFileFilter.isWithinPath(srcPath, ignoredDirPath));
}
static isWithinPath(targetPath, basePath) {
const relativePath = path.relative(basePath, targetPath);
return relativePath === '' || !relativePath.startsWith('..');
}
}
export { SourceFileFilter as default };
@@ -0,0 +1,31 @@
import fs from 'fs/promises';
import path from 'path';
import SourceFileFilter from './SourceFileFilter.js';
class SourceFileScanner {
static async walkSourceFiles(dir, srcPaths, acc = []) {
const entries = await fs.readdir(dir, {
withFileTypes: true
});
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
if (!SourceFileFilter.shouldEnterDirectory(entryPath, srcPaths)) {
continue;
}
await SourceFileScanner.walkSourceFiles(entryPath, srcPaths, acc);
} else {
if (SourceFileFilter.isSourceFile(entry.name)) {
acc.push(entryPath);
}
}
}
return acc;
}
static async getSourceFiles(srcPaths) {
const files = (await Promise.all(srcPaths.map(srcPath => SourceFileScanner.walkSourceFiles(srcPath, srcPaths)))).flat();
return new Set(files);
}
}
export { SourceFileScanner as default };
@@ -0,0 +1,132 @@
import fs from 'fs/promises';
import path from 'path';
import { subscribe } from '@parcel/watcher';
import SourceFileFilter from './SourceFileFilter.js';
import SourceFileScanner from './SourceFileScanner.js';
class SourceFileWatcher {
subscriptions = [];
constructor(roots, onChange) {
this.roots = roots;
this.onChange = onChange;
}
async start() {
if (this.subscriptions.length > 0) {
return;
}
const ignore = SourceFileFilter.IGNORED_DIRECTORIES.map(dir => `**/${dir}/**`);
for (const root of this.roots) {
const sub = await subscribe(root, async (err, events) => {
if (err) {
console.error(err);
return;
}
const filtered = await this.normalizeEvents(events);
if (filtered.length > 0) {
void this.onChange(filtered);
}
}, {
ignore
});
this.subscriptions.push(sub);
}
}
async normalizeEvents(events) {
const directoryCreatePaths = [];
const otherEvents = [];
// We need to expand directory creates because during rename operations,
// @parcel/watcher emits a directory create event but may not emit individual
// file events for the moved files
await Promise.all(events.map(async event => {
if (event.type === 'create') {
try {
const stats = await fs.stat(event.path);
if (stats.isDirectory()) {
directoryCreatePaths.push(event.path);
return;
}
} catch {
// Path doesn't exist or is inaccessible, treat as file
}
}
otherEvents.push(event);
}));
// Expand directory create events to find source files inside
let expandedCreateEvents = [];
if (directoryCreatePaths.length > 0) {
try {
const sourceFiles = await SourceFileScanner.getSourceFiles(directoryCreatePaths);
expandedCreateEvents = Array.from(sourceFiles).map(filePath => ({
type: 'create',
path: filePath
}));
} catch {
// Directories might have been deleted or are inaccessible
}
}
// Combine original events with expanded directory creates.
// Deduplicate by path to avoid processing the same file twice
// in case @parcel/watcher also emitted individual file events.
const allEvents = [...otherEvents, ...expandedCreateEvents];
const seenPaths = new Set();
const deduplicated = [];
for (const event of allEvents) {
const key = `${event.type}:${event.path}`;
if (!seenPaths.has(key)) {
seenPaths.add(key);
deduplicated.push(event);
}
}
return deduplicated.filter(event => {
// Keep all delete events (might be deleted directories that no longer exist)
if (event.type === 'delete') {
return true;
}
// Keep source files
return SourceFileFilter.isSourceFile(event.path);
});
}
async expandDirectoryDeleteEvents(events, prevKnownFiles) {
const expanded = [];
for (const event of events) {
if (event.type === 'delete' && !SourceFileFilter.isSourceFile(event.path)) {
const dirPath = path.resolve(event.path);
const filesInDirectory = [];
for (const filePath of prevKnownFiles) {
if (SourceFileFilter.isWithinPath(filePath, dirPath)) {
filesInDirectory.push(filePath);
}
}
// If we found files within this path, it was a directory
if (filesInDirectory.length > 0) {
for (const filePath of filesInDirectory) {
expanded.push({
type: 'delete',
path: filePath
});
}
} else {
// Not a directory or no files in it, pass through as-is
expanded.push(event);
}
} else {
// Pass through as-is
expanded.push(event);
}
}
return expanded;
}
async stop() {
await Promise.all(this.subscriptions.map(sub => sub.unsubscribe()));
this.subscriptions = [];
}
[Symbol.dispose]() {
void this.stop();
}
}
export { SourceFileWatcher as default };
+38
View File
@@ -0,0 +1,38 @@
// Essentialls lodash/set, but we avoid this dependency
function setNestedProperty(obj, keyPath, value) {
const keys = keyPath.split('.');
let current = obj;
for (let i = 0; i < keys.length - 1; i++) {
const key = keys[i];
if (!(key in current) || typeof current[key] !== 'object' || current[key] === null) {
current[key] = {};
}
current = current[key];
}
current[keys[keys.length - 1]] = value;
}
function getSortedMessages(messages) {
return messages.toSorted((messageA, messageB) => {
const refA = messageA.references?.[0];
const refB = messageB.references?.[0];
// No references: preserve original (extraction) order
if (!refA || !refB) return 0;
// Sort by path, then line. Same path+line: preserve original order
return compareReferences(refA, refB);
});
}
function localeCompare(a, b) {
return a.localeCompare(b, 'en');
}
function compareReferences(refA, refB) {
const pathCompare = localeCompare(refA.path, refB.path);
if (pathCompare !== 0) return pathCompare;
return (refA.line ?? 0) - (refB.line ?? 0);
}
function getDefaultProjectRoot() {
return process.cwd();
}
export { compareReferences, getDefaultProjectRoot, getSortedMessages, localeCompare, setNestedProperty };