6.7.0 - Alpha6 - Scrapers 工具支持打印数据变更详情

This commit is contained in:
SuperMonster003
2025-09-26 22:15:06 +08:00
parent f9a2176d8c
commit d51ba03014
11 changed files with 310 additions and 58 deletions

View File

@@ -362,7 +362,8 @@ export async function fetchStatistics(concurrency = 6) {
byAuthor.set(user.login, entry);
}
// 按最近提交倒序
// Sort by latest commit in descending order.
// zh-CN: 按最近提交倒序.
const entries = Array.from(byAuthor.values()).sort((a, b) => {
const da = a.latestCommitAt ? new Date(a.latestCommitAt).getTime() : 0;
const db = b.latestCommitAt ? new Date(b.latestCommitAt).getTime() : 0;

View File

@@ -174,20 +174,23 @@ async function main() {
const res = await runOne({ nodePath: opts.nodePath, scriptPath: script.abs, cwd: utilsDir });
const endLine = `${startLine} (${formatDuration(res.ms)})`;
const lineCountToMove = childProcessOutput.map(chunk => `${chunk}`).join('').split('\n').length;
if (res.code === 0) {
childProcessOutput.forEach((chunk) => {
process.stdout.write(chunk);
});
readline.moveCursor(process.stdout, 0, -childProcessOutput.length - 1);
readline.moveCursor(process.stdout, 0, -lineCountToMove);
readline.clearLine(process.stdout, 0);
process.stdout.write(`\r${endLine}`);
readline.moveCursor(process.stdout, 0, childProcessOutput.length + 2);
readline.moveCursor(process.stdout, 0, lineCountToMove + 1);
process.stdout.write('\r');
results.push({ ...res, name: script.name });
childProcessOutput.splice(0, childProcessOutput.length);
} else {
readline.moveCursor(process.stdout, 0, -childProcessOutput.length - 1);
process.stdout.write(`\r${endLine} [code: ${res.code}]`);
process.stdout.write('\n');
// readline.moveCursor(process.stdout, 0, -childProcessOutput.length - 1);
// process.stdout.write(`\r${endLine} [code: ${res.code}]`);
// process.stdout.write('\n');
results.push({ ...res, name: script.name });
break;
}

View File

@@ -140,9 +140,15 @@ async function updateLatestArchiveInfo(archives) {
if (JSON.stringify(updatedCommon) !== JSON.stringify(commonObj)) {
await fsp.writeFile(commonJsonPath, JSON.stringify(updatedCommon, null, 2), 'utf8');
console.log('[common.json] Updated (Android Studio information)');
if (commonObj.android_studio_latest_recommended_version_name !== updatedCommon.android_studio_latest_recommended_version_name) {
console.log(`-- '${commonObj.android_studio_latest_recommended_version_name}'`);
console.log(`-> '${updatedCommon.android_studio_latest_recommended_version_name}'`);
const from = commonObj.android_studio_latest_recommended_version_name;
const to = updatedCommon.android_studio_latest_recommended_version_name;
if (from !== to) {
const maxLength = Math.max(...[ from, to ].map(s => s.length + 5));
const SEP_EQ = '='.repeat(maxLength);
console.log(SEP_EQ);
console.log(`-- "${from}"`);
console.log(`-> "${to}"`);
console.log(SEP_EQ);
}
} else {
// console.log('[common.json] No update needed (Android Studio information)');

View File

@@ -99,8 +99,18 @@ async function updateGradleWrapperFileContent({ latestGradleVersion, latestGradl
if (messages.length > 0) {
writePropertiesSync(path, props);
const maxLength = Math.max(...messages.join('\n').split('\n').map(s => s.length));
const SEP_EQ = '='.repeat(maxLength);
const SEP_DASH = '-'.repeat(maxLength);
console.log(`[${fileName}] Updated (Gradle version)`);
messages.forEach((message) => console.log(message));
console.log(SEP_EQ);
messages.forEach((msg, idx) => {
console.log(msg);
if (idx !== messages.length - 1) {
console.log(SEP_DASH);
}
});
console.log(SEP_EQ);
} else {
// console.log(`[${fileName}] No update needed (Gradle version)`);
}

View File

@@ -19,9 +19,15 @@ async function updateTemplateReadmeRhinoBadge(latestVersion) {
if (oldVersion !== latestVersion) {
const updatedFileContent = fileContent.replace(rhinoBadgeRegex, `$1${latestVersion.replaceAll('-', '--')}$3`);
fs.writeFileSync(templateReadmePath, updatedFileContent, 'utf8');
const from = oldVersion;
const to = latestVersion;
const maxLength = Math.max(...[ from, to ].map(s => s.length + 5));
const SEP_EQ = '='.repeat(maxLength);
console.log('[template_readme.md] Updated (Rhino badge version)');
console.log(`-- ${oldVersion}`);
console.log(`-> ${latestVersion}`);
console.log(SEP_EQ);
console.log(`-- "${from}"`);
console.log(`-> "${to}"`);
console.log(SEP_EQ);
} else {
// console.log('[template_readme.md] No update needed (Rhino badge version)');
}
@@ -54,14 +60,28 @@ async function updateCommonJsonWithRhinoData(latestVersion, linenoOfLatestVersio
if (JSON.stringify(updatedCommon) !== JSON.stringify(commonObj)) {
fs.writeFileSync(commonJsonPath, JSON.stringify(updatedCommon, null, 2), 'utf8');
console.log('[common.json] Updated (Rhino information)');
Object.values(toUpdateKeys).forEach(key => {
const toPrint = Object.values(toUpdateKeys).map((key) => {
if (key in updatedCommon && key in commonObj && updatedCommon[key] !== commonObj[key]) {
console.log(`## ${key}`);
console.log(`-- ${commonObj[key]}`);
console.log(`-> ${updatedCommon[key]}`);
return [
`## ${key}`,
`-- "${commonObj[key]}"`,
`-> "${updatedCommon[key]}"`,
].join('\n');
}
return null;
}).filter(Boolean);
const maxLength = Math.max(...toPrint.join('\n').split('\n').map(s => s.length));
const SEP_EQ = '='.repeat(maxLength);
const SEP_DASH = '-'.repeat(maxLength);
console.log('[common.json] Updated (Rhino information)');
console.log(SEP_EQ);
toPrint.forEach((s, i) => {
console.log(s);
if (i !== toPrint.length - 1) {
console.log(SEP_DASH);
}
});
console.log(SEP_EQ);
} else {
// console.log('[common.json] No update needed (Rhino information)');
}

View File

@@ -68,19 +68,24 @@ async function updateVersionInGradleSettings(newVersion) {
const raw = await fsp.readFile(filePath, 'utf8');
// e.g. `foojay-resolver-convention = "0.9.0"`.
const re = /(foojay.resolver.convention\s*=\s*")(\d+(?:\.\d+)+)(?=")/;
const re = /(foojay.resolver.convention\s*=\s*)"(\d+(?:\.\d+)+)(?=")/;
const matched = raw.match(re);
if (!matched) {
throw new Error(`Cannot determine the location of ${pluginLabel} plugin information`);
}
const oldVersion = matched[2];
if (oldVersion !== newVersion) {
const updated = raw.replace(re, `$1${newVersion}`);
const from = matched[2]; // oldVersion
const to = newVersion;
if (from !== to) {
const updated = raw.replace(re, `$1${to}`);
await fsp.writeFile(filePath, updated, 'utf8');
const maxLength = Math.max(...[ from, to ].map(s => s.length + 5));
const SEP_EQ = '='.repeat(maxLength);
console.log(`[${filename}] Updated (${updatedLabel})`);
console.log(`-- ${oldVersion}`);
console.log(`-> ${newVersion}`);
console.log(SEP_EQ);
console.log(`-- ${matched[1]}"${from}"`);
console.log(`-> ${matched[1]}"${to}"`);
console.log(SEP_EQ);
} else {
// console.log(`[${filename}] No update needed (${updatedLabel})`);
}

View File

@@ -3,9 +3,11 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { fetchStatistics } from './fetch-and-parse-autojs6-merged-pr-commits-statistics.mjs';
import { objectToLines } from './utils/format.mjs';
import { printDifferences } from './utils/print.mjs';
import { toYYYYMMDD } from './utils/date.mjs';
function updateCommonJsonFile() {
const updateCommonJsonFile = () => {
const commonJsonPath = path.resolve(process.cwd(), '../.readme/common.json');
const commonRaw = fs.readFileSync(commonJsonPath, 'utf8');
const commonObj = JSON.parse(commonRaw);
@@ -15,30 +17,73 @@ function updateCommonJsonFile() {
var_date_contribution_table_data_updated: toYYYYMMDD(),
};
if (JSON.stringify(updatedCommon) !== JSON.stringify(commonObj)) {
const from = JSON.stringify(commonObj);
const to = JSON.stringify(updatedCommon);
if (from !== to) {
fs.writeFileSync(commonJsonPath, JSON.stringify(updatedCommon, null, 2), 'utf8');
console.log('[common.json] Updated (contribution statistics date)');
printDifferences(objectToLines(commonObj), objectToLines(updatedCommon), { regexForKeyMatching: /"\w+"(?=:)/ });
}
}
};
/**
* @param {string} md
* @returns {string}
*/
const markdownToLines = (md) => {
const rawLines = md
.replaceAll('‑', '-')
.replace(/<span.+?>(.+?)<\/span>/g, '$1')
.replace(/\[(.+?)]\(http.+?\)(?:\s+`\((.+?)\)`)?/g, '$1|$2')
.replaceAll('`', '')
.split('\n')
.map(line => line.split('|').map(s => s.trim()).filter(Boolean).join('\uffef'))
.filter(Boolean)
.join('\n');
const lines = [];
rawLines.split('\n').forEach((line) => {
const data = line.split('\uffef');
if (!data.length) {
return;
}
if (data.length === 3) {
data.splice(1, 0, 'null');
}
if (data.length !== 4) {
throw new Error(`Invalid line: [ ${data.join(', ')} ]`);
}
const [ name, nickname, commitsCount, latestCommit ] = data;
lines.push(`${name} { nickname: ${nickname ? `"${nickname}"` : 'null'}, commitsCount: ${commitsCount}, latestCommit: "${latestCommit}" }`);
});
return lines.join('\n');
};
(async function main() {
const path = '../.readme/template_readme.md';
const filePath = '../.readme/template_readme.md';
const filename = path.basename(filePath);
const stats = await fetchStatistics();
const newMarkdown = stats.map(stat => `| ${stat.contributorMarkdown} | ${stat.commitsCountMarkdown} | ${stat.latestCommitMarkdown} |`).join('\n');
let oldMarkdown = null;
const text = fs.readFileSync(path, { encoding: 'utf-8' });
const contributionHeaderRegex = /(table_header_contribution_\w+.+\r?\n)([\s|:\-]+\r?\n)(?:\|\s*<span style=".+(\r?\n))+/i;
const newText = text.replace(contributionHeaderRegex, (_, headerLine, separatorLine, eol) => {
const raw = fs.readFileSync(filePath, { encoding: 'utf-8' });
const contributionSectionRegex = /(table_header_contribution_\w+.+\r?\n)([\s|:\-]+\r?\n)((?:\|\s*<span style=".+(\r?\n)+)+)/i;
const updated = raw.replace(contributionSectionRegex, (_, headerLine, separatorLine, markdown, eol) => {
oldMarkdown = markdown
return `${headerLine}${separatorLine}${newMarkdown}${eol}`;
});
if (text.replace(/\s+/g, '') !== newText.replace(/\s+/g, '')) {
fs.writeFileSync(path, newText, { encoding: 'utf-8' });
console.log('[template_readme.md] Updated (contribution statistics list)');
const from = raw.replace(/\s+/g, '');
const to = updated.replace(/\s+/g, '');
if (from !== to) {
fs.writeFileSync(filePath, updated, { encoding: 'utf-8' });
console.log(`[${filename}] Updated (contribution statistics list)`);
if (oldMarkdown) {
printDifferences(markdownToLines(oldMarkdown), markdownToLines(newMarkdown));
}
updateCommonJsonFile();
} else {
// console.log('[template_readme.md] No update needed (contribution statistics list)');
// console.log('[${filename}] No update needed (contribution statistics list)');
}
})().catch(err => {
console.error(err);

View File

@@ -16,6 +16,7 @@ import * as fsp from 'node:fs/promises';
import * as path from 'node:path';
import { escapeRegExp } from './format.mjs';
import { toUpdatedStamp } from './date.mjs';
import { printDifferences } from './print.mjs';
/**
* @param {string} s
@@ -32,7 +33,7 @@ const normalize = (s) => String(s).replace(/\s+/g, '');
* @param {(block: string) => { newBlock: string, changed: boolean }} replaceBlockFn
* @returns {{ src: string, changed: boolean }}
*/
export function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
const beginTag = `// @AnchorBegin ${anchorTag}`;
const endTag = `// @AnchorEnd ${anchorTag}`;
@@ -66,7 +67,7 @@ export function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
* @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
* @returns {{ src: string, changed: boolean }}
*/
export function replaceAnchoredMapBlock(src, {
function replaceAnchoredMapBlock(src, {
anchorTag,
mapName,
lines,
@@ -111,7 +112,7 @@ export function replaceAnchoredMapBlock(src, {
* @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
* @returns {{ src: string, changed: boolean }}
*/
export function replaceAnchoredListBlock(src, {
function replaceAnchoredListBlock(src, {
anchorTag,
listName,
lines,
@@ -149,7 +150,7 @@ export function replaceAnchoredListBlock(src, {
* @param {number} [options.linesIndent=4]
* @param {string} [options.updatedLabel='']
* @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
* @param {Console} [options.logger=console]
* @param {RegExp} [options.regexForKeyMatching=null]
* @returns {Promise<{ changed: boolean, content: string }>}
*/
export async function updateAnchoredMapInFile(filePath, {
@@ -159,7 +160,7 @@ export async function updateAnchoredMapInFile(filePath, {
linesIndent = 4,
updatedLabel = '',
toUpdatedStamp: toStamp = toUpdatedStamp,
logger = console,
regexForKeyMatching = null,
}) {
const filename = path.basename(filePath);
const raw = await fsp.readFile(filePath, 'utf8');
@@ -167,9 +168,10 @@ export async function updateAnchoredMapInFile(filePath, {
if (changed) {
await fsp.writeFile(filePath, updated, 'utf8');
logger.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
printDifferences(raw, updated, { regexForKeyMatching });
} else {
// logger.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
// console.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
}
return { changed, content: updated };
}
@@ -183,7 +185,7 @@ export async function updateAnchoredMapInFile(filePath, {
* @param {number} [options.linesIndent=4]
* @param {string} [options.updatedLabel='']
* @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
* @param {Console} [options.logger=console]
* @param {RegExp} [options.regexForKeyMatching=null]
* @returns {Promise<{ changed: boolean, content: string }>}
*/
export async function updateAnchoredListInFile(filePath, {
@@ -193,7 +195,7 @@ export async function updateAnchoredListInFile(filePath, {
linesIndent = 4,
updatedLabel = '',
toUpdatedStamp: toStamp = toUpdatedStamp,
logger = console,
regexForKeyMatching = null,
}) {
const filename = path.basename(filePath);
const raw = await fsp.readFile(filePath, 'utf8');
@@ -201,9 +203,10 @@ export async function updateAnchoredListInFile(filePath, {
if (changed) {
await fsp.writeFile(filePath, updated, 'utf8');
logger.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
printDifferences(raw, updated, { regexForKeyMatching });
} else {
// logger.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
// console.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
}
return { changed, content: updated };
}
@@ -217,16 +220,19 @@ export async function updateAnchoredListInFile(filePath, {
* @param {AnchoredBlockUpdateOption[]} optionList
* @param {Object} [extraOptions={}]
* @param {(date?: Date) => string} [extraOptions.toUpdatedStamp=toUpdatedStamp]
* @param {Console} [extraOptions.logger=console]
* @param {RegExp} [extraOptions.regexForKeyMatching=null]
* @returns {Promise<{ changed: boolean, content: string }>}
*/
export async function batchUpdateAnchoredBlocks(filePath, optionList, {
toUpdatedStamp: toStamp = toUpdatedStamp,
logger = console,
regexForKeyMatching = null,
} = {}) {
const filename = path.basename(filePath);
let raw = await fsp.readFile(filePath, 'utf8');
let changedAny = false;
let updated = null;
let updatedLabel = null;
for (const opt of optionList) {
let res = { src: raw, changed: false };
@@ -250,21 +256,23 @@ export async function batchUpdateAnchoredBlocks(filePath, optionList, {
} else if (opt.type === 'custom' && typeof opt.replacer === 'function') {
res = replaceInAnchoredBlock(raw, opt.anchorTag, (block) => opt.replacer(block, { toUpdatedStamp: toStamp }));
} else {
logger.warn(`[${filename}] Unknown operation type or missing parameters:`, opt);
console.warn(`[${filename}] Unknown operation type or missing parameters:`, opt);
continue;
}
if (res.changed) {
changedAny = true;
raw = res.src;
logger.log(`[${filename}] Updated (${opt.updatedLabel ?? opt.anchorTag})`);
} else {
// logger.log(`[${filename}] No update needed (${op['updatedLabel'] ?? op.anchorTag})`);
updated = res.src;
updatedLabel = opt.updatedLabel;
}
}
if (changedAny) {
await fsp.writeFile(filePath, raw, 'utf8');
await fsp.writeFile(filePath, updated, 'utf8');
console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
printDifferences(raw, updated, { regexForKeyMatching });
} else {
// console.log(`[${filename}] No update needed (${op['updatedLabel'] ?? op.anchorTag})`);
}
return { changed: changedAny, content: raw };
return { changed: changedAny, content: changedAny ? updated : raw };
}

View File

@@ -18,7 +18,7 @@ export function bytes2GiB(bytes, fractionDigits = 2) {
* @example
* // https://example.com?a=1&b=2
* url('https://example.com', { a: 1, b: 2 });
*
*
* @param {string} url
* @param {Object} query
* @returns {string}
@@ -26,7 +26,7 @@ export function bytes2GiB(bytes, fractionDigits = 2) {
export function buildUrl(url, query) {
if (!query) return url;
const q = Object.entries(query)
.map(([key, value]) => `${key}=${encodeURIComponent(value)}`)
.map(([ key, value ]) => `${key}=${encodeURIComponent(value)}`)
.join('&');
return `${url}?${q}`;
}
@@ -41,3 +41,21 @@ export function buildUrl(url, query) {
export function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|\[\]\\]/g, '\\$&');
}
/**
* Converts an object into a string representation where each key-value pair
* is formatted as `"key": "value"` and separated by newline characters.<br>
* zh-CN:<br>
* 将对象转换为字符串表示形式, 每个键值对, 以 `"key": "value"` 的格式呈现并用换行符分隔.
*
* @example string
* "name": "John"
* "age": "30"
* "city": "New York"
*
* @param {Object} o
* @return {string}
*/
export function objectToLines(o) {
return Object.entries(o).map(([ key, value ]) => `"${key}": "${value}"`).join(`\n`);
}

136
.utils/utils/print.mjs Normal file
View File

@@ -0,0 +1,136 @@
/**
* @param {string} str
* @returns {string[]}
*/
const extractConcernedLines = (str) => str
.split(/,?\s*\r?\n/)
.map(s => s.trim())
.filter(s => s && s.match(/\w/) && !s.includes('@Updated'));
/**
* @param {Array} a1
* @param {Array} a2
* @returns {Set}
*/
const findCommonSet = (a1, a2) => {
const s2 = new Set(a2);
return new Set(a1.filter(l => s2.has(l)));
};
/**
* @param {string} src
* @param {string} other
* @param {Object} [options={}]
* @param {RegExp} [options.regexForKeyMatching]
* @return {void}
*/
export function printDifferences(src, other, options = {}) {
const leftRaw = extractConcernedLines(src);
const rightRaw = extractConcernedLines(other);
const commonSet = findCommonSet(leftRaw, rightRaw);
const left = leftRaw.filter(l => !commonSet.has(l));
const right = rightRaw.filter(r => !commonSet.has(r));
const minLineLength = Math.min(...[ ...left, ...right ].map(l => l.length));
const keyOf = (/** @type {string} */ line) => {
const reKey = options.regexForKeyMatching ?? null;
if (reKey) {
const m = line.match(reKey);
return m ? m[0] : line;
}
const reClassicMap = /^"([^"]+)"\s+to\s+("[^"]+"|\d+)$/;
const matchedMap = line.match(reClassicMap);
if (matchedMap) {
return matchedMap[1];
}
const reClassicVersion = /^"(\d+(\.\d+)?)(\.\d+)(\s*[-_]\w+\s*)*"$/;
const matchedVersion = line.match(reClassicVersion);
if (matchedVersion) {
return matchedVersion[1];
}
return line.slice(0, Math.max(1, Math.ceil(minLineLength * 0.4))).trimEnd();
};
const lMap = new Map();
const rMap = new Map();
left.forEach(l => {
const k = keyOf(l);
if (!lMap.has(k)) lMap.set(k, [ l ]);
else lMap.get(k).push(l);
});
right.forEach(r => {
const k = keyOf(r);
if (!rMap.has(k)) rMap.set(k, [ r ]);
else rMap.get(k).push(r);
});
const deletions = [];
const modifications = [];
const additions = [];
for (const l of left) {
const k = keyOf(l);
if (!rMap.has(k)) {
deletions.push(`-- ${l}`);
} else {
const lLineList = lMap.get(k);
const rLineList = rMap.get(k);
const rLine = rLineList?.[0];
const shouldRecordModification = lLineList.length === 1 && rLineList.length === 1 && l !== rLine;
if (shouldRecordModification) {
modifications.push({ from: `-- ${l}`, to: `-> ${rLine}` });
} else {
deletions.push(`-- ${l}`);
}
}
}
for (const r of right) {
const k = keyOf(r);
if (!lMap.has(k)) {
additions.push(`++ ${r}`);
} else {
const lLineList = lMap.get(k);
const rLineList = rMap.get(k);
const lLine = lLineList?.[0];
const shouldRecordModification = lLineList.length === 1 && rLineList.length === 1 && r !== lLine;
if (!shouldRecordModification) {
additions.push(`++ ${r}`);
}
}
}
const allLines = [
...deletions,
...modifications.flatMap(m => [ m.from, m.to ]),
...additions,
];
const maxLen = allLines.reduce((mx, s) => Math.max(mx, s.length), 0);
const SEP_EQ = '='.repeat(Math.max(6, maxLen));
const SEP_DASH = '-'.repeat(Math.max(6, maxLen));
const sections = [];
if (deletions.length > 0) {
sections.push(deletions.slice());
}
if (modifications.length > 0) {
const modLines = [];
modifications.forEach((m, idx) => {
if (idx > 0) modLines.push(SEP_DASH);
modLines.push(m.from);
modLines.push(m.to);
});
sections.push(modLines);
}
if (additions.length > 0) {
sections.push(additions.slice());
}
sections.forEach((lines, idx) => {
console.log(SEP_EQ);
lines.forEach(l => console.log(l));
if (idx === sections.length - 1) {
console.log(SEP_EQ);
}
});
}