6.7.0 - Alpha6 - Scrapers 工具支持打印数据变更详情

2025-09-26 22:15:06 +08:00
parent f9a2176d8c
commit d51ba03014
11 changed files with 310 additions and 58 deletions
--- a/.utils/utils/anchors.mjs
+++ b/.utils/utils/anchors.mjs
@@ -16,6 +16,7 @@ import * as fsp from 'node:fs/promises';
 import * as path from 'node:path';
 import { escapeRegExp } from './format.mjs';
 import { toUpdatedStamp } from './date.mjs';
+import { printDifferences } from './print.mjs';

 /**
 * @param {string} s
@@ -32,7 +33,7 @@ const normalize = (s) => String(s).replace(/\s+/g, '');
 * @param {(block: string) => { newBlock: string, changed: boolean }} replaceBlockFn
 * @returns {{ src: string, changed: boolean }}
 */
-export function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
+function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
    const beginTag = `// @AnchorBegin ${anchorTag}`;
    const endTag = `// @AnchorEnd ${anchorTag}`;

@@ -66,7 +67,7 @@ export function replaceInAnchoredBlock(src, anchorTag, replaceBlockFn) {
 * @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
 * @returns {{ src: string, changed: boolean }}
 */
-export function replaceAnchoredMapBlock(src, {
+function replaceAnchoredMapBlock(src, {
    anchorTag,
    mapName,
    lines,
@@ -111,7 +112,7 @@ export function replaceAnchoredMapBlock(src, {
 * @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
 * @returns {{ src: string, changed: boolean }}
 */
-export function replaceAnchoredListBlock(src, {
+function replaceAnchoredListBlock(src, {
    anchorTag,
    listName,
    lines,
@@ -149,7 +150,7 @@ export function replaceAnchoredListBlock(src, {
 * @param {number} [options.linesIndent=4]
 * @param {string} [options.updatedLabel='']
 * @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
- * @param {Console} [options.logger=console]
+ * @param {RegExp} [options.regexForKeyMatching=null]
 * @returns {Promise<{ changed: boolean, content: string }>}
 */
 export async function updateAnchoredMapInFile(filePath, {
@@ -159,7 +160,7 @@ export async function updateAnchoredMapInFile(filePath, {
    linesIndent = 4,
    updatedLabel = '',
    toUpdatedStamp: toStamp = toUpdatedStamp,
-    logger = console,
+    regexForKeyMatching = null,
 }) {
    const filename = path.basename(filePath);
    const raw = await fsp.readFile(filePath, 'utf8');
@@ -167,9 +168,10 @@ export async function updateAnchoredMapInFile(filePath, {

    if (changed) {
        await fsp.writeFile(filePath, updated, 'utf8');
-        logger.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        printDifferences(raw, updated, { regexForKeyMatching });
    } else {
-        // logger.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        // console.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
    }
    return { changed, content: updated };
 }
@@ -183,7 +185,7 @@ export async function updateAnchoredMapInFile(filePath, {
 * @param {number} [options.linesIndent=4]
 * @param {string} [options.updatedLabel='']
 * @param {(date?: Date) => string} [options.toUpdatedStamp=toUpdatedStamp]
- * @param {Console} [options.logger=console]
+ * @param {RegExp} [options.regexForKeyMatching=null]
 * @returns {Promise<{ changed: boolean, content: string }>}
 */
 export async function updateAnchoredListInFile(filePath, {
@@ -193,7 +195,7 @@ export async function updateAnchoredListInFile(filePath, {
    linesIndent = 4,
    updatedLabel = '',
    toUpdatedStamp: toStamp = toUpdatedStamp,
-    logger = console,
+    regexForKeyMatching = null,
 }) {
    const filename = path.basename(filePath);
    const raw = await fsp.readFile(filePath, 'utf8');
@@ -201,9 +203,10 @@ export async function updateAnchoredListInFile(filePath, {

    if (changed) {
        await fsp.writeFile(filePath, updated, 'utf8');
-        logger.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        printDifferences(raw, updated, { regexForKeyMatching });
    } else {
-        // logger.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        // console.log(`[${filename}] No update needed` + (updatedLabel ? ` (${updatedLabel})` : ''));
    }
    return { changed, content: updated };
 }
@@ -217,16 +220,19 @@ export async function updateAnchoredListInFile(filePath, {
 * @param {AnchoredBlockUpdateOption[]} optionList
 * @param {Object} [extraOptions={}]
 * @param {(date?: Date) => string} [extraOptions.toUpdatedStamp=toUpdatedStamp]
- * @param {Console} [extraOptions.logger=console]
+ * @param {RegExp} [extraOptions.regexForKeyMatching=null]
 * @returns {Promise<{ changed: boolean, content: string }>}
 */
 export async function batchUpdateAnchoredBlocks(filePath, optionList, {
    toUpdatedStamp: toStamp = toUpdatedStamp,
-    logger = console,
+    regexForKeyMatching = null,
 } = {}) {
    const filename = path.basename(filePath);
+
    let raw = await fsp.readFile(filePath, 'utf8');
    let changedAny = false;
+    let updated = null;
+    let updatedLabel = null;

    for (const opt of optionList) {
        let res = { src: raw, changed: false };
@@ -250,21 +256,23 @@ export async function batchUpdateAnchoredBlocks(filePath, optionList, {
        } else if (opt.type === 'custom' && typeof opt.replacer === 'function') {
            res = replaceInAnchoredBlock(raw, opt.anchorTag, (block) => opt.replacer(block, { toUpdatedStamp: toStamp }));
        } else {
-            logger.warn(`[${filename}] Unknown operation type or missing parameters:`, opt);
+            console.warn(`[${filename}] Unknown operation type or missing parameters:`, opt);
            continue;
        }

        if (res.changed) {
            changedAny = true;
-            raw = res.src;
-            logger.log(`[${filename}] Updated (${opt.updatedLabel ?? opt.anchorTag})`);
-        } else {
-            // logger.log(`[${filename}] No update needed (${op['updatedLabel'] ?? op.anchorTag})`);
+            updated = res.src;
+            updatedLabel = opt.updatedLabel;
        }
    }

    if (changedAny) {
-        await fsp.writeFile(filePath, raw, 'utf8');
+        await fsp.writeFile(filePath, updated, 'utf8');
+        console.log(`[${filename}] Updated` + (updatedLabel ? ` (${updatedLabel})` : ''));
+        printDifferences(raw, updated, { regexForKeyMatching });
+    } else {
+        // console.log(`[${filename}] No update needed (${op['updatedLabel'] ?? op.anchorTag})`);
    }
-    return { changed: changedAny, content: raw };
+    return { changed: changedAny, content: changedAny ? updated : raw };
 }
--- a/.utils/utils/format.mjs
+++ b/.utils/utils/format.mjs
@@ -18,7 +18,7 @@ export function bytes2GiB(bytes, fractionDigits = 2) {
 * @example
 * // https://example.com?a=1&b=2
 * url('https://example.com', { a: 1, b: 2 });
- * 
+ *
 * @param {string} url
 * @param {Object} query
 * @returns {string}
@@ -26,7 +26,7 @@ export function bytes2GiB(bytes, fractionDigits = 2) {
 export function buildUrl(url, query) {
    if (!query) return url;
    const q = Object.entries(query)
-        .map(([key, value]) => `${key}=${encodeURIComponent(value)}`)
+        .map(([ key, value ]) => `${key}=${encodeURIComponent(value)}`)
        .join('&');
    return `${url}?${q}`;
 }
@@ -41,3 +41,21 @@ export function buildUrl(url, query) {
 export function escapeRegExp(string) {
    return string.replace(/[.*+?^${}()|\[\]\\]/g, '\\$&');
 }
+
+/**
+ * Converts an object into a string representation where each key-value pair
+ * is formatted as `"key": "value"` and separated by newline characters.<br>
+ * zh-CN:<br>
+ * 将对象转换为字符串表示形式, 每个键值对, 以 `"key": "value"` 的格式呈现并用换行符分隔.
+ *
+ * @example string
+ * "name": "John"
+ * "age": "30"
+ * "city": "New York"
+ *
+ * @param {Object} o 
+ * @return {string}
+ */
+export function objectToLines(o) {
+    return Object.entries(o).map(([ key, value ]) => `"${key}": "${value}"`).join(`\n`);
+}
--- a/.utils/utils/print.mjs
+++ b/.utils/utils/print.mjs
@@ -0,0 +1,136 @@
+/**
+ * @param {string} str
+ * @returns {string[]}
+ */
+const extractConcernedLines = (str) => str
+    .split(/,?\s*\r?\n/)
+    .map(s => s.trim())
+    .filter(s => s && s.match(/\w/) && !s.includes('@Updated'));
+
+/**
+ * @param {Array} a1
+ * @param {Array} a2
+ * @returns {Set}
+ */
+const findCommonSet = (a1, a2) => {
+    const s2 = new Set(a2);
+    return new Set(a1.filter(l => s2.has(l)));
+};
+
+/**
+ * @param {string} src
+ * @param {string} other
+ * @param {Object} [options={}]
+ * @param {RegExp} [options.regexForKeyMatching]
+ * @return {void}
+ */
+export function printDifferences(src, other, options = {}) {
+    const leftRaw = extractConcernedLines(src);
+    const rightRaw = extractConcernedLines(other);
+    const commonSet = findCommonSet(leftRaw, rightRaw);
+    const left = leftRaw.filter(l => !commonSet.has(l));
+    const right = rightRaw.filter(r => !commonSet.has(r));
+    const minLineLength = Math.min(...[ ...left, ...right ].map(l => l.length));
+
+    const keyOf = (/** @type {string} */ line) => {
+        const reKey = options.regexForKeyMatching ?? null;
+        if (reKey) {
+            const m = line.match(reKey);
+            return m ? m[0] : line;
+        }
+        const reClassicMap = /^"([^"]+)"\s+to\s+("[^"]+"|\d+)$/;
+        const matchedMap = line.match(reClassicMap);
+        if (matchedMap) {
+            return matchedMap[1];
+        }
+        const reClassicVersion = /^"(\d+(\.\d+)?)(\.\d+)(\s*[-_]\w+\s*)*"$/;
+        const matchedVersion = line.match(reClassicVersion);
+        if (matchedVersion) {
+            return matchedVersion[1];
+        }
+        return line.slice(0, Math.max(1, Math.ceil(minLineLength * 0.4))).trimEnd();
+    };
+
+    const lMap = new Map();
+    const rMap = new Map();
+    left.forEach(l => {
+        const k = keyOf(l);
+        if (!lMap.has(k)) lMap.set(k, [ l ]);
+        else lMap.get(k).push(l);
+    });
+    right.forEach(r => {
+        const k = keyOf(r);
+        if (!rMap.has(k)) rMap.set(k, [ r ]);
+        else rMap.get(k).push(r);
+    });
+
+    const deletions = [];
+    const modifications = [];
+    const additions = [];
+
+    for (const l of left) {
+        const k = keyOf(l);
+        if (!rMap.has(k)) {
+            deletions.push(`-- ${l}`);
+        } else {
+            const lLineList = lMap.get(k);
+            const rLineList = rMap.get(k);
+            const rLine = rLineList?.[0];
+            const shouldRecordModification = lLineList.length === 1 && rLineList.length === 1 && l !== rLine;
+            if (shouldRecordModification) {
+                modifications.push({ from: `-- ${l}`, to: `-> ${rLine}` });
+            } else {
+                deletions.push(`-- ${l}`);
+            }
+        }
+    }
+    for (const r of right) {
+        const k = keyOf(r);
+        if (!lMap.has(k)) {
+            additions.push(`++ ${r}`);
+        } else {
+            const lLineList = lMap.get(k);
+            const rLineList = rMap.get(k);
+            const lLine = lLineList?.[0];
+            const shouldRecordModification = lLineList.length === 1 && rLineList.length === 1 && r !== lLine;
+            if (!shouldRecordModification) {
+                additions.push(`++ ${r}`);
+            }
+        }
+    }
+
+    const allLines = [
+        ...deletions,
+        ...modifications.flatMap(m => [ m.from, m.to ]),
+        ...additions,
+    ];
+    const maxLen = allLines.reduce((mx, s) => Math.max(mx, s.length), 0);
+    const SEP_EQ = '='.repeat(Math.max(6, maxLen));
+    const SEP_DASH = '-'.repeat(Math.max(6, maxLen));
+
+    const sections = [];
+
+    if (deletions.length > 0) {
+        sections.push(deletions.slice());
+    }
+    if (modifications.length > 0) {
+        const modLines = [];
+        modifications.forEach((m, idx) => {
+            if (idx > 0) modLines.push(SEP_DASH);
+            modLines.push(m.from);
+            modLines.push(m.to);
+        });
+        sections.push(modLines);
+    }
+    if (additions.length > 0) {
+        sections.push(additions.slice());
+    }
+
+    sections.forEach((lines, idx) => {
+        console.log(SEP_EQ);
+        lines.forEach(l => console.log(l));
+        if (idx === sections.length - 1) {
+            console.log(SEP_EQ);
+        }
+    });
+}