6.7.0 - Alpha6 - Scrapers 工具 JSDoc @typedef 转换为 TypeScript 声明文件
This commit is contained in:
126
.utils/declarations/index.d.ts
vendored
Normal file
126
.utils/declarations/index.d.ts
vendored
Normal file
@@ -0,0 +1,126 @@
|
||||
type Page = import('puppeteer').Page;
|
||||
type Frame = import('puppeteer').Frame;
|
||||
type ElementHandle<T extends Node> = import('puppeteer').ElementHandle<T>;
|
||||
|
||||
type CommitsData = import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/commits']['response']['data'];
|
||||
type ReleasesData = import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/releases']['response']['data']
|
||||
type PullsData = import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/pulls']['response']['data'];
|
||||
type PullCommitsData = import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/pulls/{pull_number}/commits']['response']['data'];
|
||||
|
||||
type AndroidStudioStableArchiveItemKind = 'exe' | 'zip' | 'tar' | 'other';
|
||||
|
||||
type FindTargetRowsFilter = string | RegExp | ((s: string) => boolean);
|
||||
|
||||
type TableDataStructureItemName = string;
|
||||
type TableDataStructureItem = RegExp | ((s: string) => boolean | string);
|
||||
type TableDataStructureItemForPageEvaluate = string | RegExp | ((s: string) => boolean | string);
|
||||
|
||||
interface AndroidStudioArchiveItem {
|
||||
title: string;
|
||||
date: string;
|
||||
version: string | null;
|
||||
links: Array<{
|
||||
text: string;
|
||||
href: string;
|
||||
}>;
|
||||
checksums: { [filename: string]: string };
|
||||
}
|
||||
|
||||
interface AndroidStudioStableArchiveItem {
|
||||
platform: string;
|
||||
filename: string;
|
||||
size: string;
|
||||
sha256: string;
|
||||
url: string | null;
|
||||
kind: AndroidStudioStableArchiveItemKind;
|
||||
}
|
||||
|
||||
interface FindTargetRowsOptionsBase {
|
||||
/** @default 'table' */
|
||||
tableSelector?: 'table' | string;
|
||||
tableFilter?: { [selector: string]: FindTargetRowsFilter | FindTargetRowsFilter[] };
|
||||
/** @default 'tbody tr' */
|
||||
tableRowSelector?: 'tbody tr' | string;
|
||||
/** @default 'td' */
|
||||
tableDataSelector?: 'td' | string;
|
||||
/** @default [] */
|
||||
tableDataStructure?: Array<{ [dataItemName: TableDataStructureItemName]: any } | TableDataStructureItemName>;
|
||||
}
|
||||
|
||||
interface FindTargetRowsOptions extends FindTargetRowsOptionsBase {
|
||||
/** @default [] */
|
||||
tableDataStructure?: Array<{ [dataItemName: TableDataStructureItemName]: TableDataStructureItem } | TableDataStructureItemName>;
|
||||
}
|
||||
|
||||
interface FindTargetRowsOptionsForPageEvaluate extends FindTargetRowsOptionsBase {
|
||||
/** @default [] */
|
||||
tableDataStructure?: Array<{ [dataItemName: TableDataStructureItemName]: TableDataStructureItemForPageEvaluate } | TableDataStructureItemName>;
|
||||
}
|
||||
|
||||
interface PuppeteerOptions {
|
||||
url: string;
|
||||
/** @default 120000 */
|
||||
pageGoToTimeout?: 120000 | number;
|
||||
/** @default 30000 */
|
||||
findTargetRowsTimeout?: 30000 | number;
|
||||
}
|
||||
|
||||
interface KspRelease {
|
||||
version: string;
|
||||
name: string;
|
||||
publishedAt: string;
|
||||
}
|
||||
|
||||
interface UserProfile {
|
||||
login: string;
|
||||
name: string | null;
|
||||
}
|
||||
|
||||
interface StatisticsEntry {
|
||||
login: string;
|
||||
name: string | null;
|
||||
htmlUrl: string;
|
||||
totalCommitsInMergedPRs: number;
|
||||
latestCommitAt: string | null;
|
||||
}
|
||||
|
||||
interface DebugRecord {
|
||||
sha: string;
|
||||
belongs: boolean;
|
||||
author_login: string | null;
|
||||
committer_login: string | null;
|
||||
author_name: string | null;
|
||||
author_email: string | null;
|
||||
time: string | null;
|
||||
}
|
||||
|
||||
interface GradleRelease {
|
||||
versionName: string;
|
||||
releaseDate: string;
|
||||
link: {
|
||||
bin: string;
|
||||
all: string;
|
||||
checksums: string;
|
||||
}
|
||||
}
|
||||
|
||||
interface GradleReleaseConfig {
|
||||
majorVersionLimit?: number | string | null;
|
||||
format?: 'bin' | 'all';
|
||||
}
|
||||
|
||||
interface ScriptItem {
|
||||
name: string;
|
||||
abs: string;
|
||||
}
|
||||
|
||||
interface AnchoredBlockUpdateOption {
|
||||
type: 'map' | 'list' | 'custom';
|
||||
anchorTag: string;
|
||||
mapName?: string;
|
||||
listName?: string;
|
||||
lines: string[];
|
||||
linesIndent?: number;
|
||||
updatedLabel?: string;
|
||||
replacer?: (srcInBlock: string, options: { toUpdatedStamp?: (date?: Date) => string }) => { newBlock: string, changed: boolean };
|
||||
}
|
||||
@@ -6,21 +6,6 @@ import { fileURLToPath } from 'node:url';
|
||||
import { readPropertiesSync } from './utils/properties.mjs';
|
||||
import { sleep } from './utils/async.mjs';
|
||||
|
||||
/** @typedef {import('puppeteer').Page} Page */
|
||||
/** @typedef {import('puppeteer').Frame} Frame */
|
||||
/**
|
||||
* @template {Node} T
|
||||
* @typedef {import('puppeteer').ElementHandle<T>} ElementHandle
|
||||
*/
|
||||
/**
|
||||
* @typedef {Object} ArchiveItem
|
||||
* @property {string} title
|
||||
* @property {string} date
|
||||
* @property {string | null} version
|
||||
* @property {{ text: string, href: string }[]} links
|
||||
* @property {{ [filename: string]: string }} checksums
|
||||
*/
|
||||
|
||||
const URL = 'https://developer.android.com/studio/archive?hl=en';
|
||||
const SELECTOR_PRIMARY_BUTTON = 'button.button-primary';
|
||||
const SELECTOR_DEVSITE_EXPANDABLE = 'devsite-expandable';
|
||||
@@ -100,7 +85,7 @@ async function waitForFrameWithSelector(page, selector, timeoutMs = 30000) {
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<ArchiveItem[]>}
|
||||
* @returns {Promise<AndroidStudioArchiveItem[]>}
|
||||
*/
|
||||
export async function getAndroidStudioArchives() {
|
||||
const browser = await puppeteer.launch({
|
||||
@@ -156,7 +141,7 @@ export async function getAndroidStudioArchives() {
|
||||
throw new Error('Failed to find content frame');
|
||||
}
|
||||
|
||||
/** @type {ArchiveItem[]} */
|
||||
/** @type {AndroidStudioArchiveItem[]} */
|
||||
const archives = await contentFrame.$$eval(SELECTOR_DEVSITE_EXPANDABLE, nodes => {
|
||||
const pickText = (/** @type {Node | null} */ el) => String(el?.textContent ?? '').trim();
|
||||
return nodes.map(n => {
|
||||
|
||||
@@ -1,16 +1,5 @@
|
||||
// fetch-and-parse-android-studio-latest-stable-version.mjs
|
||||
|
||||
/** @typedef {'exe' | 'zip' | 'tar' | 'other'} StableArchiveItemKind */
|
||||
/**
|
||||
* @typedef {Object} StableArchiveItem
|
||||
* @property {string} platform
|
||||
* @property {string} filename
|
||||
* @property {string} size
|
||||
* @property {string} sha256
|
||||
* @property {string | null} url
|
||||
* @property {StableArchiveItemKind} kind
|
||||
*/
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
import { bytes2GiB } from './utils/format.mjs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -43,6 +32,7 @@ function buildDownloadUrlFromFilename(filename, kind) {
|
||||
}
|
||||
return `https://redirector.gvt1.com/edgedl/android/studio/${urlType}/${version}/${filename}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @returns {Promise<string>}
|
||||
@@ -60,7 +50,7 @@ async function fetchHtml(url) {
|
||||
|
||||
/**
|
||||
* @param {string} html
|
||||
* @returns {Promise<StableArchiveItem[]>}
|
||||
* @returns {Promise<AndroidStudioStableArchiveItem[]>}
|
||||
*/
|
||||
async function parseItemsFromHtml(html) {
|
||||
const $ = cheerio.load(html);
|
||||
@@ -106,7 +96,7 @@ async function parseItemsFromHtml(html) {
|
||||
* 返回形如: [ { platform, filename, size, sha256, url, kind: 'exe'|'zip'|'tar' } ]
|
||||
*
|
||||
* @param {string} [sourceUrl=URL]
|
||||
* @returns {Promise<StableArchiveItem[]>}
|
||||
* @returns {Promise<AndroidStudioStableArchiveItem[]>}
|
||||
*/
|
||||
export async function getLatestStableArchives(sourceUrl = URL) {
|
||||
const html = await fetchHtml(sourceUrl);
|
||||
|
||||
@@ -1,31 +1,5 @@
|
||||
// fetch-and-parse-autojs6-merged-pr-commits-statistics.mjs
|
||||
|
||||
/** @typedef {import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/pulls']['response']['data']} PullsData */
|
||||
/** @typedef {import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/pulls/{pull_number}/commits']['response']['data']} PullCommitsData */
|
||||
/**
|
||||
* @typedef {Object} UserProfile
|
||||
* @property {string} login
|
||||
* @property {string | null} name
|
||||
*/
|
||||
/**
|
||||
* @typedef {Object} StatisticsEntry
|
||||
* @property {string} login
|
||||
* @property {string | null} name
|
||||
* @property {string} htmlUrl
|
||||
* @property {number} totalCommitsInMergedPRs
|
||||
* @property {string | null} latestCommitAt
|
||||
*/
|
||||
/**
|
||||
* @typedef {Object} DebugRecord
|
||||
* @property {string} sha
|
||||
* @property {boolean} belongs
|
||||
* @property {string | null} author_login
|
||||
* @property {string | null} committer_login
|
||||
* @property {string | null} author_name
|
||||
* @property {string | null} author_email
|
||||
* @property {string | null} time
|
||||
*/
|
||||
|
||||
import * as dotenv from 'dotenv';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { httpFetch } from './utils/fetch.mjs';
|
||||
|
||||
@@ -1,15 +1,5 @@
|
||||
// fetch-and-parse-gradle-releases.mjs
|
||||
|
||||
/**
|
||||
* @typedef {Object} GradleRelease
|
||||
* @property {string} versionName
|
||||
* @property {string} releaseDate
|
||||
* @property {Object} link
|
||||
* @property {string} link.bin
|
||||
* @property {string} link.all
|
||||
* @property {string} link.checksums
|
||||
*/
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
// run-scrapers.mjs
|
||||
|
||||
/**
|
||||
* @typedef {Object} ScriptItem
|
||||
* @property {string} name
|
||||
* @property {string} abs
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as readline from 'node:readline';
|
||||
|
||||
@@ -1,14 +1,5 @@
|
||||
// scrape-and-inject-android-studio-codename_maps.mjs
|
||||
|
||||
/** @typedef {import('puppeteer').Page} Page */
|
||||
/** @typedef {import('puppeteer').Frame} Frame */
|
||||
/** @typedef {import('./fetch-and-parse-android-studio-latest-stable-version.mjs').StableArchiveItem} Row */
|
||||
/** @typedef {import('./fetch-and-parse-android-studio-archives.mjs').ArchiveItem} ArchiveItem */
|
||||
/**
|
||||
* @template {Node} T
|
||||
* @typedef {import('puppeteer').ElementHandle<T>} ElementHandle
|
||||
*/
|
||||
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import { batchUpdateAnchoredBlocks } from './utils/anchors.mjs';
|
||||
@@ -47,7 +38,7 @@ const manualCodenameOverrides = {};
|
||||
* complete and update common.json.<br>
|
||||
* zh-CN: 用 "最新稳定版" 的校验和/文件名, 在归档中定位条目, 补全并更新 common.json.
|
||||
*
|
||||
* @param {ArchiveItem[]} archives
|
||||
* @param {AndroidStudioArchiveItem[]} archives
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function updateLatestArchiveInfo(archives) {
|
||||
@@ -63,9 +54,9 @@ async function updateLatestArchiveInfo(archives) {
|
||||
* Search the archive: first try to match by sha256, then by filename.<br>
|
||||
* zh-CN: 在归档中查找: 优先用 sha256 命中, 其次用文件名.
|
||||
*
|
||||
* @param {ArchiveItem[]} rows
|
||||
* @param {Row} target
|
||||
* @returns {ArchiveItem | null}
|
||||
* @param {AndroidStudioArchiveItem[]} rows
|
||||
* @param {AndroidStudioStableArchiveItem} target
|
||||
* @returns {AndroidStudioArchiveItem | null}
|
||||
*/
|
||||
const matchArchive = (rows, target) => {
|
||||
for (const arc of rows) {
|
||||
@@ -159,7 +150,7 @@ async function updateLatestArchiveInfo(archives) {
|
||||
* Summarize codename-to-version mappings and codename first release dates.<br>
|
||||
* zh-CN: 汇总代号版本映射以及代号的首发日期.
|
||||
*
|
||||
* @param {ArchiveItem[]} archives
|
||||
* @param {AndroidStudioArchiveItem[]} archives
|
||||
*/
|
||||
function getCodenameMapLinesInfo(archives) {
|
||||
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
// scrape-and-inject-gradle-kotlin-compatibility-list.mjs
|
||||
|
||||
/** @typedef {import('puppeteer').Page} Page */
|
||||
|
||||
import { findTargetRows } from './utils/puppeteer-helpers.mjs';
|
||||
import { compareVersionStrings } from './utils/versioning.mjs';
|
||||
import { getMinSupportedGradleVersion } from './utils/properties.mjs';
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
// scrape-and-inject-ksp-releases.mjs
|
||||
|
||||
/** @typedef {import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/releases']['response']['data']} ReleasesData */
|
||||
/**
|
||||
* @typedef {Object} KspRelease
|
||||
* @property {string} version
|
||||
* @property {string} name
|
||||
* @property {string} publishedAt
|
||||
*/
|
||||
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import { httpFetch } from './utils/fetch.mjs';
|
||||
import { readProperties } from './utils/properties.mjs';
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
// scrape-and-inject-latest-gradle-wrapper.mjs
|
||||
|
||||
/** @typedef {import('./fetch-and-parse-gradle-releases.mjs').GradleRelease} GradleRelease */
|
||||
/**
|
||||
* @typedef {Object} GradleReleaseConfig
|
||||
* @property {number | string | null} [majorVersionLimit=null]
|
||||
* @property {'bin' | 'all'} [format='bin']
|
||||
*/
|
||||
|
||||
import { compareVersionStrings } from './utils/versioning.mjs';
|
||||
import { fetchGradleReleases } from './fetch-and-parse-gradle-releases.mjs';
|
||||
import { readPropertiesSync, writePropertiesSync } from './utils/properties.mjs';
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
// utils/anchors.mjs
|
||||
|
||||
/**
|
||||
* @typedef {Object} AnchoredBlockUpdateOption
|
||||
* @property {'map' | 'list' | 'custom'} type
|
||||
* @property {string} anchorTag
|
||||
* @property {string} [mapName]
|
||||
* @property {string} [listName]
|
||||
* @property {string[]} lines
|
||||
* @property {number} [linesIndent=4]
|
||||
* @property {string} [updatedLabel]
|
||||
* @property {(srcInBlock: string, options: { toUpdatedStamp?: (date?: Date) => string }) => { newBlock: string, changed: boolean }} [replacer]
|
||||
*/
|
||||
|
||||
import * as fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import { escapeRegExp } from './format.mjs';
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
// utils/fetch.mjs
|
||||
|
||||
/** @typedef {import('@octokit/types').Endpoints['GET /repos/{owner}/{repo}/commits']['response']['data']} CommitsData */
|
||||
|
||||
import * as dotenv from 'dotenv';
|
||||
import * as https from 'https';
|
||||
import nodeFetch from 'node-fetch';
|
||||
|
||||
@@ -1,33 +1,5 @@
|
||||
// utils/puppeteer-helpers.mjs
|
||||
|
||||
/** @typedef {import('puppeteer').Page} Page */
|
||||
/** @typedef {string | RegExp | ((s: string) => boolean)} FindTargetRowsFilter */
|
||||
/** @typedef {string} TableDataStructureItemName */
|
||||
/** @typedef {RegExp | ((s: string) => boolean | string)} TableDataStructureItem */
|
||||
/** @typedef {string | RegExp | ((s: string) => boolean | string)} TableDataStructureItemForPageEvaluate */
|
||||
/**
|
||||
* @typedef {object} FindTargetRowsOptions
|
||||
* @property {string} [tableSelector='table']
|
||||
* @property {{ [selector: string]: FindTargetRowsFilter | FindTargetRowsFilter[] }}[tableFilter={}]
|
||||
* @property {string} [tableRowSelector='tbody tr']
|
||||
* @property {string} [tableDataSelector='td']
|
||||
* @property {Array<{ [dataItemName: TableDataStructureItemName]: TableDataStructureItem } | TableDataStructureItemName>} [tableDataStructure=[]]
|
||||
*/
|
||||
/**
|
||||
* @typedef {object} FindTargetRowsOptionsForPageEvaluate
|
||||
* @property {string} [tableSelector='table']
|
||||
* @property {{ [selector: string]: FindTargetRowsFilter | FindTargetRowsFilter[] }}[tableFilter={}]
|
||||
* @property {string} [tableRowSelector='tbody tr']
|
||||
* @property {string} [tableDataSelector='td']
|
||||
* @property {Array<{ [dataItemName: TableDataStructureItemName]: TableDataStructureItemForPageEvaluate } | TableDataStructureItemName>} [tableDataStructure=[]]
|
||||
*/
|
||||
/**
|
||||
* @typedef {object} PuppeteerOptions
|
||||
* @property {string} url
|
||||
* @property {number} [pageGoToTimeout=120000]
|
||||
* @property {number} [findTargetRowsTimeout=30000]
|
||||
*/
|
||||
|
||||
import puppeteer from 'puppeteer';
|
||||
import { sleep } from './async.mjs';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user