Skip to content

N
News Hub Skills
Commit 29d558a9 authored by 谢宇轩's avatar 谢宇轩
Browse files
Options

fix: ensure chromium bug

parent 95cd2d21
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 36 deletions
skills/news-harvester/scripts/ensure-chromium.js
View file @ 29d558a9
......@@ -30,12 +30,20 @@ const fs = require('fs');
const path = require('path');
const http = require('http');
const os = require('os');
const { spawn, execFileSync } = require('child_process');
const { spawn, execFileSync, execSync } = require('child_process');
const SKILL_DIR = path.dirname(__dirname);
const CONFIG_PATH = path.join(SKILL_DIR, 'config.json');
const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json');
// Persistent default profile. Using os.tmpdir() (the old default) is unsafe on
// distros that mount /tmp as tmpfs (e.g. Arch Linux) — it is wiped on reboot,
// taking login cookies with it. A fixed path under the home dir survives reboots
// and is shared by both --login (windowed) and headless harvest launches, so a
// login done via --login is reused by later harvests. This matches the profile
// --login already defaults to, keeping the two code paths consistent.
const DEFAULT_PROFILE = path.join(os.homedir(), '.news-harvester', 'chromium-profile');
// ── Config ──────────────────────────────────────────────────────────────────
function loadConfig() {
......@@ -50,7 +58,7 @@ function resolveOptions(config, portOverride) {
const chromium = (config && config.chromium) || {};
const cdpPort = parseInt(portOverride || process.env.CDP_PORT_OVERRIDE || chromium.cdpPort || '9222', 10);
const headless = chromium.headless !== false; // default true, set false to keep a window
const userDataDir = chromium.userDataDir || path.join(os.tmpdir(), 'news-harvester-chromium');
const userDataDir = chromium.userDataDir || DEFAULT_PROFILE;
return { cdpPort, headless, userDataDir, executablePath: chromium.executablePath || null };
}
......@@ -178,7 +186,7 @@ function cleanStaleLocks(userDataDir) {
// ── Launch ──────────────────────────────────────────────────────────────────
function spawnBrowser(bin, opts) {
const { cdpPort, headless, userDataDir, url = 'about:blank', windowsHide = true } = opts;
const { cdpPort, headless, userDataDir, url = 'chrome://newtab', windowsHide = true } = opts;
fs.mkdirSync(userDataDir, { recursive: true });
cleanStaleLocks(userDataDir);
......@@ -276,7 +284,11 @@ function looksLikeDisplayFailure(log) {
// When headless=false, we must keep at least one tab open as a "keep-alive" tab
// so that harvest sessions closing their own tabs don't accidentally kill the
// browser. We keep the FIRST startup tab and only close the rest (if any).
const STARTUP_TAB_URLS = new Set(['about:blank', 'chrome://newtab/', 'chrome://newtab']);
const STARTUP_TAB_URLS = new Set([
'about:blank',
'chrome://newtab/', 'chrome://newtab',
'chrome://new-tab-page/', 'chrome://new-tab-page',
]);
function closeStartupTabs(port, keepAlive = false) {
return new Promise((resolve) => {
......@@ -376,7 +388,7 @@ WHAT IT DOES
Override discovery with config.chromium.executablePath.
3. Cleans stale SingletonLock files left by a crashed browser.
4. Spawns the browser (headless=new by default; --no-sandbox --disable-gpu,
remote-debugging-port=<port>, user-data-dir=<dir>, about:blank).
remote-debugging-port=<port>, user-data-dir=<dir>, chrome://newtab).
5. Polls /json/version until ready (up to 30s, every 500ms).
6. On timeout, prints binary path / port / userDataDir / stderr-log tail.
......@@ -389,7 +401,7 @@ WHAT IT DOES
Launches a WINDOWED browser (headless=false) with CDP enabled so the user can
log into a site interactively. The profile persists, so later headless harvests
reuse the same cookies/login state.
--url <url> Page to open (default about:blank; pass the source homepage)
--url <url> Page to open (default chrome://newtab; pass the source homepage)
--port <port> CDP port (default config.chromium.cdpPort / 9222)
--profile <dir> Profile dir (default config.chromium.userDataDir, else
~/.news-harvester/chromium-profile — a fixed path, NOT the
......@@ -403,10 +415,10 @@ CONFIG (config.json, all optional — see SKILL.md "Configuration")
chromium.cdpPort CDP port (default 9222; env CDP_PORT_OVERRIDE wins)
chromium.headless true (default) | false (keep a window)
chromium.executablePath full path to browser binary (null = auto-discover)
chromium.userDataDir browser profile dir (null = <tmp>/news-harvester-chromium).
Set this to a fixed profile you've logged into once
(via --login) so future auto-launches keep your
cookies/login state.
chromium.userDataDir browser profile dir (null = ~/.news-harvester/chromium-profile,
a fixed path so login state survives reboots). Set this to a
profile you've logged into once (via --login) so future
auto-launches keep your cookies/login state.
EXIT CODES
0 CDP is up (already running or just launched); --list ok; --login ok/already-running
......@@ -452,27 +464,62 @@ async function cmdList(config) {
console.log(` cdpPort: ${opts.cdpPort}`);
console.log(` headless: ${opts.headless}`);
console.log(` executablePath: ${opts.executablePath || '(auto-discover)'}`);
console.log(` userDataDir: ${opts.userDataDir || '(ephemeral tmp — set a fixed path for login persistence)'}`);
console.log(` userDataDir: ${opts.userDataDir || '(default ~/.news-harvester/chromium-profile)'}`);
process.exit(0);
}
// ── --login: windowed browser for interactive login ─────────────────────────
const DEFAULT_PROFILE = path.join(os.homedir(), '.news-harvester', 'chromium-profile');
// Find the PID actually listening on `port`. /json/version only tells us the
// browser is *somehow* reachable; it can't tell us which process to kill when
// --login wants the port but a headless harvest (or a stale Chromium) holds it.
// Returns a PID string or null. Best-effort: lsof is common on macOS/Linux;
// netstat on Windows. Errors are swallowed (we fall back to the generic hint).
function listenerPid(port) {
const cmd = process.platform === 'win32'
? `netstat -ano -p TCP | findstr "LISTENING" | findstr ":${port}"`
: `lsof -nP -iTCP:${port} -sTCP:LISTEN`;
try {
const out = execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString();
if (process.platform === 'win32') {
// netstat -ano line: ...LISTENING <pid>
const m = out.match(/(\d+)\s*$/m);
return m ? m[1] : null;
}
// lsof line: COMMAND PID USER ... TCP *:9222 (LISTEN)
const m = out.match(/^\S+\s+(\d+)\s/m);
return m ? m[1] : null;
} catch (e) { return null; }
}
async function cmdLogin(config, flagUrl, flagPort, flagProfile) {
const chromium = (config && config.chromium) || {};
const port = parseInt(flagPort || chromium.cdpPort || '9222', 10);
// A persistent profile is MANDATORY for login; never fall back to the tmp dir.
const profile = flagProfile || chromium.userDataDir || DEFAULT_PROFILE;
const url = flagUrl || 'about:blank';
const url = flagUrl || 'chrome://newtab';
// 1. Port already occupied? Report and exit 0 (don't clash).
// 1. Port already occupied? Don't clash — print the exact kill command.
// We only print; we never kill the process ourselves (it may hold
// unsaved state, or be a harvest run the user didn't mean to interrupt).
try {
const v = await cdpVersion(port, 1500);
console.log(`ℹ️ A browser is already running on CDP port :${port} — ${v.Browser}`);
console.log(' If that is a headless harvest instance, close it first (Cmd+Q / Ctrl+Q), then re-run --login.');
console.log(' (Not reusing it: it may be a different profile, and a second instance would hit a SingletonLock.)');
console.log(`ℹ️ CDP port :${port} is already occupied by a running browser — ${v.Browser}`);
console.log(' --login does not reuse it (it may be a headless harvest instance using a');
console.log(' different profile, and a second instance would hit a SingletonLock).');
const pid = listenerPid(port);
if (pid) {
const kill = process.platform === 'win32' ? `taskkill /F /PID ${pid}` : `kill ${pid}`;
console.log(`\n To free the port, stop that browser then re-run --login:`);
console.log(` ${kill} # PID ${pid} (verify with: ${process.platform === 'win32' ? `tasklist /FI "PID eq ${pid}"` : `ps -p ${pid}`})`);
console.log(` # or fully quit the browser: Cmd+Q (macOS) / Ctrl+Q (Linux/Windows)`);
} else {
console.log('\n To free the port, fully quit the browser holding it, then re-run --login.');
console.log(' (Could not identify the listening PID automatically; check with:');
console.log(process.platform === 'win32'
? ` netstat -ano -p TCP | findstr ":${port}"`
: ` lsof -nP -iTCP:${port} -sTCP:LISTEN)`);
}
process.exit(0);
} catch (e) { /* port free — proceed to launch */ }
......@@ -510,7 +557,7 @@ async function cmdLogin(config, flagUrl, flagPort, flagProfile) {
}
// 5. Guide the user through login.
const siteHint = url !== 'about:blank' ? ` at ${url}` : '';
const siteHint = url !== 'chrome://newtab' ? ` at ${url}` : '';
console.log(`\n🔐 A browser window has opened${siteHint}.`);
console.log(' 1. Log into the site inside that window (it uses the profile above).');
console.log(' 2. When done, QUIT the browser fully: Cmd+Q (macOS) / Ctrl+Q (Linux/Windows).');
......
skills/news-source-analyzer/scripts/ensure-chromium.js
View file @ 29d558a9
......@@ -30,12 +30,20 @@ const fs = require('fs');
const path = require('path');
const http = require('http');
const os = require('os');
const { spawn, execFileSync } = require('child_process');
const { spawn, execFileSync, execSync } = require('child_process');
const SKILL_DIR = path.dirname(__dirname);
const CONFIG_PATH = path.join(SKILL_DIR, 'config.json');
const TEMPLATE_PATH = path.join(SKILL_DIR, 'references', 'config-template.json');
// Persistent default profile. Using os.tmpdir() (the old default) is unsafe on
// distros that mount /tmp as tmpfs (e.g. Arch Linux) — it is wiped on reboot,
// taking login cookies with it. A fixed path under the home dir survives reboots
// and is shared by both --login (windowed) and headless harvest launches, so a
// login done via --login is reused by later harvests. This matches the profile
// --login already defaults to, keeping the two code paths consistent.
const DEFAULT_PROFILE = path.join(os.homedir(), '.news-harvester', 'chromium-profile');
// ── Config ──────────────────────────────────────────────────────────────────
function loadConfig() {
......@@ -50,7 +58,7 @@ function resolveOptions(config, portOverride) {
const chromium = (config && config.chromium) || {};
const cdpPort = parseInt(portOverride || process.env.CDP_PORT_OVERRIDE || chromium.cdpPort || '9222', 10);
const headless = chromium.headless !== false; // default true, set false to keep a window
const userDataDir = chromium.userDataDir || path.join(os.tmpdir(), 'news-harvester-chromium');
const userDataDir = chromium.userDataDir || DEFAULT_PROFILE;
return { cdpPort, headless, userDataDir, executablePath: chromium.executablePath || null };
}
......@@ -178,7 +186,7 @@ function cleanStaleLocks(userDataDir) {
// ── Launch ──────────────────────────────────────────────────────────────────
function spawnBrowser(bin, opts) {
const { cdpPort, headless, userDataDir, url = 'about:blank', windowsHide = true } = opts;
const { cdpPort, headless, userDataDir, url = 'chrome://newtab', windowsHide = true } = opts;
fs.mkdirSync(userDataDir, { recursive: true });
cleanStaleLocks(userDataDir);
......@@ -276,7 +284,11 @@ function looksLikeDisplayFailure(log) {
// When headless=false, we must keep at least one tab open as a "keep-alive" tab
// so that harvest sessions closing their own tabs don't accidentally kill the
// browser. We keep the FIRST startup tab and only close the rest (if any).
const STARTUP_TAB_URLS = new Set(['about:blank', 'chrome://newtab/', 'chrome://newtab']);
const STARTUP_TAB_URLS = new Set([
'about:blank',
'chrome://newtab/', 'chrome://newtab',
'chrome://new-tab-page/', 'chrome://new-tab-page',
]);
function closeStartupTabs(port, keepAlive = false) {
return new Promise((resolve) => {
......@@ -376,7 +388,7 @@ WHAT IT DOES
Override discovery with config.chromium.executablePath.
3. Cleans stale SingletonLock files left by a crashed browser.
4. Spawns the browser (headless=new by default; --no-sandbox --disable-gpu,
remote-debugging-port=<port>, user-data-dir=<dir>, about:blank).
remote-debugging-port=<port>, user-data-dir=<dir>, chrome://newtab).
5. Polls /json/version until ready (up to 30s, every 500ms).
6. On timeout, prints binary path / port / userDataDir / stderr-log tail.
......@@ -389,7 +401,7 @@ WHAT IT DOES
Launches a WINDOWED browser (headless=false) with CDP enabled so the user can
log into a site interactively. The profile persists, so later headless harvests
reuse the same cookies/login state.
--url <url> Page to open (default about:blank; pass the source homepage)
--url <url> Page to open (default chrome://newtab; pass the source homepage)
--port <port> CDP port (default config.chromium.cdpPort / 9222)
--profile <dir> Profile dir (default config.chromium.userDataDir, else
~/.news-harvester/chromium-profile — a fixed path, NOT the
......@@ -403,10 +415,10 @@ CONFIG (config.json, all optional — see SKILL.md "Configuration")
chromium.cdpPort CDP port (default 9222; env CDP_PORT_OVERRIDE wins)
chromium.headless true (default) | false (keep a window)
chromium.executablePath full path to browser binary (null = auto-discover)
chromium.userDataDir browser profile dir (null = <tmp>/news-harvester-chromium).
Set this to a fixed profile you've logged into once
(via --login) so future auto-launches keep your
cookies/login state.
chromium.userDataDir browser profile dir (null = ~/.news-harvester/chromium-profile,
a fixed path so login state survives reboots). Set this to a
profile you've logged into once (via --login) so future
auto-launches keep your cookies/login state.
EXIT CODES
0 CDP is up (already running or just launched); --list ok; --login ok/already-running
......@@ -452,27 +464,62 @@ async function cmdList(config) {
console.log(` cdpPort: ${opts.cdpPort}`);
console.log(` headless: ${opts.headless}`);
console.log(` executablePath: ${opts.executablePath || '(auto-discover)'}`);
console.log(` userDataDir: ${opts.userDataDir || '(ephemeral tmp — set a fixed path for login persistence)'}`);
console.log(` userDataDir: ${opts.userDataDir || '(default ~/.news-harvester/chromium-profile)'}`);
process.exit(0);
}
// ── --login: windowed browser for interactive login ─────────────────────────
const DEFAULT_PROFILE = path.join(os.homedir(), '.news-harvester', 'chromium-profile');
// Find the PID actually listening on `port`. /json/version only tells us the
// browser is *somehow* reachable; it can't tell us which process to kill when
// --login wants the port but a headless harvest (or a stale Chromium) holds it.
// Returns a PID string or null. Best-effort: lsof is common on macOS/Linux;
// netstat on Windows. Errors are swallowed (we fall back to the generic hint).
function listenerPid(port) {
const cmd = process.platform === 'win32'
? `netstat -ano -p TCP | findstr "LISTENING" | findstr ":${port}"`
: `lsof -nP -iTCP:${port} -sTCP:LISTEN`;
try {
const out = execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] }).toString();
if (process.platform === 'win32') {
// netstat -ano line: ...LISTENING <pid>
const m = out.match(/(\d+)\s*$/m);
return m ? m[1] : null;
}
// lsof line: COMMAND PID USER ... TCP *:9222 (LISTEN)
const m = out.match(/^\S+\s+(\d+)\s/m);
return m ? m[1] : null;
} catch (e) { return null; }
}
async function cmdLogin(config, flagUrl, flagPort, flagProfile) {
const chromium = (config && config.chromium) || {};
const port = parseInt(flagPort || chromium.cdpPort || '9222', 10);
// A persistent profile is MANDATORY for login; never fall back to the tmp dir.
const profile = flagProfile || chromium.userDataDir || DEFAULT_PROFILE;
const url = flagUrl || 'about:blank';
const url = flagUrl || 'chrome://newtab';
// 1. Port already occupied? Report and exit 0 (don't clash).
// 1. Port already occupied? Don't clash — print the exact kill command.
// We only print; we never kill the process ourselves (it may hold
// unsaved state, or be a harvest run the user didn't mean to interrupt).
try {
const v = await cdpVersion(port, 1500);
console.log(`ℹ️ A browser is already running on CDP port :${port} — ${v.Browser}`);
console.log(' If that is a headless harvest instance, close it first (Cmd+Q / Ctrl+Q), then re-run --login.');
console.log(' (Not reusing it: it may be a different profile, and a second instance would hit a SingletonLock.)');
console.log(`ℹ️ CDP port :${port} is already occupied by a running browser — ${v.Browser}`);
console.log(' --login does not reuse it (it may be a headless harvest instance using a');
console.log(' different profile, and a second instance would hit a SingletonLock).');
const pid = listenerPid(port);
if (pid) {
const kill = process.platform === 'win32' ? `taskkill /F /PID ${pid}` : `kill ${pid}`;
console.log(`\n To free the port, stop that browser then re-run --login:`);
console.log(` ${kill} # PID ${pid} (verify with: ${process.platform === 'win32' ? `tasklist /FI "PID eq ${pid}"` : `ps -p ${pid}`})`);
console.log(` # or fully quit the browser: Cmd+Q (macOS) / Ctrl+Q (Linux/Windows)`);
} else {
console.log('\n To free the port, fully quit the browser holding it, then re-run --login.');
console.log(' (Could not identify the listening PID automatically; check with:');
console.log(process.platform === 'win32'
? ` netstat -ano -p TCP | findstr ":${port}"`
: ` lsof -nP -iTCP:${port} -sTCP:LISTEN)`);
}
process.exit(0);
} catch (e) { /* port free — proceed to launch */ }
......@@ -510,7 +557,7 @@ async function cmdLogin(config, flagUrl, flagPort, flagProfile) {
}
// 5. Guide the user through login.
const siteHint = url !== 'about:blank' ? ` at ${url}` : '';
const siteHint = url !== 'chrome://newtab' ? ` at ${url}` : '';
console.log(`\n🔐 A browser window has opened${siteHint}.`);
console.log(' 1. Log into the site inside that window (it uses the profile above).');
console.log(' 2. When done, QUIT the browser fully: Cmd+Q (macOS) / Ctrl+Q (Linux/Windows).');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment