mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2026-10-16 21:13:18 +08:00
Merge pull request #999 from sora-ot/release/0.6.5
Fix macOS packaging process and fix the bug preventing multiple URL execution.
This commit is contained in:
+1
-2
@@ -58,7 +58,6 @@ xattr: [Errno 13] Permission denied: 'EasySpider.app/Contents/Resources
|
||||
|
||||
Now, you can design tasks as you would in other operating systems.
|
||||
|
||||
|
||||
- Execution phase
|
||||
|
||||
The operation is the same as the design phase of the Intel version. When running the 'easyspider_executestage' program for the first time, you need to set "Allow Anyway" in System Settings -> Security & Privacy, and re-run the "./easyspider_executestage EID" command, and click "Open" to run the task.
|
||||
@@ -67,4 +66,4 @@ During the execution of the task, if an error similar to the following occurs, i
|
||||
|
||||
Traceback (most recent call last): File "multiprocessing/resource_tracker.py", line 209, in main KeyError: '/mp-5dxyey7c'
|
||||
|
||||
File access permissions must be granted, but microphone permissions are not needed at all. The author is also unclear why microphone access permissions would be requested, so you can refuse.
|
||||
File access permissions must be granted, but microphone permissions are not needed at all. The author is also unclear why microphone access permissions would be requested, so you can refuse.
|
||||
|
||||
@@ -55,8 +55,6 @@ xattr: [Errno 13] Permission denied: 'EasySpider.app/Contents/Resources/app/node
|
||||
|
||||
现在,您就可以像在其他操作系统中一样设计任务了。
|
||||
|
||||
|
||||
|
||||
- 执行阶段
|
||||
|
||||
与Intel版本设计阶段操作相同,首次运行'easyspider_executestage'程序时,需要在系统设置 -> 安全性与隐私中设置“仍然允许”,并重新运行"./easyspider_executestage EID"命令,并点击“打开”来运行任务。
|
||||
@@ -65,4 +63,4 @@ xattr: [Errno 13] Permission denied: 'EasySpider.app/Contents/Resources/app/node
|
||||
|
||||
Traceback (most recent call last): File "multiprocessing/resource_tracker.py", line 209, in main KeyError: '/mp-5dxyey7c'
|
||||
|
||||
必须授予文件访问权限,但根本不需要麦克风权限。作者也不清楚为什么会请求麦克风访问权限,所以可以拒绝。
|
||||
必须授予文件访问权限,但根本不需要麦克风权限。作者也不清楚为什么会请求麦克风访问权限,所以可以拒绝。
|
||||
|
||||
@@ -16,6 +16,7 @@ execution_instances/*
|
||||
info.log
|
||||
npminstall-debug.log
|
||||
mysql_config.json
|
||||
macos_command_folder.json
|
||||
EasySpider_en/
|
||||
EasySpider_zh/
|
||||
TempUserDataFolder/
|
||||
|
||||
+155
-11
@@ -36,6 +36,64 @@ let config_context = JSON.parse(
|
||||
fs.readFileSync(path.join(task_server.getDir(), `config.json`), "utf8")
|
||||
); //仅在当前进程中使用,不会写入文件
|
||||
|
||||
const macOSCommandFolderConfigPath = path.join(
|
||||
task_server.getDir(),
|
||||
"macos_command_folder.json"
|
||||
);
|
||||
|
||||
function getMacOSCommandLauncher(folder) {
|
||||
if (!folder) {
|
||||
return "";
|
||||
}
|
||||
const absoluteFolder = path.resolve(folder);
|
||||
const appPath = path.join(absoluteFolder, "EasySpider.app");
|
||||
const scriptPath = path.join(absoluteFolder, "execute_macos.sh");
|
||||
const executablePath = path.join(absoluteFolder, "easyspider_executestage");
|
||||
if (!fs.existsSync(appPath) || !fs.existsSync(executablePath)) {
|
||||
return "";
|
||||
}
|
||||
// New packages include the wrapper so it can resolve its own directory.
|
||||
// Retain compatibility with older portable releases that only contain
|
||||
// the execution-stage binary.
|
||||
return fs.existsSync(scriptPath) ? scriptPath : executablePath;
|
||||
}
|
||||
|
||||
function readSavedMacOSCommandFolder() {
|
||||
try {
|
||||
const saved = JSON.parse(
|
||||
fs.readFileSync(macOSCommandFolderConfigPath, "utf8")
|
||||
);
|
||||
return getMacOSCommandLauncher(saved.folder) ? path.resolve(saved.folder) : "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function detectMacOSCommandFolder() {
|
||||
const savedFolder = readSavedMacOSCommandFolder();
|
||||
if (savedFolder) {
|
||||
return savedFolder;
|
||||
}
|
||||
if (process.platform !== "darwin" || !app.isPackaged) {
|
||||
return "";
|
||||
}
|
||||
// This resolves to the directory containing EasySpider.app during a
|
||||
// normal launch. Under App Translocation it resolves to .../d, which
|
||||
// does not contain the portable executor and therefore fails validation.
|
||||
const appBundlePath = path.resolve(process.resourcesPath, "../..");
|
||||
const candidate = path.dirname(appBundlePath);
|
||||
return getMacOSCommandLauncher(candidate) ? candidate : "";
|
||||
}
|
||||
|
||||
function saveMacOSCommandFolder(folder) {
|
||||
const absoluteFolder = path.resolve(folder);
|
||||
fs.writeFileSync(
|
||||
macOSCommandFolderConfigPath,
|
||||
JSON.stringify({folder: absoluteFolder}, null, 2)
|
||||
);
|
||||
return absoluteFolder;
|
||||
}
|
||||
|
||||
if (config.debug) {
|
||||
let logPath = "info.log";
|
||||
let logFile = fs.createWriteStream(logPath, {flags: "a"});
|
||||
@@ -97,7 +155,11 @@ if (process.platform === "win32" && process.arch === "ia32") {
|
||||
__dirname,
|
||||
"chrome_mac64.app/Contents/MacOS/Google Chrome"
|
||||
);
|
||||
execute_path = path.join(__dirname, "");
|
||||
// Keep the macOS executor inside the signed application bundle. An app
|
||||
// opened through Gatekeeper's "Allow Anyway" flow can be App Translocated;
|
||||
// sibling files next to the .app then resolve to a temporary private path.
|
||||
// The packaged macOS build copies this binary into Resources/app.
|
||||
execute_path = path.join(__dirname, "easyspider_executestage");
|
||||
} else if (process.platform === "linux") {
|
||||
driverPath = path.join(__dirname, "chrome_linux64/chromedriver_linux64");
|
||||
chromeBinaryPath = path.join(__dirname, "chrome_linux64/chrome");
|
||||
@@ -1093,20 +1155,49 @@ async function beginInvoke(msg, ws) {
|
||||
// });
|
||||
|
||||
let spawn = require("child_process").spawn;
|
||||
if (
|
||||
process.platform != "darwin" &&
|
||||
msg.message.execute_type == 1 &&
|
||||
msg.message.id != -1
|
||||
) {
|
||||
let child_process = spawn(execute_path, parameters);
|
||||
child_process.stdout.on("data", function (data) {
|
||||
console.log(data.toString());
|
||||
});
|
||||
let execution_started = false;
|
||||
if (msg.message.execute_type == 1 && msg.message.id != -1) {
|
||||
if (process.platform === "darwin" && !fs.existsSync(execute_path)) {
|
||||
console.error("macOS execution stage is missing from the app bundle:", execute_path);
|
||||
} else {
|
||||
const child_process = spawn(execute_path, parameters, {
|
||||
// All mutable task state belongs in Electron's stable
|
||||
// userData directory, never in a translocated app path.
|
||||
cwd: task_server.getDir(),
|
||||
env: {
|
||||
...process.env,
|
||||
PYTHONUNBUFFERED: "1",
|
||||
PYTHONUTF8: "1",
|
||||
EASYSPIDER_DATA_DIR: task_server.getDir(),
|
||||
EASYSPIDER_APP_RESOURCES: __dirname,
|
||||
},
|
||||
});
|
||||
execution_started = true;
|
||||
child_process.on("error", (error) => {
|
||||
console.error("Failed to start execution stage:", error);
|
||||
});
|
||||
child_process.stdout.on("data", function (data) {
|
||||
console.log(data.toString());
|
||||
});
|
||||
}
|
||||
}
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
config_folder: task_server.getDir() + "/",
|
||||
easyspider_location: task_server.getEasySpiderLocation(),
|
||||
// Never expose an App Translocation directory as a runnable
|
||||
// command-line location. macOS uses a validated portable
|
||||
// release folder instead.
|
||||
easyspider_location:
|
||||
process.platform === "darwin"
|
||||
? detectMacOSCommandFolder()
|
||||
: task_server.getEasySpiderLocation(),
|
||||
macos_command_folder:
|
||||
process.platform === "darwin" ? detectMacOSCommandFolder() : "",
|
||||
macos_command_launcher:
|
||||
process.platform === "darwin"
|
||||
? getMacOSCommandLauncher(detectMacOSCommandFolder())
|
||||
: "",
|
||||
execution_started,
|
||||
})
|
||||
);
|
||||
} else if (msg.type == 6) {
|
||||
@@ -1134,6 +1225,59 @@ async function beginInvoke(msg, ws) {
|
||||
} catch {
|
||||
console.log("Cannot get Cookies");
|
||||
}
|
||||
} else if (msg.type == 8) {
|
||||
if (process.platform !== "darwin") {
|
||||
ws.send(JSON.stringify({
|
||||
response_type: "macos_command_folder",
|
||||
canceled: true,
|
||||
error: "The portable execution folder is only used on macOS.",
|
||||
}));
|
||||
return;
|
||||
}
|
||||
const currentFolder = detectMacOSCommandFolder();
|
||||
const parentWindow =
|
||||
invoke_window && !invoke_window.isDestroyed()
|
||||
? invoke_window
|
||||
: mainWindow;
|
||||
const selection = await dialog.showOpenDialog(parentWindow, {
|
||||
title:
|
||||
language === "zh"
|
||||
? "选择 EasySpider_MacOS 发布文件夹"
|
||||
: "Select the EasySpider_MacOS release folder",
|
||||
defaultPath: currentFolder || app.getPath("downloads"),
|
||||
properties: ["openDirectory"],
|
||||
message:
|
||||
language === "zh"
|
||||
? "请选择同时包含 EasySpider.app 和 easyspider_executestage 的文件夹"
|
||||
: "Select the folder containing EasySpider.app and easyspider_executestage",
|
||||
});
|
||||
if (selection.canceled || selection.filePaths.length === 0) {
|
||||
ws.send(JSON.stringify({
|
||||
response_type: "macos_command_folder",
|
||||
canceled: true,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
const selectedFolder = path.resolve(selection.filePaths[0]);
|
||||
const launcher = getMacOSCommandLauncher(selectedFolder);
|
||||
if (!launcher) {
|
||||
ws.send(JSON.stringify({
|
||||
response_type: "macos_command_folder",
|
||||
canceled: false,
|
||||
error:
|
||||
language === "zh"
|
||||
? "所选文件夹必须同时包含 EasySpider.app 和 easyspider_executestage。"
|
||||
: "The selected folder must contain both EasySpider.app and easyspider_executestage.",
|
||||
}));
|
||||
return;
|
||||
}
|
||||
saveMacOSCommandFolder(selectedFolder);
|
||||
ws.send(JSON.stringify({
|
||||
response_type: "macos_command_folder",
|
||||
canceled: false,
|
||||
macos_command_folder: selectedFolder,
|
||||
macos_command_launcher: launcher,
|
||||
}));
|
||||
} else if (msg.type == 30) {
|
||||
send_message_to_browser(
|
||||
JSON.stringify({
|
||||
|
||||
@@ -50,6 +50,7 @@ rm -rf "$STAGE_DIR/Code" "$STAGE_DIR/Sample Tasks" "$STAGE_DIR/Data" \
|
||||
rm -f "$STAGE_DIR/config.json" "$STAGE_DIR/mysql_config.json"
|
||||
mkdir -p "$STAGE_DIR/Code" "$STAGE_DIR/Sample Tasks" "$STAGE_DIR/Data" \
|
||||
"$STAGE_DIR/execution_instances"
|
||||
cp "$SCRIPT_DIR/execute_macos.sh" "$STAGE_DIR/execute_macos.sh"
|
||||
cp "$REPO_ROOT/ExecuteStage"/*.py "$STAGE_DIR/Code/"
|
||||
cp "$REPO_ROOT/ExecuteStage/requirements.txt" "$STAGE_DIR/Code/"
|
||||
cp "$REPO_ROOT/ExecuteStage/Readme.md" "$STAGE_DIR/Code/"
|
||||
@@ -69,6 +70,20 @@ find "$STAGE_DIR" -type d -name .pytest_cache -prune -exec rm -rf {} +
|
||||
chmod 755 "$STAGE_DIR/first_time_run.sh" "$STAGE_DIR/execute_macos.sh" 2>/dev/null || true
|
||||
chmod 755 "$STAGE_DIR/easyspider_executestage" "$STAGE_DIR/easyspider_executestage_full" 2>/dev/null || true
|
||||
|
||||
# Keep copies inside the signed app as well as in the portable release folder.
|
||||
# Gatekeeper may App-Translocate the .app when a user approves it in Privacy &
|
||||
# Security; an external sibling executable would then no longer be reachable
|
||||
# at the path shown by the translocated process. Electron launches these
|
||||
# bundled copies with the stable userData directory as their working directory.
|
||||
for executor in easyspider_executestage easyspider_executestage_full; do
|
||||
if [[ ! -f "$STAGE_DIR/$executor" ]]; then
|
||||
echo "Missing macOS execution stage: $STAGE_DIR/$executor" >&2
|
||||
exit 1
|
||||
fi
|
||||
cp "$STAGE_DIR/$executor" "$APP_RESOURCES/$executor"
|
||||
chmod 755 "$APP_RESOURCES/$executor"
|
||||
done
|
||||
|
||||
# Sign only after all files have been copied/deleted. Set CODESIGN_IDENTITY to
|
||||
# a Developer ID identity for a distributable build; the default '-' is ad hoc.
|
||||
CODESIGN_IDENTITY="${CODESIGN_IDENTITY:--}"
|
||||
@@ -82,6 +97,8 @@ sign_path() {
|
||||
}
|
||||
sign_path "$STAGE_DIR/easyspider_executestage"
|
||||
sign_path "$STAGE_DIR/easyspider_executestage_full"
|
||||
sign_path "$APP_RESOURCES/easyspider_executestage"
|
||||
sign_path "$APP_RESOURCES/easyspider_executestage_full"
|
||||
sign_path "$APP_PATH"
|
||||
codesign --verify --deep --strict "$APP_PATH"
|
||||
echo "macOS $HOST_ARCH application staged at $STAGE_DIR"
|
||||
|
||||
+14
-17
@@ -33,26 +33,23 @@ function compare(p) {
|
||||
}
|
||||
|
||||
function getDir() {
|
||||
if (__dirname.indexOf("app") >= 0 && __dirname.indexOf("sources") >= 0) {
|
||||
if (process.platform == "darwin") {
|
||||
return app.getPath("userData");
|
||||
} else {
|
||||
return path.join(__dirname, "../../..");
|
||||
}
|
||||
} else {
|
||||
return __dirname;
|
||||
}
|
||||
// On macOS, never use the packaged app's __dirname for mutable data.
|
||||
// Gatekeeper may launch the app through App Translocation, where that path
|
||||
// points into a temporary private directory. Electron's userData path is
|
||||
// stable across normal launches and security approval flows. Preserve the
|
||||
// historical portable-root behaviour on Windows and Linux.
|
||||
if (app.isPackaged && process.platform === "darwin") return app.getPath("userData");
|
||||
if (app.isPackaged) return path.resolve(process.resourcesPath, "../..");
|
||||
return __dirname;
|
||||
}
|
||||
function getEasySpiderLocation() {
|
||||
if (__dirname.indexOf("app") >= 0 && __dirname.indexOf("sources") >= 0) {
|
||||
if (process.platform == "darwin") {
|
||||
return path.join(__dirname, "../../../");
|
||||
} else {
|
||||
return path.join(__dirname, "../../../");
|
||||
}
|
||||
} else {
|
||||
return __dirname;
|
||||
if (app.isPackaged) {
|
||||
// This is retained for the Windows/Linux resource lookup and diagnostics.
|
||||
// macOS execution no longer depends on this path: the executor is bundled
|
||||
// under Resources/app and is started directly by Electron.
|
||||
return path.resolve(process.resourcesPath, "../..");
|
||||
}
|
||||
return __dirname;
|
||||
}
|
||||
if (!fs.existsSync(path.join(getDir(), "tasks"))) {
|
||||
fs.mkdirSync(path.join(getDir(), "tasks"));
|
||||
|
||||
@@ -92,8 +92,17 @@
|
||||
<label><a href="https://github.com/NaiboWang/EasySpider/wiki/Argument-Instruction"
|
||||
target="_blank">{{`Click Here~点击这里` | lang}}</a> {{`Here to see argument instruction.~这里查看参数配置说明。` | lang}}</label>
|
||||
<label v-if="OS=='MacOS'">{{`对于MacOS系统,EasySpider提供了两个不同的执行程序,分别为easyspider_executestage和easyspider_executestage_full,前者执行时加载速度较快,并提供了除OCR识别和数据去重以外的全部功能;后者则提供了包括OCR识别和数据去重在内的全部功能,但运行时加载速度较慢,需要等待2-10分钟才能执行程序,请根据自己的需求选择执行哪个程序。~For MacOS system, EasySpider provides two different execution programs, 'easyspider_executestage' and 'easyspider_executestage_full', the former loads faster when executing, and provides all functions except OCR recognition and data deduplication; the latter provides all functions including OCR recognition and data deduplication, but the loading speed is slower when running, and it takes 2-10 minutes to wait for the program to execute, please choose which program to execute according to your needs.` | lang}}</label>
|
||||
<label>{{ `Please open a terminal (For Windows, please use PowerShell instead of CMD), go to EasySpider's folder, and then copy (Command/Ctrl + c) the following command to run the task (EasySpider can quit when executing command for ease of timed execution, and you can set --read_type to "remote" for remote execution):~请在EasySpider目录下打开命令行工具Terminal (Windows请使用PowerShell而不是CMD),然后复制(Command/Ctrl + c)和运行以下命令以执行任务(执行命令时可以退出EasySpider以方便定时执行,如需要远程调用则需要将--read_type设置为remote并设置远程地址):` | lang }}</label>
|
||||
<textarea class="form-control" style="height:150px">cd {{easyspider_location}}
|
||||
<label v-if="OS=='MacOS'">{{ `Copy the following command into Terminal. The command uses the real EasySpider_MacOS release folder selected below and does not depend on an App Translocation path. You may quit EasySpider after the execution instance has been generated.~请将以下命令复制到Terminal运行。该命令使用下方选择的真实EasySpider_MacOS发布文件夹,不依赖App Translocation临时路径;生成执行实例后可以退出EasySpider。` | lang }}</label>
|
||||
<label v-else>{{ `Please open a terminal (For Windows, please use PowerShell instead of CMD), go to EasySpider's folder, and then copy (Command/Ctrl + c) the following command to run the task (EasySpider can quit when executing command for ease of timed execution, and you can set --read_type to "remote" for remote execution):~请在EasySpider目录下打开命令行工具Terminal (Windows请使用PowerShell而不是CMD),然后复制(Command/Ctrl + c)和运行以下命令以执行任务(执行命令时可以退出EasySpider以方便定时执行,如需要远程调用则需要将--read_type设置为remote并设置远程地址):` | lang }}</label>
|
||||
<div v-if="OS=='MacOS'" style="margin: 8px 0 10px 0;">
|
||||
<button type="button" class="btn btn-outline-primary btn-sm"
|
||||
v-on:click="selectMacOSCommandFolder">{{ `Select/change EasySpider_MacOS folder~选择或更改EasySpider_MacOS文件夹` | lang }}</button>
|
||||
<div style="margin-top: 6px; word-break: break-all;">
|
||||
{{macos_command_folder}}
|
||||
</div>
|
||||
</div>
|
||||
<textarea v-if="OS=='MacOS'" class="form-control" style="height:170px" readonly>{{macos_command_line}}</textarea>
|
||||
<textarea v-else class="form-control" style="height:150px">cd {{easyspider_location}}
|
||||
{{command}} --config_folder "{{config_folder}}" --headless 0 --read_type local --config_file_name config.json --saved_file_name </textarea>
|
||||
</div>
|
||||
<!-- <div class="modal-footer">
|
||||
@@ -348,6 +357,11 @@
|
||||
config_folder: "",
|
||||
easyspider_location: "",
|
||||
mysql_config_path: "",
|
||||
executionStarted: false,
|
||||
macos_command_folder: "",
|
||||
macos_command_launcher: "",
|
||||
macos_command_line: "",
|
||||
showCommandAfterFolderSelection: false,
|
||||
OS: "Windows",
|
||||
}, mounted() {
|
||||
$.get(this.backEndAddressServiceWrapper + "/getConfig", function (result) {
|
||||
@@ -397,6 +411,16 @@
|
||||
},
|
||||
readFromExcel: function () {
|
||||
$('#excelModal').modal('show');
|
||||
}, selectMacOSCommandFolder: function () {
|
||||
this.showCommandAfterFolderSelection = true;
|
||||
ws.send(JSON.stringify({type: 8}));
|
||||
}, showMacOSCommandDialog: function () {
|
||||
changeCommand();
|
||||
if (this.macos_command_launcher !== "") {
|
||||
$('#myModal').modal('show');
|
||||
} else {
|
||||
this.selectMacOSCommandFolder();
|
||||
}
|
||||
}, submitFile: function () {
|
||||
let form_data = new FormData();
|
||||
this.file = $('#excelFile').prop('files')[0];
|
||||
@@ -521,8 +545,12 @@
|
||||
}
|
||||
};
|
||||
ws.send(JSON.stringify(message));
|
||||
changeCommand();
|
||||
$('#myModal').modal('show');
|
||||
if (detectOperatingSystemAndArch().OS == 'MacOS') {
|
||||
this.showMacOSCommandDialog();
|
||||
} else {
|
||||
changeCommand();
|
||||
$('#myModal').modal('show');
|
||||
}
|
||||
// }
|
||||
},
|
||||
remoteExecute: function () {
|
||||
@@ -558,13 +586,22 @@
|
||||
}
|
||||
};
|
||||
app.$data.ID = result;
|
||||
app.$data.executionStarted = false;
|
||||
ws.send(JSON.stringify(message));
|
||||
// 使用函数并打印结果
|
||||
const systemInfo = detectOperatingSystemAndArch();
|
||||
// $.get(app.$data.backEndAddressServiceWrapper + "/queryOSVersion", function (OSInfo) {
|
||||
if (systemInfo.OS == 'MacOS') {
|
||||
changeCommand();
|
||||
$('#myModal').modal('show');
|
||||
// The packaged macOS executor is now launched inside the
|
||||
// signed app bundle. Wait briefly for the websocket
|
||||
// acknowledgement before falling back to the manual
|
||||
// command dialog used by older packages.
|
||||
setTimeout(function () {
|
||||
if (!app.$data.executionStarted) {
|
||||
app.showMacOSCommandDialog();
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
// });
|
||||
});
|
||||
@@ -587,16 +624,29 @@
|
||||
} else if (systemInfo.OS == 'Linux') {
|
||||
app.$data.command = "./EasySpider/resources/app/chrome_linux64/easyspider_executestage --ids '[" + app.$data.ID.toString() + "]' --user_data " + (app.$data.with_user_data ? "1" : "0") + " --server_address " + app.$data.backEndAddressServiceWrapper;
|
||||
} else if (systemInfo.OS == 'MacOS') {
|
||||
if (getUrlParam("lang") == "zh") {
|
||||
app.$data.easyspider_location = "你的EasySpider文件夹,如:cd /Users/" + app.$data.config_folder.split("/")[2] + "/Downloads/EasySpider_MacOS";
|
||||
if (app.$data.macos_command_launcher !== "") {
|
||||
app.$data.macos_command_line =
|
||||
"cd " + quotePOSIXShell(app.$data.macos_command_folder) + "\n" +
|
||||
quotePOSIXShell(app.$data.macos_command_launcher) +
|
||||
" --ids " + quotePOSIXShell("[" + app.$data.ID.toString() + "]") +
|
||||
" --user_data " + (app.$data.with_user_data ? "1" : "0") +
|
||||
" --server_address " + quotePOSIXShell(app.$data.backEndAddressServiceWrapper) +
|
||||
" --config_folder " + quotePOSIXShell(app.$data.config_folder) +
|
||||
" --headless 0 --read_type local" +
|
||||
" --config_file_name config.json --saved_file_name ''";
|
||||
} else {
|
||||
app.$data.easyspider_location = "Your EasySpider folder, such as: cd /Users/" + app.$data.config_folder.split("/")[2] + "/Downloads/EasySpider_MacOS";
|
||||
app.$data.macos_command_line = getUrlParam("lang") == "zh"
|
||||
? "请先选择包含 EasySpider.app 和 easyspider_executestage 的 EasySpider_MacOS 文件夹。"
|
||||
: "Select the EasySpider_MacOS folder containing EasySpider.app and easyspider_executestage first.";
|
||||
}
|
||||
app.$data.command = "./easyspider_executestage --ids '[" + app.$data.ID.toString() + "]' --user_data " + (app.$data.with_user_data ? "1" : "0") + " --server_address " + app.$data.backEndAddressServiceWrapper;
|
||||
}
|
||||
// });
|
||||
}
|
||||
|
||||
function quotePOSIXShell(value) {
|
||||
return "'" + String(value).replaceAll("'", "'\"'\"'") + "'";
|
||||
}
|
||||
|
||||
$.get(app.$data.backEndAddressServiceWrapper + "/queryTask?id=" + sId, function (result) {
|
||||
app.$data.task = result;
|
||||
app.$data.show = true;
|
||||
@@ -626,8 +676,39 @@
|
||||
};
|
||||
ws.onmessage = function (message) {
|
||||
message = JSON.parse(message.data);
|
||||
app.$data.config_folder = message.config_folder.replaceAll("\\", "/");
|
||||
app.$data.easyspider_location = message.easyspider_location.replace("/EasySpider.app/", "");
|
||||
if (message.response_type === "macos_command_folder") {
|
||||
if (message.error) {
|
||||
alert(message.error);
|
||||
app.$data.showCommandAfterFolderSelection = false;
|
||||
return;
|
||||
}
|
||||
if (message.canceled) {
|
||||
app.$data.showCommandAfterFolderSelection = false;
|
||||
return;
|
||||
}
|
||||
app.$data.macos_command_folder = message.macos_command_folder;
|
||||
app.$data.macos_command_launcher = message.macos_command_launcher;
|
||||
changeCommand();
|
||||
if (app.$data.showCommandAfterFolderSelection) {
|
||||
app.$data.showCommandAfterFolderSelection = false;
|
||||
$('#myModal').modal('show');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message.config_folder) {
|
||||
app.$data.config_folder = message.config_folder.replaceAll("\\", "/");
|
||||
}
|
||||
if (message.macos_command_folder) {
|
||||
app.$data.macos_command_folder = message.macos_command_folder;
|
||||
}
|
||||
if (message.macos_command_launcher) {
|
||||
app.$data.macos_command_launcher = message.macos_command_launcher;
|
||||
}
|
||||
if (message.easyspider_location && detectOperatingSystemAndArch().OS != 'MacOS') {
|
||||
app.$data.easyspider_location = message.easyspider_location.replace("/EasySpider.app/", "");
|
||||
}
|
||||
changeCommand();
|
||||
app.$data.executionStarted = message.execution_started === true;
|
||||
}
|
||||
ws.onclose = function () {
|
||||
// 关闭 websocket
|
||||
|
||||
@@ -40,6 +40,7 @@ import re
|
||||
# import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
# from urllib import parse
|
||||
# import base64
|
||||
# import hashlib
|
||||
@@ -73,6 +74,38 @@ desired_capabilities = DesiredCapabilities.CHROME
|
||||
desired_capabilities["pageLoadStrategy"] = "none"
|
||||
|
||||
|
||||
def _runtime_directory():
|
||||
"""Return the directory containing the running executor binary/script."""
|
||||
if getattr(sys, "frozen", False):
|
||||
return Path(sys.executable).resolve().parent
|
||||
return Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def _data_directory():
|
||||
"""Resolve mutable task data independently from the app's cwd.
|
||||
|
||||
Electron supplies EASYSPIDER_DATA_DIR when it starts the bundled macOS
|
||||
executor. A manually launched portable executable keeps the historical
|
||||
behaviour of writing beside the executable; a frozen macOS executable
|
||||
launched directly falls back to Electron's userData location.
|
||||
"""
|
||||
configured = os.environ.get("EASYSPIDER_DATA_DIR", "").strip()
|
||||
if configured:
|
||||
return Path(configured).expanduser().resolve()
|
||||
if sys.platform == "darwin" and getattr(sys, "frozen", False):
|
||||
return Path(os.path.expanduser("~/Library/Application Support/EasySpider")).resolve()
|
||||
return Path.cwd().resolve()
|
||||
|
||||
|
||||
RUNTIME_DIRECTORY = _runtime_directory()
|
||||
DATA_DIRECTORY = _data_directory()
|
||||
DATA_DIRECTORY.mkdir(parents=True, exist_ok=True)
|
||||
# The executor historically uses relative Data/, execution_instances/ and
|
||||
# TempUserDataFolder paths throughout the file. Normalize cwd once so those
|
||||
# paths remain valid even when macOS launches the app through App Translocation.
|
||||
os.chdir(DATA_DIRECTORY)
|
||||
|
||||
|
||||
class BrowserThread(Thread):
|
||||
def __init__(self, browser_t, id, service, version, event, saveName, config, option, shutdown_event, commandline_config=""):
|
||||
Thread.__init__(self)
|
||||
@@ -133,8 +166,7 @@ class BrowserThread(Thread):
|
||||
"将从头开始执行,如果需要从上次退出的步骤开始执行,请在保存任务时设置是否从上次保存位置开始执行为“是”。")
|
||||
self.print_and_log("In this mode, task ID", self.id,
|
||||
"will start from the beginning, if you want to start from the last step, please set the option 'start from the last step' to 'yes' when saving the task.")
|
||||
stealth_path = driver_path[:driver_path.find(
|
||||
"chromedriver")] + "stealth.min.js"
|
||||
stealth_path = os.path.join(os.path.dirname(driver_path), "stealth.min.js")
|
||||
with open(stealth_path, 'r') as f:
|
||||
js = f.read()
|
||||
self.print_and_log("Loading stealth.min.js")
|
||||
@@ -473,7 +505,8 @@ class BrowserThread(Thread):
|
||||
self.print_and_log("接收到终止信号,正在中断任务... | Received termination signal, interrupting task...")
|
||||
break
|
||||
self.event.wait() # 暂停/恢复
|
||||
self.executeNode(self.startSteps, self.links[i], "", i)
|
||||
self.urlId = i
|
||||
self.executeNode(0)
|
||||
# files = os.listdir("Data/Task_" + str(self.id) + "/" + self.saveName)
|
||||
# 如果目录为空,则删除该目录
|
||||
# if not files:
|
||||
@@ -2325,8 +2358,31 @@ if __name__ == '__main__':
|
||||
if not os.path.exists(os.getcwd() + "/Data"):
|
||||
os.mkdir(os.getcwd() + "/Data")
|
||||
if sys.platform == "darwin" and platform.architecture()[0] == "64bit":
|
||||
options.binary_location = "EasySpider.app/Contents/Resources/app/chrome_mac64.app/Contents/MacOS/Google Chrome"
|
||||
driver_path = "EasySpider.app/Contents/Resources/app/chromedriver_mac64"
|
||||
resource_override = os.environ.get("EASYSPIDER_APP_RESOURCES", "").strip()
|
||||
resource_candidates = []
|
||||
if resource_override:
|
||||
resource_candidates.append(Path(resource_override).expanduser().resolve())
|
||||
# Bundled executor: RUNTIME_DIRECTORY is Resources/app.
|
||||
resource_candidates.append(RUNTIME_DIRECTORY)
|
||||
# Portable release fallback: executor is next to EasySpider.app.
|
||||
resource_candidates.append(RUNTIME_DIRECTORY / "EasySpider.app" / "Contents" / "Resources" / "app")
|
||||
# Development fallback when running the Python entry point directly.
|
||||
resource_candidates.append(RUNTIME_DIRECTORY.parent / "ElectronJS")
|
||||
resource_root = next(
|
||||
(candidate for candidate in resource_candidates
|
||||
if (candidate / "chromedriver_mac64").exists() and
|
||||
(candidate / "chrome_mac64.app").exists()),
|
||||
None,
|
||||
)
|
||||
if resource_root is None:
|
||||
raise FileNotFoundError(
|
||||
"Cannot locate the bundled macOS Chrome resources. "
|
||||
"Checked: " + ", ".join(str(candidate) for candidate in resource_candidates)
|
||||
)
|
||||
options.binary_location = str(
|
||||
resource_root / "chrome_mac64.app" / "Contents" / "MacOS" / "Google Chrome"
|
||||
)
|
||||
driver_path = str(resource_root / "chromedriver_mac64")
|
||||
print(driver_path)
|
||||
if c.config_folder == "":
|
||||
c.config_folder = os.path.expanduser(
|
||||
|
||||
Reference in New Issue
Block a user