Compare commits
6 Commits
WindowsAPP
...
v2.1.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
37a6e2ad09 | ||
|
|
9f548309f8 | ||
|
|
251dd77e49 | ||
|
|
c6a69c6ada | ||
|
|
4b65cbbe84 | ||
|
|
610a20d12d |
@@ -5,7 +5,18 @@
|
|||||||
"Bash(python:*)",
|
"Bash(python:*)",
|
||||||
"Bash(build_exe.bat)",
|
"Bash(build_exe.bat)",
|
||||||
"Bash(cmd /c \"cd scripts && build_exe.bat\")",
|
"Bash(cmd /c \"cd scripts && build_exe.bat\")",
|
||||||
"Bash(dir:*)"
|
"Bash(dir:*)",
|
||||||
|
"Bash(python3 -c \"import PyQt5\")",
|
||||||
|
"Bash(python3 -c \"import selenium; print\\('selenium', selenium.__version__\\)\")",
|
||||||
|
"Bash(python3 -m pip --version)",
|
||||||
|
"Bash(curl -s https://pypi.org/pypi/PyQt5/json)",
|
||||||
|
"Bash(python3 -c ' *)",
|
||||||
|
"Bash(python3 -m venv .venv)",
|
||||||
|
"Bash(.venv/bin/pip install *)",
|
||||||
|
"Bash(.venv/bin/python -c ' *)",
|
||||||
|
"Bash(.venv/bin/python gui_main.py)",
|
||||||
|
"Skill(claude-in-chrome)",
|
||||||
|
"Bash(pkill -f \"gui_main.py\")"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
37
.gitignore
vendored
Normal file
37
.gitignore
vendored
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.so
|
||||||
|
.Python
|
||||||
|
*.egg-info/
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
*.egg
|
||||||
|
.pytest_cache/
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Virtual Environment
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
ENV/
|
||||||
|
|
||||||
|
# IDE
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Credentials
|
||||||
|
credentials.json
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Claude
|
||||||
|
.claude/settings.local.json
|
||||||
@@ -6,7 +6,7 @@ a = Analysis(
|
|||||||
pathex=[],
|
pathex=[],
|
||||||
binaries=[],
|
binaries=[],
|
||||||
datas=[],
|
datas=[],
|
||||||
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'],
|
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'],
|
||||||
hookspath=[],
|
hookspath=[],
|
||||||
hooksconfig={},
|
hooksconfig={},
|
||||||
runtime_hooks=[],
|
runtime_hooks=[],
|
||||||
|
|||||||
129
build.py
Normal file
129
build.py
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Simple build wrapper for EBoek.info Scraper
|
||||||
|
Creates a standalone executable using PyInstaller
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Main build process."""
|
||||||
|
print("=" * 50)
|
||||||
|
print("EBoek.info Scraper - Executable Builder")
|
||||||
|
print("=" * 50)
|
||||||
|
print()
|
||||||
|
|
||||||
|
project_root = Path(__file__).parent
|
||||||
|
os.chdir(project_root)
|
||||||
|
print(f"Working from: {project_root}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Check if PyInstaller is installed
|
||||||
|
try:
|
||||||
|
import PyInstaller
|
||||||
|
print(f"PyInstaller {PyInstaller.__version__} found")
|
||||||
|
except ImportError:
|
||||||
|
print("Installing PyInstaller...")
|
||||||
|
subprocess.run([sys.executable, "-m", "pip", "install", "pyinstaller"], check=True)
|
||||||
|
|
||||||
|
# Clean previous builds
|
||||||
|
print()
|
||||||
|
for directory in ["dist", "build"]:
|
||||||
|
if Path(directory).exists():
|
||||||
|
shutil.rmtree(directory)
|
||||||
|
print(f"Cleaned {directory}/")
|
||||||
|
|
||||||
|
# Determine platform-specific settings
|
||||||
|
is_windows = sys.platform.startswith('win')
|
||||||
|
app_name = "EBoek_Scraper"
|
||||||
|
|
||||||
|
# Build command
|
||||||
|
build_cmd = [
|
||||||
|
sys.executable, "-m", "PyInstaller",
|
||||||
|
"--onefile",
|
||||||
|
"--windowed",
|
||||||
|
"--name", app_name,
|
||||||
|
# Hidden imports for PyQt5
|
||||||
|
"--hidden-import", "PyQt5.QtCore",
|
||||||
|
"--hidden-import", "PyQt5.QtGui",
|
||||||
|
"--hidden-import", "PyQt5.QtWidgets",
|
||||||
|
# Hidden imports for Selenium
|
||||||
|
"--hidden-import", "selenium",
|
||||||
|
"--hidden-import", "selenium.webdriver",
|
||||||
|
"--hidden-import", "selenium.webdriver.chrome",
|
||||||
|
"--hidden-import", "selenium.webdriver.chrome.options",
|
||||||
|
"--hidden-import", "selenium.webdriver.common.by",
|
||||||
|
# Hidden imports for our modules
|
||||||
|
"--hidden-import", "core.scraper",
|
||||||
|
"--hidden-import", "core.scraper_thread",
|
||||||
|
"--hidden-import", "core.credentials",
|
||||||
|
"--hidden-import", "gui.main_window",
|
||||||
|
"--hidden-import", "gui.login_dialog",
|
||||||
|
"--hidden-import", "gui.progress_dialog",
|
||||||
|
"--hidden-import", "utils.validators",
|
||||||
|
# Bookys scraper modules
|
||||||
|
"--hidden-import", "core.bookys_scraper",
|
||||||
|
"--hidden-import", "core.bookys_thread",
|
||||||
|
"--hidden-import", "gui.bookys_window",
|
||||||
|
"--hidden-import", "gui.startup_dialog",
|
||||||
|
# Exclude unnecessary modules
|
||||||
|
"--exclude-module", "tkinter",
|
||||||
|
"--exclude-module", "matplotlib",
|
||||||
|
"--exclude-module", "numpy",
|
||||||
|
"--exclude-module", "pandas",
|
||||||
|
# Main script
|
||||||
|
"gui_main.py"
|
||||||
|
]
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Building executable...")
|
||||||
|
print("This may take a few minutes...")
|
||||||
|
print()
|
||||||
|
|
||||||
|
# Run PyInstaller
|
||||||
|
result = subprocess.run(build_cmd)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
print("Build failed!")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print("Build completed successfully!")
|
||||||
|
|
||||||
|
# Check if executable was created
|
||||||
|
exe_name = "EBoek_Scraper.exe" if is_windows else "EBoek_Scraper"
|
||||||
|
exe_path = Path("dist") / exe_name
|
||||||
|
|
||||||
|
if not exe_path.exists():
|
||||||
|
print(f"Error: Executable not found at {exe_path}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Display results
|
||||||
|
file_size = exe_path.stat().st_size / (1024 * 1024) # MB
|
||||||
|
print()
|
||||||
|
print("=" * 50)
|
||||||
|
print("BUILD SUCCESSFUL!")
|
||||||
|
print("=" * 50)
|
||||||
|
print(f"Executable location: {exe_path}")
|
||||||
|
print(f"File size: {file_size:.1f} MB")
|
||||||
|
print()
|
||||||
|
|
||||||
|
if is_windows:
|
||||||
|
print("Windows Instructions:")
|
||||||
|
print(f" Run: {exe_path}")
|
||||||
|
print(" Windows may show security warning on first run")
|
||||||
|
print(" Click 'More Info' -> 'Run Anyway' if prompted")
|
||||||
|
else:
|
||||||
|
print("macOS/Linux Instructions:")
|
||||||
|
print(f" Run: ./{exe_path}")
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Ready for distribution!")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Binary file not shown.
BIN
core/__pycache__/bookys_scraper.cpython-314.pyc
Normal file
BIN
core/__pycache__/bookys_scraper.cpython-314.pyc
Normal file
Binary file not shown.
BIN
core/__pycache__/bookys_thread.cpython-314.pyc
Normal file
BIN
core/__pycache__/bookys_thread.cpython-314.pyc
Normal file
Binary file not shown.
BIN
core/__pycache__/credentials.cpython-314.pyc
Normal file
BIN
core/__pycache__/credentials.cpython-314.pyc
Normal file
Binary file not shown.
BIN
core/__pycache__/scraper.cpython-314.pyc
Normal file
BIN
core/__pycache__/scraper.cpython-314.pyc
Normal file
Binary file not shown.
BIN
core/__pycache__/scraper_thread.cpython-314.pyc
Normal file
BIN
core/__pycache__/scraper_thread.cpython-314.pyc
Normal file
Binary file not shown.
461
core/bookys_scraper.py
Normal file
461
core/bookys_scraper.py
Normal file
@@ -0,0 +1,461 @@
|
|||||||
|
"""
|
||||||
|
Bookys-ebooks.com scraper.
|
||||||
|
|
||||||
|
Unlike the EBoek scraper (which downloads files), this scraper harvests the
|
||||||
|
direct 1fichier.com download URLs for each item in a category and writes them,
|
||||||
|
one per line, to a plain text file.
|
||||||
|
|
||||||
|
Key differences from the EBoek scraper:
|
||||||
|
* Never runs headless - the site is behind Cloudflare and the user must pass
|
||||||
|
the "I am not a robot" challenge manually. The scraper auto-detects when the
|
||||||
|
challenge clears and continues on its own.
|
||||||
|
* Never clicks host/download links. The site spawns pop-up ads on click, so we
|
||||||
|
read the href and navigate to it directly (driver.get). Pop-ups are also
|
||||||
|
neutralised by overriding window.open on every document.
|
||||||
|
* Only follows 1fichier hosts (per requirements), ignoring other file hosts.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
from selenium.webdriver.chrome.options import Options
|
||||||
|
import time
|
||||||
|
import random
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import urllib3
|
||||||
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||||
|
|
||||||
|
|
||||||
|
# Category slug -> listing path on the site. All share the same page structure
|
||||||
|
# (20 items per page, ?page=N pagination, .bys-item links), so adding one is
|
||||||
|
# just a new entry here - the rest of the scraper and the UI adapt automatically.
|
||||||
|
CATEGORIES = {
|
||||||
|
"bd": {
|
||||||
|
"name": "Bandes dessinées (BD)",
|
||||||
|
"path": "bandes-dessinees/bd",
|
||||||
|
},
|
||||||
|
"comic": {
|
||||||
|
"name": "Comics",
|
||||||
|
"path": "bandes-dessinees/comic", # note: singular "comic" on the site
|
||||||
|
},
|
||||||
|
"manga": {
|
||||||
|
"name": "Manga",
|
||||||
|
"path": "bandes-dessinees/manga",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
BASE_DOMAIN = "https://www7.bookys-ebooks.com"
|
||||||
|
|
||||||
|
# Persistent Chrome profile for the Bookys scraper. Keeping this between runs is
|
||||||
|
# what stops Cloudflare from re-challenging on every launch.
|
||||||
|
DEFAULT_PROFILE_DIR = Path.home() / ".eboek_scraper" / "bookys_chrome_profile"
|
||||||
|
|
||||||
|
|
||||||
|
class BookysScraper:
|
||||||
|
"""Harvests 1fichier download URLs from bookys-ebooks.com category listings."""
|
||||||
|
|
||||||
|
# Selectors, kept together so a future site change is easy to patch.
|
||||||
|
ITEM_SELECTOR = ".bys-items-container a.bys-item"
|
||||||
|
HOST_LINK_SELECTOR = "a.bys-link.bys-host"
|
||||||
|
|
||||||
|
def __init__(self, progress_callback=None, category="bd", output_file=None,
|
||||||
|
timing_config=None, profile_dir=None):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
progress_callback (callable): callback(event_type: str, data: dict)
|
||||||
|
category (str): category slug (see CATEGORIES)
|
||||||
|
output_file (str|Path): path to the .txt file to write URLs into
|
||||||
|
timing_config (dict): optional timing overrides
|
||||||
|
profile_dir (str|Path): persistent Chrome profile directory. Keeping a
|
||||||
|
profile between runs preserves the Cloudflare clearance cookie, so
|
||||||
|
the challenge doesn't reappear on every launch.
|
||||||
|
"""
|
||||||
|
self.progress_callback = progress_callback
|
||||||
|
self._stop_requested = False
|
||||||
|
self.category = category if category in CATEGORIES else "bd"
|
||||||
|
self.output_file = Path(output_file) if output_file else (
|
||||||
|
Path.home() / "Downloads" / "bookys_1fichier_links.txt"
|
||||||
|
)
|
||||||
|
# Dedicated profile - deliberately NOT the user's everyday Chrome profile,
|
||||||
|
# which Chrome refuses to open while a normal Chrome window is running.
|
||||||
|
self.profile_dir = Path(profile_dir) if profile_dir else DEFAULT_PROFILE_DIR
|
||||||
|
|
||||||
|
self.timing = timing_config or {}
|
||||||
|
self._setup_timing_defaults()
|
||||||
|
|
||||||
|
# Collected URLs (also written to disk as we go) and a dedupe set.
|
||||||
|
self.collected_urls = []
|
||||||
|
self._seen_urls = set()
|
||||||
|
|
||||||
|
# Popup-neutralising script injected into every new document.
|
||||||
|
self._popup_kill_js = "window.open = function(){ return null; };"
|
||||||
|
|
||||||
|
chrome_options = Options()
|
||||||
|
# NOTE: never headless - Cloudflare needs a real, visible browser.
|
||||||
|
|
||||||
|
# Persistent profile: this is what keeps the Cloudflare clearance cookie
|
||||||
|
# alive between runs, so the challenge isn't shown on every single launch.
|
||||||
|
try:
|
||||||
|
self.profile_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
chrome_options.add_argument(f'--user-data-dir={self.profile_dir}')
|
||||||
|
chrome_options.add_argument('--profile-directory=Default')
|
||||||
|
|
||||||
|
chrome_options.add_argument('--ignore-ssl-errors')
|
||||||
|
chrome_options.add_argument('--ignore-certificate-errors')
|
||||||
|
chrome_options.add_argument('--disable-blink-features=AutomationControlled')
|
||||||
|
chrome_options.add_experimental_option("excludeSwitches",
|
||||||
|
["enable-automation", "enable-logging"])
|
||||||
|
chrome_options.add_experimental_option('useAutomationExtension', False)
|
||||||
|
chrome_options.add_argument('--log-level=3')
|
||||||
|
# No user-agent override on purpose: a spoofed UA that disagrees with the
|
||||||
|
# real Chrome build invalidates the clearance cookie and re-triggers the
|
||||||
|
# challenge. Letting Chrome send its genuine UA is both safer and stealthier.
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.driver = webdriver.Chrome(options=chrome_options)
|
||||||
|
except Exception as e:
|
||||||
|
# Most common cause: the profile is already locked by another run.
|
||||||
|
self._emit("scraper_init_failed", {
|
||||||
|
"error": str(e),
|
||||||
|
"profile_dir": str(self.profile_dir),
|
||||||
|
"hint": "If a previous scraper Chrome window is still open, close it "
|
||||||
|
"and try again (the profile can only be used by one at a time).",
|
||||||
|
})
|
||||||
|
raise
|
||||||
|
self.driver.execute_script(
|
||||||
|
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
|
||||||
|
)
|
||||||
|
# Kill window.open before any page script runs, on every navigation.
|
||||||
|
try:
|
||||||
|
self.driver.execute_cdp_cmd(
|
||||||
|
'Page.addScriptToEvaluateOnNewDocument',
|
||||||
|
{'source': self._popup_kill_js}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Non-Chromium drivers won't support CDP; per-page fallback still runs.
|
||||||
|
pass
|
||||||
|
|
||||||
|
self._emit("scraper_initialized", {"category": self.category,
|
||||||
|
"output_file": str(self.output_file)})
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ helpers
|
||||||
|
|
||||||
|
def _emit(self, event_type, data):
|
||||||
|
if self.progress_callback:
|
||||||
|
try:
|
||||||
|
self.progress_callback(event_type, data)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def request_stop(self):
|
||||||
|
"""Ask the scraper to stop gracefully at the next checkpoint."""
|
||||||
|
self._stop_requested = True
|
||||||
|
self._emit("stop_requested", {})
|
||||||
|
|
||||||
|
def _setup_timing_defaults(self):
|
||||||
|
defaults = {
|
||||||
|
'action_delay_min': 0.8,
|
||||||
|
'action_delay_max': 2.5,
|
||||||
|
'page_break_min': 3,
|
||||||
|
'page_break_max': 8,
|
||||||
|
}
|
||||||
|
for key, value in defaults.items():
|
||||||
|
self.timing.setdefault(key, value)
|
||||||
|
|
||||||
|
def _delay(self, min_sec=None, max_sec=None):
|
||||||
|
if self._stop_requested:
|
||||||
|
return
|
||||||
|
if min_sec is None:
|
||||||
|
min_sec = self.timing['action_delay_min']
|
||||||
|
if max_sec is None:
|
||||||
|
max_sec = self.timing['action_delay_max']
|
||||||
|
time.sleep(random.uniform(min_sec, max_sec))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _link_text(element):
|
||||||
|
"""
|
||||||
|
Read an element's text via textContent rather than Selenium's .text.
|
||||||
|
|
||||||
|
The host links on Bookys live in a collapsed container, so they report as
|
||||||
|
not-displayed and .text returns an empty string for all of them.
|
||||||
|
textContent reads the DOM directly and works regardless of visibility.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return (element.get_attribute("textContent") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def _kill_popups(self):
|
||||||
|
"""Override window.open on the current document and close any stray tabs."""
|
||||||
|
try:
|
||||||
|
self.driver.execute_script(self._popup_kill_js)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self._close_extra_tabs()
|
||||||
|
|
||||||
|
def _close_extra_tabs(self):
|
||||||
|
"""Close any tab that isn't the main one (defensive - ads can slip a tab in)."""
|
||||||
|
try:
|
||||||
|
handles = self.driver.window_handles
|
||||||
|
if len(handles) <= 1:
|
||||||
|
return
|
||||||
|
main = handles[0]
|
||||||
|
for handle in handles[1:]:
|
||||||
|
try:
|
||||||
|
self.driver.switch_to.window(handle)
|
||||||
|
self.driver.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.driver.switch_to.window(main)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _navigate(self, url):
|
||||||
|
"""Navigate directly to a URL and neutralise pop-ups afterwards."""
|
||||||
|
if self._stop_requested:
|
||||||
|
return False
|
||||||
|
self.driver.get(url)
|
||||||
|
self._kill_popups()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _listing_url(self, page_num):
|
||||||
|
path = CATEGORIES[self.category]["path"]
|
||||||
|
base = f"{BASE_DOMAIN}/{path}"
|
||||||
|
if page_num <= 1:
|
||||||
|
return base
|
||||||
|
return f"{base}?page={page_num}"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- cloudflare
|
||||||
|
|
||||||
|
def wait_for_content(self, timeout=600, poll_interval=2):
|
||||||
|
"""
|
||||||
|
Wait until the category listing renders real items, giving the user time
|
||||||
|
to clear the Cloudflare challenge. Auto-detects success - no button.
|
||||||
|
|
||||||
|
Returns True once items are found, False on timeout/stop.
|
||||||
|
"""
|
||||||
|
self._emit("cloudflare_check", {"message": "Waiting for Cloudflare / page to load..."})
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
announced = False
|
||||||
|
while time.time() < deadline:
|
||||||
|
if self._stop_requested:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
items = self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR)
|
||||||
|
if items:
|
||||||
|
self._emit("cloudflare_passed", {"item_count": len(items)})
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not announced:
|
||||||
|
# Only nudge the user once, when content isn't immediately present.
|
||||||
|
self._emit("cloudflare_waiting", {
|
||||||
|
"message": "If a Cloudflare check is shown, please tick the "
|
||||||
|
"checkbox in the browser. Scraping continues automatically."
|
||||||
|
})
|
||||||
|
announced = True
|
||||||
|
time.sleep(poll_interval)
|
||||||
|
|
||||||
|
self._emit("cloudflare_timeout", {"timeout": timeout})
|
||||||
|
return False
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------- write
|
||||||
|
|
||||||
|
def _record_url(self, url, context):
|
||||||
|
if not url or "1fichier" not in url:
|
||||||
|
return False
|
||||||
|
if url in self._seen_urls:
|
||||||
|
self._emit("link_duplicate", {"url": url, **context})
|
||||||
|
return False
|
||||||
|
self._seen_urls.add(url)
|
||||||
|
self.collected_urls.append(url)
|
||||||
|
try:
|
||||||
|
self.output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(self.output_file, "a", encoding="utf-8") as f:
|
||||||
|
f.write(url + "\n")
|
||||||
|
except Exception as e:
|
||||||
|
self._emit("write_error", {"error": str(e), "url": url})
|
||||||
|
self._emit("link_found", {"url": url, "total": len(self.collected_urls), **context})
|
||||||
|
return True
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- detail
|
||||||
|
|
||||||
|
def _process_book(self, book_url, page_num, book_index, total_books):
|
||||||
|
"""Open a book detail page and harvest its 1fichier link(s)."""
|
||||||
|
self._emit("book_started", {
|
||||||
|
"url": book_url, "page_number": page_num,
|
||||||
|
"book_index": book_index, "total_books": total_books,
|
||||||
|
})
|
||||||
|
if not self._navigate(book_url):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
self._delay()
|
||||||
|
title = ""
|
||||||
|
try:
|
||||||
|
title = self.driver.find_element(By.CSS_SELECTOR, "h1").text.strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Collect intermediate /dl/<id> hrefs for 1fichier hosts only.
|
||||||
|
# A page typically lists several hosts (1fichier, DailyUploads, Filefox...);
|
||||||
|
# everything except 1fichier is deliberately ignored.
|
||||||
|
dl_hrefs = []
|
||||||
|
hosts_seen = []
|
||||||
|
try:
|
||||||
|
host_links = self.driver.find_elements(By.CSS_SELECTOR, self.HOST_LINK_SELECTOR)
|
||||||
|
if not host_links:
|
||||||
|
# Fallback if the site changes its classes: any /dl/ link.
|
||||||
|
host_links = self.driver.find_elements(By.CSS_SELECTOR, "a[href*='/dl/']")
|
||||||
|
|
||||||
|
for link in host_links:
|
||||||
|
try:
|
||||||
|
text = self._link_text(link)
|
||||||
|
if text:
|
||||||
|
hosts_seen.append(text)
|
||||||
|
if "1fichier" in text.lower():
|
||||||
|
href = link.get_attribute("href")
|
||||||
|
if href:
|
||||||
|
dl_hrefs.append(href)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
self._emit("book_error", {"url": book_url, "error": str(e)})
|
||||||
|
return 0
|
||||||
|
|
||||||
|
self._emit("hosts_listed", {"url": book_url, "hosts": hosts_seen,
|
||||||
|
"fichier_count": len(dl_hrefs)})
|
||||||
|
|
||||||
|
found = 0
|
||||||
|
for dl_href in dl_hrefs:
|
||||||
|
if self._stop_requested:
|
||||||
|
break
|
||||||
|
# The /dl/ page carries the direct 1fichier URL as an anchor href.
|
||||||
|
if not self._navigate(dl_href):
|
||||||
|
continue
|
||||||
|
self._delay()
|
||||||
|
final_url = None
|
||||||
|
try:
|
||||||
|
# Primary: any anchor already pointing at 1fichier. This is the
|
||||||
|
# most direct signal and doesn't depend on the button's wording.
|
||||||
|
for a in self.driver.find_elements(By.CSS_SELECTOR, "a[href*='1fichier']"):
|
||||||
|
href = a.get_attribute("href")
|
||||||
|
if href:
|
||||||
|
final_url = href
|
||||||
|
break
|
||||||
|
|
||||||
|
# Fallback: locate the call-to-action by its text (textContent,
|
||||||
|
# not .text - the element may report as not displayed).
|
||||||
|
if not final_url:
|
||||||
|
for a in self.driver.find_elements(By.TAG_NAME, "a"):
|
||||||
|
try:
|
||||||
|
text = self._link_text(a).lower()
|
||||||
|
if "cliquez ici" in text or "chargement" in text:
|
||||||
|
href = a.get_attribute("href")
|
||||||
|
if href and "1fichier" in href:
|
||||||
|
final_url = href
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
self._emit("book_error", {"url": dl_href, "error": str(e)})
|
||||||
|
|
||||||
|
if final_url and self._record_url(final_url, {"title": title, "book_url": book_url}):
|
||||||
|
found += 1
|
||||||
|
|
||||||
|
self._emit("book_completed", {
|
||||||
|
"url": book_url, "title": title, "links_found": found,
|
||||||
|
"page_number": page_num, "book_index": book_index,
|
||||||
|
})
|
||||||
|
return found
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------- scrape
|
||||||
|
|
||||||
|
def scrape(self, start_page=1, end_page=1):
|
||||||
|
"""Walk pages start_page..end_page and harvest 1fichier URLs."""
|
||||||
|
if self._stop_requested:
|
||||||
|
return {"success": False, "reason": "Cancelled before starting"}
|
||||||
|
|
||||||
|
total_pages = end_page - start_page + 1
|
||||||
|
self._emit("scraping_started", {
|
||||||
|
"start_page": start_page, "end_page": end_page,
|
||||||
|
"total_pages": total_pages,
|
||||||
|
"category": CATEGORIES[self.category]["name"],
|
||||||
|
"output_file": str(self.output_file),
|
||||||
|
})
|
||||||
|
|
||||||
|
errors = []
|
||||||
|
pages_done = 0
|
||||||
|
|
||||||
|
for page_num in range(start_page, end_page + 1):
|
||||||
|
if self._stop_requested:
|
||||||
|
break
|
||||||
|
|
||||||
|
page_url = self._listing_url(page_num)
|
||||||
|
self._emit("page_started", {
|
||||||
|
"page_number": page_num,
|
||||||
|
"page_index": page_num - start_page + 1,
|
||||||
|
"total_pages": total_pages,
|
||||||
|
"url": page_url,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not self._navigate(page_url):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# First page load may show Cloudflare; wait it out (auto-detect).
|
||||||
|
if not self.wait_for_content():
|
||||||
|
errors.append(f"Timed out waiting for content on page {page_num}")
|
||||||
|
self._emit("page_error", {"page_number": page_num,
|
||||||
|
"error": "content did not load"})
|
||||||
|
break
|
||||||
|
|
||||||
|
try:
|
||||||
|
book_urls = [a.get_attribute("href") for a
|
||||||
|
in self.driver.find_elements(By.CSS_SELECTOR, self.ITEM_SELECTOR)]
|
||||||
|
book_urls = [u for u in book_urls if u]
|
||||||
|
except Exception as e:
|
||||||
|
errors.append(f"Page {page_num}: {e}")
|
||||||
|
self._emit("page_error", {"page_number": page_num, "error": str(e)})
|
||||||
|
continue
|
||||||
|
|
||||||
|
self._emit("page_items_found", {"page_number": page_num,
|
||||||
|
"item_count": len(book_urls)})
|
||||||
|
|
||||||
|
for i, book_url in enumerate(book_urls, 1):
|
||||||
|
if self._stop_requested:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
self._process_book(book_url, page_num, i, len(book_urls))
|
||||||
|
except Exception as e:
|
||||||
|
errors.append(f"Book {book_url}: {e}")
|
||||||
|
self._emit("book_error", {"url": book_url, "error": str(e)})
|
||||||
|
self._delay()
|
||||||
|
|
||||||
|
pages_done += 1
|
||||||
|
self._emit("page_completed", {"page_number": page_num,
|
||||||
|
"items_processed": len(book_urls)})
|
||||||
|
|
||||||
|
# Short break between pages.
|
||||||
|
if page_num < end_page and not self._stop_requested:
|
||||||
|
time.sleep(random.uniform(self.timing['page_break_min'],
|
||||||
|
self.timing['page_break_max']))
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"success": not self._stop_requested and not errors,
|
||||||
|
"cancelled": self._stop_requested,
|
||||||
|
"total_pages_processed": pages_done,
|
||||||
|
"total_links_found": len(self.collected_urls),
|
||||||
|
"output_file": str(self.output_file),
|
||||||
|
"errors": errors,
|
||||||
|
}
|
||||||
|
self._emit("scraping_completed", summary)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
try:
|
||||||
|
self.driver.quit()
|
||||||
|
self._emit("scraper_closed", {})
|
||||||
|
except Exception as e:
|
||||||
|
self._emit("scraper_close_error", {"error": str(e)})
|
||||||
104
core/bookys_thread.py
Normal file
104
core/bookys_thread.py
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
"""
|
||||||
|
QThread wrapper for BookysScraper, translating callback events into Qt signals.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PyQt5.QtCore import QThread, pyqtSignal
|
||||||
|
from .bookys_scraper import BookysScraper
|
||||||
|
|
||||||
|
|
||||||
|
class BookysThread(QThread):
|
||||||
|
"""Runs BookysScraper off the GUI thread and emits progress signals."""
|
||||||
|
|
||||||
|
# High-level lifecycle
|
||||||
|
scraping_started = pyqtSignal(dict)
|
||||||
|
scraping_completed = pyqtSignal(dict)
|
||||||
|
error_occurred = pyqtSignal(str)
|
||||||
|
|
||||||
|
# Cloudflare handshake
|
||||||
|
cloudflare_waiting = pyqtSignal(str) # message
|
||||||
|
cloudflare_passed = pyqtSignal(int) # item_count
|
||||||
|
cloudflare_timeout = pyqtSignal(int) # timeout seconds
|
||||||
|
|
||||||
|
# Progress
|
||||||
|
page_started = pyqtSignal(int, int, int, str) # page_number, index, total, url
|
||||||
|
page_items_found = pyqtSignal(int, int) # page_number, item_count
|
||||||
|
page_completed = pyqtSignal(int, int) # page_number, items_processed
|
||||||
|
book_started = pyqtSignal(int, int, int, str) # page_number, book_index, total, url
|
||||||
|
book_completed = pyqtSignal(str, int) # title, links_found
|
||||||
|
link_found = pyqtSignal(str, int) # url, running_total
|
||||||
|
|
||||||
|
# Catch-all textual status for the log
|
||||||
|
status_update = pyqtSignal(str)
|
||||||
|
|
||||||
|
def __init__(self, category="bd", start_page=1, end_page=1,
|
||||||
|
output_file=None, timing_config=None):
|
||||||
|
super().__init__()
|
||||||
|
self.category = category
|
||||||
|
self.start_page = start_page
|
||||||
|
self.end_page = end_page
|
||||||
|
self.output_file = output_file
|
||||||
|
self.timing_config = timing_config
|
||||||
|
self.scraper = None
|
||||||
|
self._is_running = False
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
self._is_running = True
|
||||||
|
self.scraper = BookysScraper(
|
||||||
|
progress_callback=self._handle_progress,
|
||||||
|
category=self.category,
|
||||||
|
output_file=self.output_file,
|
||||||
|
timing_config=self.timing_config,
|
||||||
|
)
|
||||||
|
summary = self.scraper.scrape(self.start_page, self.end_page)
|
||||||
|
self.scraping_completed.emit(summary)
|
||||||
|
except Exception as e:
|
||||||
|
self.error_occurred.emit(f"Unexpected error: {e}")
|
||||||
|
finally:
|
||||||
|
if self.scraper:
|
||||||
|
self.scraper.close()
|
||||||
|
self._is_running = False
|
||||||
|
|
||||||
|
def _handle_progress(self, event_type, data):
|
||||||
|
try:
|
||||||
|
if event_type == "scraping_started":
|
||||||
|
self.scraping_started.emit(data)
|
||||||
|
elif event_type == "scraping_completed":
|
||||||
|
# Emitted from run() as well; skip here to avoid a double signal.
|
||||||
|
pass
|
||||||
|
elif event_type in ("cloudflare_waiting", "cloudflare_check"):
|
||||||
|
self.cloudflare_waiting.emit(data.get("message", "Waiting for page..."))
|
||||||
|
elif event_type == "cloudflare_passed":
|
||||||
|
self.cloudflare_passed.emit(data.get("item_count", 0))
|
||||||
|
elif event_type == "cloudflare_timeout":
|
||||||
|
self.cloudflare_timeout.emit(data.get("timeout", 0))
|
||||||
|
elif event_type == "page_started":
|
||||||
|
self.page_started.emit(
|
||||||
|
data.get("page_number", 1), data.get("page_index", 1),
|
||||||
|
data.get("total_pages", 1), data.get("url", ""))
|
||||||
|
elif event_type == "page_items_found":
|
||||||
|
self.page_items_found.emit(data.get("page_number", 1),
|
||||||
|
data.get("item_count", 0))
|
||||||
|
elif event_type == "page_completed":
|
||||||
|
self.page_completed.emit(data.get("page_number", 1),
|
||||||
|
data.get("items_processed", 0))
|
||||||
|
elif event_type == "book_started":
|
||||||
|
self.book_started.emit(
|
||||||
|
data.get("page_number", 1), data.get("book_index", 1),
|
||||||
|
data.get("total_books", 1), data.get("url", ""))
|
||||||
|
elif event_type == "book_completed":
|
||||||
|
self.book_completed.emit(data.get("title", ""),
|
||||||
|
data.get("links_found", 0))
|
||||||
|
elif event_type == "link_found":
|
||||||
|
self.link_found.emit(data.get("url", ""), data.get("total", 0))
|
||||||
|
else:
|
||||||
|
self.status_update.emit(f"{event_type}: {data}")
|
||||||
|
except Exception as e:
|
||||||
|
self.error_occurred.emit(f"Signal emission error: {e}")
|
||||||
|
|
||||||
|
def request_stop(self):
|
||||||
|
if self.scraper:
|
||||||
|
self.scraper.request_stop()
|
||||||
|
|
||||||
|
def is_running(self):
|
||||||
|
return self._is_running and self.isRunning()
|
||||||
@@ -257,7 +257,17 @@ class CredentialManager:
|
|||||||
'download_path': str(Path.home() / "Downloads"),
|
'download_path': str(Path.home() / "Downloads"),
|
||||||
'default_start_page': 1,
|
'default_start_page': 1,
|
||||||
'default_end_page': 1,
|
'default_end_page': 1,
|
||||||
'scraping_mode': 0 # 0=All Comics, 1=Latest Comics
|
'scraping_mode': 0, # 0=All Comics, 1=Latest Comics
|
||||||
|
# Timing configuration defaults
|
||||||
|
'action_delay_min': 0.5, # Minimum delay between actions (seconds)
|
||||||
|
'action_delay_max': 2.0, # Maximum delay between actions (seconds)
|
||||||
|
'page_break_chance': 70, # Percentage chance of taking a break between pages
|
||||||
|
'page_break_min': 15, # Minimum page break duration (seconds)
|
||||||
|
'page_break_max': 45, # Maximum page break duration (seconds)
|
||||||
|
'batch_break_interval': 5, # Take a break every N comics
|
||||||
|
'batch_break_min': 3, # Minimum batch break duration (seconds)
|
||||||
|
'batch_break_max': 7, # Maximum batch break duration (seconds)
|
||||||
|
'typing_delay': 0.1, # Delay between character typing (seconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
def export_settings(self, export_path):
|
def export_settings(self, export_path):
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class Scraper:
|
|||||||
callback mechanisms for progress updates to a GUI application.
|
callback mechanisms for progress updates to a GUI application.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, headless=False, progress_callback=None, scraping_mode=0):
|
def __init__(self, headless=False, progress_callback=None, scraping_mode=0, timing_config=None):
|
||||||
"""
|
"""
|
||||||
Initialize the scraper with optional GUI callback support.
|
Initialize the scraper with optional GUI callback support.
|
||||||
|
|
||||||
@@ -34,11 +34,16 @@ class Scraper:
|
|||||||
progress_callback (callable): Optional callback function for progress updates
|
progress_callback (callable): Optional callback function for progress updates
|
||||||
Callback signature: callback(event_type: str, data: dict)
|
Callback signature: callback(event_type: str, data: dict)
|
||||||
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
|
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
|
||||||
|
timing_config (dict): Timing configuration settings
|
||||||
"""
|
"""
|
||||||
self.progress_callback = progress_callback
|
self.progress_callback = progress_callback
|
||||||
self._stop_requested = False
|
self._stop_requested = False
|
||||||
self.scraping_mode = scraping_mode
|
self.scraping_mode = scraping_mode
|
||||||
|
|
||||||
|
# Set up timing configuration with defaults
|
||||||
|
self.timing = timing_config or {}
|
||||||
|
self._setup_timing_defaults()
|
||||||
|
|
||||||
# Set up Chrome options with anti-detection measures
|
# Set up Chrome options with anti-detection measures
|
||||||
chrome_options = Options()
|
chrome_options = Options()
|
||||||
if headless:
|
if headless:
|
||||||
@@ -103,16 +108,42 @@ class Scraper:
|
|||||||
self._stop_requested = True
|
self._stop_requested = True
|
||||||
self._emit_progress("stop_requested", {})
|
self._emit_progress("stop_requested", {})
|
||||||
|
|
||||||
def human_delay(self, min_sec=0.5, max_sec=2):
|
def _setup_timing_defaults(self):
|
||||||
|
"""Set up timing configuration with default values."""
|
||||||
|
defaults = {
|
||||||
|
'action_delay_min': 0.5,
|
||||||
|
'action_delay_max': 2.0,
|
||||||
|
'page_break_chance': 70,
|
||||||
|
'page_break_min': 15,
|
||||||
|
'page_break_max': 45,
|
||||||
|
'batch_break_interval': 5,
|
||||||
|
'batch_break_min': 3,
|
||||||
|
'batch_break_max': 7,
|
||||||
|
'typing_delay': 0.1,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fill in any missing values with defaults
|
||||||
|
for key, default_value in defaults.items():
|
||||||
|
if key not in self.timing:
|
||||||
|
self.timing[key] = default_value
|
||||||
|
|
||||||
|
def human_delay(self, min_sec=None, max_sec=None):
|
||||||
"""
|
"""
|
||||||
Simulate human-like delay with cancellation support.
|
Simulate human-like delay with cancellation support.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
min_sec (float): Minimum delay time
|
min_sec (float): Minimum delay time (uses config default if None)
|
||||||
max_sec (float): Maximum delay time
|
max_sec (float): Maximum delay time (uses config default if None)
|
||||||
"""
|
"""
|
||||||
if self._stop_requested:
|
if self._stop_requested:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Use configured timing if no specific values provided
|
||||||
|
if min_sec is None:
|
||||||
|
min_sec = self.timing['action_delay_min']
|
||||||
|
if max_sec is None:
|
||||||
|
max_sec = self.timing['action_delay_max']
|
||||||
|
|
||||||
delay_time = random.uniform(min_sec, max_sec)
|
delay_time = random.uniform(min_sec, max_sec)
|
||||||
self._emit_progress("delay_started", {"duration": delay_time})
|
self._emit_progress("delay_started", {"duration": delay_time})
|
||||||
time.sleep(delay_time)
|
time.sleep(delay_time)
|
||||||
@@ -129,7 +160,8 @@ class Scraper:
|
|||||||
if self._stop_requested:
|
if self._stop_requested:
|
||||||
return
|
return
|
||||||
element.send_keys(char)
|
element.send_keys(char)
|
||||||
time.sleep(random.uniform(0.05, 0.15))
|
typing_delay = self.timing['typing_delay']
|
||||||
|
time.sleep(random.uniform(typing_delay * 0.5, typing_delay * 1.5))
|
||||||
|
|
||||||
def navigate(self, url):
|
def navigate(self, url):
|
||||||
"""
|
"""
|
||||||
@@ -330,8 +362,9 @@ class Scraper:
|
|||||||
|
|
||||||
# Take a break between pages (more likely and longer)
|
# Take a break between pages (more likely and longer)
|
||||||
if page_num > start_page:
|
if page_num > start_page:
|
||||||
if random.random() < 0.7: # 70% chance of break
|
break_chance = self.timing['page_break_chance'] / 100.0
|
||||||
break_time = random.uniform(15, 45) # 15-45 seconds
|
if random.random() < break_chance:
|
||||||
|
break_time = random.uniform(self.timing['page_break_min'], self.timing['page_break_max'])
|
||||||
self._emit_progress("page_break_started", {
|
self._emit_progress("page_break_started", {
|
||||||
"duration": break_time,
|
"duration": break_time,
|
||||||
"page_number": page_num
|
"page_number": page_num
|
||||||
@@ -446,9 +479,10 @@ class Scraper:
|
|||||||
"comic_index": i
|
"comic_index": i
|
||||||
})
|
})
|
||||||
|
|
||||||
# Take a longer break every 5 comics
|
# Take a longer break every N comics (configurable)
|
||||||
if i % 5 == 0 and i < len(comic_urls):
|
batch_interval = self.timing['batch_break_interval']
|
||||||
break_time = random.uniform(3, 7)
|
if i % batch_interval == 0 and i < len(comic_urls):
|
||||||
|
break_time = random.uniform(self.timing['batch_break_min'], self.timing['batch_break_max'])
|
||||||
self._emit_progress("comic_batch_break", {
|
self._emit_progress("comic_batch_break", {
|
||||||
"duration": break_time,
|
"duration": break_time,
|
||||||
"comics_processed": i
|
"comics_processed": i
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ class ScraperThread(QThread):
|
|||||||
comic_batch_break = pyqtSignal(float, int) # duration, comics_processed
|
comic_batch_break = pyqtSignal(float, int) # duration, comics_processed
|
||||||
download_delay = pyqtSignal(float, int) # duration, remaining_downloads
|
download_delay = pyqtSignal(float, int) # duration, remaining_downloads
|
||||||
|
|
||||||
def __init__(self, username, password, start_page, end_page, scraping_mode=0, headless=True):
|
def __init__(self, username, password, start_page, end_page, scraping_mode=0, headless=True, timing_config=None):
|
||||||
"""
|
"""
|
||||||
Initialize the scraper thread.
|
Initialize the scraper thread.
|
||||||
|
|
||||||
@@ -69,6 +69,7 @@ class ScraperThread(QThread):
|
|||||||
end_page (int): Ending page number
|
end_page (int): Ending page number
|
||||||
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
|
scraping_mode (int): Scraping mode (0=All Comics, 1=Latest Comics)
|
||||||
headless (bool): Whether to run Chrome in headless mode
|
headless (bool): Whether to run Chrome in headless mode
|
||||||
|
timing_config (dict): Timing configuration settings
|
||||||
"""
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.username = username
|
self.username = username
|
||||||
@@ -77,6 +78,7 @@ class ScraperThread(QThread):
|
|||||||
self.end_page = end_page
|
self.end_page = end_page
|
||||||
self.scraping_mode = scraping_mode
|
self.scraping_mode = scraping_mode
|
||||||
self.headless = headless
|
self.headless = headless
|
||||||
|
self.timing_config = timing_config
|
||||||
self.scraper = None
|
self.scraper = None
|
||||||
self._is_running = False
|
self._is_running = False
|
||||||
|
|
||||||
@@ -92,7 +94,8 @@ class ScraperThread(QThread):
|
|||||||
self.scraper = Scraper(
|
self.scraper = Scraper(
|
||||||
headless=self.headless,
|
headless=self.headless,
|
||||||
progress_callback=self._handle_scraper_progress,
|
progress_callback=self._handle_scraper_progress,
|
||||||
scraping_mode=self.scraping_mode
|
scraping_mode=self.scraping_mode,
|
||||||
|
timing_config=self.timing_config
|
||||||
)
|
)
|
||||||
|
|
||||||
# Perform login
|
# Perform login
|
||||||
|
|||||||
Binary file not shown.
BIN
gui/__pycache__/bookys_window.cpython-314.pyc
Normal file
BIN
gui/__pycache__/bookys_window.cpython-314.pyc
Normal file
Binary file not shown.
BIN
gui/__pycache__/login_dialog.cpython-314.pyc
Normal file
BIN
gui/__pycache__/login_dialog.cpython-314.pyc
Normal file
Binary file not shown.
BIN
gui/__pycache__/main_window.cpython-314.pyc
Normal file
BIN
gui/__pycache__/main_window.cpython-314.pyc
Normal file
Binary file not shown.
BIN
gui/__pycache__/progress_dialog.cpython-314.pyc
Normal file
BIN
gui/__pycache__/progress_dialog.cpython-314.pyc
Normal file
Binary file not shown.
BIN
gui/__pycache__/startup_dialog.cpython-314.pyc
Normal file
BIN
gui/__pycache__/startup_dialog.cpython-314.pyc
Normal file
Binary file not shown.
405
gui/bookys_window.py
Normal file
405
gui/bookys_window.py
Normal file
@@ -0,0 +1,405 @@
|
|||||||
|
"""
|
||||||
|
Main window for the Bookys (bookys-ebooks.com) scraper.
|
||||||
|
|
||||||
|
Harvests direct 1fichier download URLs for a category and writes them to a .txt
|
||||||
|
file. Kept entirely separate from the EBoek window so the EBoek flow is untouched.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from PyQt5.QtWidgets import (
|
||||||
|
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||||
|
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox, QComboBox,
|
||||||
|
QLineEdit, QProgressBar, QMessageBox, QFileDialog, QApplication, QAction
|
||||||
|
)
|
||||||
|
from PyQt5.QtCore import Qt
|
||||||
|
|
||||||
|
project_root = Path(__file__).parent.parent
|
||||||
|
sys.path.insert(0, str(project_root))
|
||||||
|
|
||||||
|
from core.credentials import CredentialManager
|
||||||
|
from core.bookys_thread import BookysThread
|
||||||
|
from core.bookys_scraper import CATEGORIES, DEFAULT_PROFILE_DIR
|
||||||
|
from utils.validators import validate_page_range, format_error_message
|
||||||
|
|
||||||
|
|
||||||
|
class BookysWindow(QMainWindow):
|
||||||
|
"""Interface for scraping 1fichier links from bookys-ebooks.com."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.credential_manager = CredentialManager()
|
||||||
|
self.scraper_thread = None
|
||||||
|
|
||||||
|
# Bookys settings live under their own key in the shared config.
|
||||||
|
all_settings = self.credential_manager.load_app_settings() or {}
|
||||||
|
self.settings = all_settings.get('bookys', {})
|
||||||
|
|
||||||
|
self.init_ui()
|
||||||
|
self.apply_light_theme()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- set-up
|
||||||
|
|
||||||
|
def init_ui(self):
|
||||||
|
self.setWindowTitle("Bookys Scraper — 1fichier link harvester")
|
||||||
|
self.setMinimumSize(760, 620)
|
||||||
|
self.resize(900, 720)
|
||||||
|
|
||||||
|
self.create_menu_bar()
|
||||||
|
|
||||||
|
central = QWidget()
|
||||||
|
self.setCentralWidget(central)
|
||||||
|
layout = QVBoxLayout(central)
|
||||||
|
layout.setSpacing(16)
|
||||||
|
layout.setContentsMargins(20, 20, 20, 20)
|
||||||
|
|
||||||
|
self.create_config_section(layout)
|
||||||
|
self.create_output_section(layout)
|
||||||
|
self.create_control_section(layout)
|
||||||
|
self.create_log_section(layout)
|
||||||
|
|
||||||
|
self.statusBar().showMessage("Ready")
|
||||||
|
|
||||||
|
def apply_light_theme(self):
|
||||||
|
self.setStyleSheet("QMainWindow { background-color: #ffffff; }")
|
||||||
|
|
||||||
|
def create_menu_bar(self):
|
||||||
|
menubar = self.menuBar()
|
||||||
|
|
||||||
|
file_menu = menubar.addMenu('File')
|
||||||
|
open_out = QAction('Open Output Folder', self)
|
||||||
|
open_out.triggered.connect(self.open_output_folder)
|
||||||
|
file_menu.addAction(open_out)
|
||||||
|
file_menu.addSeparator()
|
||||||
|
exit_action = QAction('Exit', self)
|
||||||
|
exit_action.triggered.connect(self.close)
|
||||||
|
file_menu.addAction(exit_action)
|
||||||
|
|
||||||
|
mode_menu = menubar.addMenu('Mode')
|
||||||
|
to_eboek = QAction('Switch to EBoek', self)
|
||||||
|
to_eboek.triggered.connect(self._switch_to_eboek)
|
||||||
|
mode_menu.addAction(to_eboek)
|
||||||
|
|
||||||
|
browser_menu = menubar.addMenu('Browser')
|
||||||
|
reset_profile = QAction('Reset Browser Profile', self)
|
||||||
|
reset_profile.triggered.connect(self.reset_browser_profile)
|
||||||
|
browser_menu.addAction(reset_profile)
|
||||||
|
|
||||||
|
help_menu = menubar.addMenu('Help')
|
||||||
|
about_action = QAction('About', self)
|
||||||
|
about_action.triggered.connect(self.show_about)
|
||||||
|
help_menu.addAction(about_action)
|
||||||
|
|
||||||
|
def create_config_section(self, parent_layout):
|
||||||
|
group = QGroupBox("Scraping Configuration")
|
||||||
|
grid = QGridLayout(group)
|
||||||
|
|
||||||
|
grid.addWidget(QLabel("Category:"), 0, 0)
|
||||||
|
self.category_combo = QComboBox()
|
||||||
|
self._category_keys = list(CATEGORIES.keys())
|
||||||
|
for key in self._category_keys:
|
||||||
|
self.category_combo.addItem(CATEGORIES[key]["name"], key)
|
||||||
|
saved_cat = self.settings.get('category', 'bd')
|
||||||
|
if saved_cat in self._category_keys:
|
||||||
|
self.category_combo.setCurrentIndex(self._category_keys.index(saved_cat))
|
||||||
|
grid.addWidget(self.category_combo, 0, 1, 1, 3)
|
||||||
|
|
||||||
|
grid.addWidget(QLabel("Start Page:"), 1, 0)
|
||||||
|
self.start_page_spin = QSpinBox()
|
||||||
|
self.start_page_spin.setRange(1, 99999)
|
||||||
|
self.start_page_spin.setValue(self.settings.get('start_page', 1))
|
||||||
|
grid.addWidget(self.start_page_spin, 1, 1)
|
||||||
|
|
||||||
|
grid.addWidget(QLabel("End Page:"), 1, 2)
|
||||||
|
self.end_page_spin = QSpinBox()
|
||||||
|
self.end_page_spin.setRange(1, 99999)
|
||||||
|
self.end_page_spin.setValue(self.settings.get('end_page', 1))
|
||||||
|
grid.addWidget(self.end_page_spin, 1, 3)
|
||||||
|
|
||||||
|
hint = QLabel("💡 The browser opens visibly. If Cloudflare appears, tick "
|
||||||
|
"its checkbox — scraping resumes automatically.")
|
||||||
|
hint.setWordWrap(True)
|
||||||
|
hint.setStyleSheet("color: #666;")
|
||||||
|
grid.addWidget(hint, 2, 0, 1, 4)
|
||||||
|
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
def create_output_section(self, parent_layout):
|
||||||
|
group = QGroupBox("Output File (.txt)")
|
||||||
|
layout = QHBoxLayout(group)
|
||||||
|
|
||||||
|
default_out = self.settings.get(
|
||||||
|
'output_file', str(Path.home() / "Downloads" / "bookys_1fichier_links.txt"))
|
||||||
|
self.output_edit = QLineEdit(default_out)
|
||||||
|
layout.addWidget(self.output_edit)
|
||||||
|
|
||||||
|
browse_btn = QPushButton("Browse…")
|
||||||
|
browse_btn.clicked.connect(self.browse_output)
|
||||||
|
layout.addWidget(browse_btn)
|
||||||
|
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
def create_control_section(self, parent_layout):
|
||||||
|
group = QGroupBox("Status & Controls")
|
||||||
|
layout = QVBoxLayout(group)
|
||||||
|
|
||||||
|
row = QHBoxLayout()
|
||||||
|
info = QVBoxLayout()
|
||||||
|
self.status_label = QLabel("Ready to start.")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||||
|
info.addWidget(self.status_label)
|
||||||
|
|
||||||
|
self.links_label = QLabel("Links found: 0")
|
||||||
|
info.addWidget(self.links_label)
|
||||||
|
|
||||||
|
self.progress_bar = QProgressBar()
|
||||||
|
self.progress_bar.setVisible(False)
|
||||||
|
info.addWidget(self.progress_bar)
|
||||||
|
|
||||||
|
row.addLayout(info)
|
||||||
|
row.addStretch()
|
||||||
|
|
||||||
|
buttons = QVBoxLayout()
|
||||||
|
self.start_btn = QPushButton("Start Scraping")
|
||||||
|
self.start_btn.clicked.connect(self.start_scraping)
|
||||||
|
buttons.addWidget(self.start_btn)
|
||||||
|
|
||||||
|
self.stop_btn = QPushButton("Stop")
|
||||||
|
self.stop_btn.clicked.connect(self.stop_scraping)
|
||||||
|
self.stop_btn.setEnabled(False)
|
||||||
|
buttons.addWidget(self.stop_btn)
|
||||||
|
|
||||||
|
row.addLayout(buttons)
|
||||||
|
layout.addLayout(row)
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
def create_log_section(self, parent_layout):
|
||||||
|
group = QGroupBox("Activity Log")
|
||||||
|
layout = QVBoxLayout(group)
|
||||||
|
self.log_view = QTextEdit()
|
||||||
|
self.log_view.setReadOnly(True)
|
||||||
|
layout.addWidget(self.log_view)
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ actions
|
||||||
|
|
||||||
|
def log(self, message):
|
||||||
|
self.log_view.append(message)
|
||||||
|
|
||||||
|
def browse_output(self):
|
||||||
|
path, _ = QFileDialog.getSaveFileName(
|
||||||
|
self, "Choose output file", self.output_edit.text(),
|
||||||
|
"Text files (*.txt);;All files (*.*)")
|
||||||
|
if path:
|
||||||
|
self.output_edit.setText(path)
|
||||||
|
|
||||||
|
def _current_category(self):
|
||||||
|
return self.category_combo.currentData() or 'bd'
|
||||||
|
|
||||||
|
def save_settings(self):
|
||||||
|
all_settings = self.credential_manager.load_app_settings() or {}
|
||||||
|
all_settings['bookys'] = {
|
||||||
|
'category': self._current_category(),
|
||||||
|
'start_page': self.start_page_spin.value(),
|
||||||
|
'end_page': self.end_page_spin.value(),
|
||||||
|
'output_file': self.output_edit.text(),
|
||||||
|
}
|
||||||
|
self.credential_manager.save_app_settings(all_settings)
|
||||||
|
self.settings = all_settings['bookys']
|
||||||
|
|
||||||
|
def start_scraping(self):
|
||||||
|
start_page = self.start_page_spin.value()
|
||||||
|
end_page = self.end_page_spin.value()
|
||||||
|
|
||||||
|
validation = validate_page_range(start_page, end_page)
|
||||||
|
if not validation['valid']:
|
||||||
|
QMessageBox.warning(self, "Invalid Page Range",
|
||||||
|
format_error_message(validation['errors']))
|
||||||
|
return
|
||||||
|
|
||||||
|
output_file = self.output_edit.text().strip()
|
||||||
|
if not output_file:
|
||||||
|
QMessageBox.warning(self, "No Output File",
|
||||||
|
"Please choose where to save the links (.txt).")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.save_settings()
|
||||||
|
self.log_view.clear()
|
||||||
|
self.log(f"Starting Bookys scrape — {CATEGORIES[self._current_category()]['name']}, "
|
||||||
|
f"pages {start_page}–{end_page}")
|
||||||
|
self.log(f"Writing links to: {output_file}")
|
||||||
|
|
||||||
|
self.scraper_thread = BookysThread(
|
||||||
|
category=self._current_category(),
|
||||||
|
start_page=start_page,
|
||||||
|
end_page=end_page,
|
||||||
|
output_file=output_file,
|
||||||
|
)
|
||||||
|
self._connect_signals()
|
||||||
|
self.scraper_thread.start()
|
||||||
|
|
||||||
|
self.start_btn.setEnabled(False)
|
||||||
|
self.stop_btn.setEnabled(True)
|
||||||
|
self.progress_bar.setVisible(True)
|
||||||
|
self.progress_bar.setRange(0, 0) # indeterminate until first page completes
|
||||||
|
self.status_label.setText("Opening browser…")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
||||||
|
|
||||||
|
def stop_scraping(self):
|
||||||
|
if self.scraper_thread and self.scraper_thread.is_running():
|
||||||
|
self.scraper_thread.request_stop()
|
||||||
|
self.log("Stop requested — finishing current item…")
|
||||||
|
self.stop_btn.setEnabled(False)
|
||||||
|
|
||||||
|
def _connect_signals(self):
|
||||||
|
t = self.scraper_thread
|
||||||
|
t.cloudflare_waiting.connect(self.on_cloudflare_waiting)
|
||||||
|
t.cloudflare_passed.connect(self.on_cloudflare_passed)
|
||||||
|
t.cloudflare_timeout.connect(self.on_cloudflare_timeout)
|
||||||
|
t.page_started.connect(self.on_page_started)
|
||||||
|
t.page_items_found.connect(self.on_page_items_found)
|
||||||
|
t.page_completed.connect(self.on_page_completed)
|
||||||
|
t.book_started.connect(self.on_book_started)
|
||||||
|
t.book_completed.connect(self.on_book_completed)
|
||||||
|
t.link_found.connect(self.on_link_found)
|
||||||
|
t.scraping_completed.connect(self.on_scraping_completed)
|
||||||
|
t.error_occurred.connect(lambda m: self.log(f"ERROR: {m}"))
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------- signals
|
||||||
|
|
||||||
|
def on_cloudflare_waiting(self, message):
|
||||||
|
self.status_label.setText("Waiting for Cloudflare…")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
||||||
|
self.log(f"⏳ {message}")
|
||||||
|
|
||||||
|
def on_cloudflare_passed(self, item_count):
|
||||||
|
self.log(f"✅ Page loaded ({item_count} items).")
|
||||||
|
self.status_label.setText("Scraping…")
|
||||||
|
|
||||||
|
def on_cloudflare_timeout(self, timeout):
|
||||||
|
self.log(f"⚠️ Timed out after {timeout}s waiting for the page/Cloudflare.")
|
||||||
|
|
||||||
|
def on_page_started(self, page_number, page_index, total_pages, url):
|
||||||
|
self.log(f"— Page {page_number} ({page_index}/{total_pages})")
|
||||||
|
if total_pages > 0:
|
||||||
|
self.progress_bar.setRange(0, total_pages)
|
||||||
|
self.progress_bar.setValue(page_index - 1)
|
||||||
|
|
||||||
|
def on_page_items_found(self, page_number, item_count):
|
||||||
|
self.log(f" Found {item_count} items on page {page_number}.")
|
||||||
|
|
||||||
|
def on_page_completed(self, page_number, items_processed):
|
||||||
|
self.log(f" Page {page_number} done ({items_processed} items).")
|
||||||
|
self.progress_bar.setValue(self.progress_bar.value() + 1)
|
||||||
|
|
||||||
|
def on_book_started(self, page_number, book_index, total_books, url):
|
||||||
|
self.status_label.setText(f"Page {page_number}: item {book_index}/{total_books}")
|
||||||
|
|
||||||
|
def on_book_completed(self, title, links_found):
|
||||||
|
label = title if title else "(untitled)"
|
||||||
|
self.log(f" • {label} — {links_found} 1fichier link(s)")
|
||||||
|
|
||||||
|
def on_link_found(self, url, total):
|
||||||
|
self.links_label.setText(f"Links found: {total}")
|
||||||
|
|
||||||
|
def on_scraping_completed(self, summary):
|
||||||
|
self.start_btn.setEnabled(True)
|
||||||
|
self.stop_btn.setEnabled(False)
|
||||||
|
self.progress_bar.setRange(0, 1)
|
||||||
|
self.progress_bar.setValue(1)
|
||||||
|
|
||||||
|
total = summary.get('total_links_found', 0)
|
||||||
|
if summary.get('cancelled'):
|
||||||
|
self.status_label.setText("Cancelled")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
||||||
|
elif summary.get('success'):
|
||||||
|
self.status_label.setText("Completed")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||||||
|
else:
|
||||||
|
self.status_label.setText("Completed with errors")
|
||||||
|
self.status_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
||||||
|
|
||||||
|
self.log(f"Done. {total} link(s) written to {summary.get('output_file', '')}")
|
||||||
|
errors = summary.get('errors') or []
|
||||||
|
if errors:
|
||||||
|
self.log(f"{len(errors)} error(s) occurred:")
|
||||||
|
for e in errors[:10]:
|
||||||
|
self.log(f" ! {e}")
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------- misc
|
||||||
|
|
||||||
|
def open_output_folder(self):
|
||||||
|
folder = Path(self.output_edit.text()).parent
|
||||||
|
try:
|
||||||
|
if sys.platform == "win32":
|
||||||
|
os.startfile(folder)
|
||||||
|
elif sys.platform == "darwin":
|
||||||
|
subprocess.run(["open", str(folder)])
|
||||||
|
else:
|
||||||
|
subprocess.run(["xdg-open", str(folder)])
|
||||||
|
except Exception as e:
|
||||||
|
QMessageBox.information(self, "Output Folder",
|
||||||
|
f"Links are saved to:\n{folder}\n\n"
|
||||||
|
f"Could not open folder automatically: {e}")
|
||||||
|
|
||||||
|
def reset_browser_profile(self):
|
||||||
|
"""Delete the persistent Chrome profile (forces a fresh Cloudflare pass)."""
|
||||||
|
if self.scraper_thread and self.scraper_thread.is_running():
|
||||||
|
QMessageBox.warning(self, "Scraping in Progress",
|
||||||
|
"Stop the current scrape before resetting the profile.")
|
||||||
|
return
|
||||||
|
|
||||||
|
reply = QMessageBox.question(
|
||||||
|
self, "Reset Browser Profile",
|
||||||
|
f"Delete the saved browser profile?\n\n{DEFAULT_PROFILE_DIR}\n\n"
|
||||||
|
"You'll need to pass the Cloudflare check once more on the next run. "
|
||||||
|
"Use this if the browser stops loading pages correctly.",
|
||||||
|
QMessageBox.Yes | QMessageBox.No)
|
||||||
|
|
||||||
|
if reply != QMessageBox.Yes:
|
||||||
|
return
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
try:
|
||||||
|
if DEFAULT_PROFILE_DIR.exists():
|
||||||
|
shutil.rmtree(DEFAULT_PROFILE_DIR)
|
||||||
|
self.log("Browser profile reset.")
|
||||||
|
QMessageBox.information(self, "Profile Reset",
|
||||||
|
"The browser profile has been cleared.")
|
||||||
|
except Exception as e:
|
||||||
|
QMessageBox.warning(self, "Reset Failed", f"Could not reset profile:\n{e}")
|
||||||
|
|
||||||
|
def show_about(self):
|
||||||
|
QMessageBox.about(self, "About Bookys Scraper",
|
||||||
|
"Bookys Scraper\n\n"
|
||||||
|
"Harvests direct 1fichier download URLs from "
|
||||||
|
"bookys-ebooks.com and saves them to a .txt file.\n\n"
|
||||||
|
"• Visible browser (Cloudflare-friendly)\n"
|
||||||
|
"• Auto-detects when the challenge clears\n"
|
||||||
|
"• Ignores pop-up ads\n\n"
|
||||||
|
"Built with Python and PyQt5.")
|
||||||
|
|
||||||
|
def _switch_to_eboek(self):
|
||||||
|
app = QApplication.instance()
|
||||||
|
if hasattr(app, 'show_eboek'):
|
||||||
|
app.show_eboek()
|
||||||
|
|
||||||
|
def closeEvent(self, event):
|
||||||
|
if self.scraper_thread and self.scraper_thread.is_running():
|
||||||
|
reply = QMessageBox.question(
|
||||||
|
self, "Scraping in Progress",
|
||||||
|
"Scraping is running. Stop and continue?",
|
||||||
|
QMessageBox.Yes | QMessageBox.No)
|
||||||
|
if reply == QMessageBox.Yes:
|
||||||
|
self.scraper_thread.request_stop()
|
||||||
|
self.scraper_thread.wait(3000)
|
||||||
|
event.accept()
|
||||||
|
else:
|
||||||
|
event.ignore()
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self.save_settings()
|
||||||
|
event.accept()
|
||||||
@@ -6,9 +6,9 @@ import sys
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from PyQt5.QtWidgets import (
|
from PyQt5.QtWidgets import (
|
||||||
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||||||
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox,
|
QPushButton, QLabel, QSpinBox, QDoubleSpinBox, QTextEdit, QGroupBox,
|
||||||
QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction,
|
QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction,
|
||||||
QComboBox
|
QComboBox, QFrame, QScrollArea, QSizePolicy
|
||||||
)
|
)
|
||||||
from PyQt5.QtCore import Qt, QTimer, pyqtSignal
|
from PyQt5.QtCore import Qt, QTimer, pyqtSignal
|
||||||
from PyQt5.QtGui import QFont, QIcon
|
from PyQt5.QtGui import QFont, QIcon
|
||||||
@@ -54,33 +54,55 @@ class MainWindow(QMainWindow):
|
|||||||
self.app_settings = self.credential_manager.get_default_settings()
|
self.app_settings = self.credential_manager.get_default_settings()
|
||||||
|
|
||||||
self.init_ui()
|
self.init_ui()
|
||||||
|
self.apply_light_theme()
|
||||||
self.update_credential_status()
|
self.update_credential_status()
|
||||||
|
|
||||||
def init_ui(self):
|
def init_ui(self):
|
||||||
"""Initialize the user interface."""
|
"""Initialize the user interface."""
|
||||||
self.setWindowTitle("EBoek.info Scraper")
|
self.setWindowTitle("EBoek.info Scraper")
|
||||||
self.setMinimumSize(600, 500)
|
self.setMinimumSize(900, 650)
|
||||||
self.resize(700, 600)
|
self.resize(1200, 750)
|
||||||
|
|
||||||
# Create menu bar
|
# Create menu bar
|
||||||
self.create_menu_bar()
|
self.create_menu_bar()
|
||||||
|
|
||||||
# Create central widget
|
# Create central widget with scroll area for better responsiveness
|
||||||
central_widget = QWidget()
|
scroll_area = QScrollArea()
|
||||||
self.setCentralWidget(central_widget)
|
scroll_widget = QWidget()
|
||||||
|
scroll_area.setWidget(scroll_widget)
|
||||||
|
scroll_area.setWidgetResizable(True)
|
||||||
|
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||||||
|
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||||||
|
self.setCentralWidget(scroll_area)
|
||||||
|
|
||||||
# Main layout
|
# Main layout with better spacing
|
||||||
layout = QVBoxLayout(central_widget)
|
layout = QVBoxLayout(scroll_widget)
|
||||||
|
layout.setSpacing(20)
|
||||||
|
layout.setContentsMargins(20, 20, 20, 20)
|
||||||
|
|
||||||
# Create sections
|
# Create sections in a more organized way
|
||||||
self.create_credential_section(layout)
|
self.create_credential_section(layout)
|
||||||
self.create_scraping_section(layout)
|
self.create_scraping_section(layout)
|
||||||
|
self.create_timing_section(layout)
|
||||||
self.create_status_section(layout)
|
self.create_status_section(layout)
|
||||||
self.create_control_section(layout)
|
self.create_control_section(layout)
|
||||||
|
|
||||||
|
# Add stretch to push content to top
|
||||||
|
layout.addStretch()
|
||||||
|
|
||||||
# Status bar
|
# Status bar
|
||||||
self.statusBar().showMessage("Ready")
|
self.statusBar().showMessage("Ready")
|
||||||
|
|
||||||
|
def apply_light_theme(self):
|
||||||
|
"""Apply minimal light theme styling with completely default system controls."""
|
||||||
|
# Very minimal stylesheet - only style the main containers, no form controls
|
||||||
|
light_stylesheet = """
|
||||||
|
QMainWindow {
|
||||||
|
background-color: #ffffff;
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
self.setStyleSheet(light_stylesheet)
|
||||||
|
|
||||||
def create_menu_bar(self):
|
def create_menu_bar(self):
|
||||||
"""Create the menu bar."""
|
"""Create the menu bar."""
|
||||||
menubar = self.menuBar()
|
menubar = self.menuBar()
|
||||||
@@ -102,6 +124,12 @@ class MainWindow(QMainWindow):
|
|||||||
exit_action.triggered.connect(self.close)
|
exit_action.triggered.connect(self.close)
|
||||||
file_menu.addAction(exit_action)
|
file_menu.addAction(exit_action)
|
||||||
|
|
||||||
|
# Mode menu - switch between the EBoek and Bookys scrapers
|
||||||
|
mode_menu = menubar.addMenu('Mode')
|
||||||
|
to_bookys_action = QAction('Switch to Bookys', self)
|
||||||
|
to_bookys_action.triggered.connect(self._switch_to_bookys)
|
||||||
|
mode_menu.addAction(to_bookys_action)
|
||||||
|
|
||||||
# Settings menu
|
# Settings menu
|
||||||
settings_menu = menubar.addMenu('Settings')
|
settings_menu = menubar.addMenu('Settings')
|
||||||
|
|
||||||
@@ -116,16 +144,28 @@ class MainWindow(QMainWindow):
|
|||||||
about_action.triggered.connect(self.show_about)
|
about_action.triggered.connect(self.show_about)
|
||||||
help_menu.addAction(about_action)
|
help_menu.addAction(about_action)
|
||||||
|
|
||||||
|
def _switch_to_bookys(self):
|
||||||
|
"""Switch to the Bookys scraper window."""
|
||||||
|
from PyQt5.QtWidgets import QApplication
|
||||||
|
app = QApplication.instance()
|
||||||
|
if hasattr(app, 'show_bookys'):
|
||||||
|
app.show_bookys()
|
||||||
|
|
||||||
def create_credential_section(self, parent_layout):
|
def create_credential_section(self, parent_layout):
|
||||||
"""Create the credential management section."""
|
"""Create the credential management section."""
|
||||||
group = QGroupBox("Credentials")
|
group = QGroupBox("Account Credentials")
|
||||||
layout = QHBoxLayout(group)
|
layout = QHBoxLayout(group)
|
||||||
|
|
||||||
|
# Status with icon
|
||||||
|
status_layout = QHBoxLayout()
|
||||||
self.credential_status_label = QLabel("No credentials configured")
|
self.credential_status_label = QLabel("No credentials configured")
|
||||||
layout.addWidget(self.credential_status_label)
|
self.credential_status_label.setStyleSheet("font-size: 13px;")
|
||||||
|
status_layout.addWidget(self.credential_status_label)
|
||||||
|
|
||||||
|
layout.addLayout(status_layout)
|
||||||
layout.addStretch()
|
layout.addStretch()
|
||||||
|
|
||||||
|
# Button with default system styling
|
||||||
self.change_credentials_btn = QPushButton("Change Credentials")
|
self.change_credentials_btn = QPushButton("Change Credentials")
|
||||||
self.change_credentials_btn.clicked.connect(self.show_login_dialog)
|
self.change_credentials_btn.clicked.connect(self.show_login_dialog)
|
||||||
layout.addWidget(self.change_credentials_btn)
|
layout.addWidget(self.change_credentials_btn)
|
||||||
@@ -135,51 +175,71 @@ class MainWindow(QMainWindow):
|
|||||||
def create_scraping_section(self, parent_layout):
|
def create_scraping_section(self, parent_layout):
|
||||||
"""Create the scraping configuration section."""
|
"""Create the scraping configuration section."""
|
||||||
group = QGroupBox("Scraping Configuration")
|
group = QGroupBox("Scraping Configuration")
|
||||||
layout = QGridLayout(group)
|
main_layout = QVBoxLayout(group)
|
||||||
|
|
||||||
|
# Create horizontal layout for better space usage
|
||||||
|
top_section = QHBoxLayout()
|
||||||
|
|
||||||
|
# Left side - Mode selection
|
||||||
|
mode_group = QGroupBox("Scraping Mode")
|
||||||
|
mode_layout = QVBoxLayout(mode_group)
|
||||||
|
|
||||||
# Scraping mode selection
|
|
||||||
layout.addWidget(QLabel("Mode:"), 0, 0)
|
|
||||||
self.mode_combo = QComboBox()
|
self.mode_combo = QComboBox()
|
||||||
self.mode_combo.addItems([
|
self.mode_combo.addItems([
|
||||||
"All Comics (stripverhalen-alle)",
|
"All Comics (Complete Archive)",
|
||||||
"Latest Comics (laatste)"
|
"Latest Comics (Recent Additions)"
|
||||||
])
|
])
|
||||||
self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0))
|
self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0))
|
||||||
self.mode_combo.setToolTip("Select which page type to scrape")
|
|
||||||
self.mode_combo.currentIndexChanged.connect(self.on_mode_changed)
|
self.mode_combo.currentIndexChanged.connect(self.on_mode_changed)
|
||||||
layout.addWidget(self.mode_combo, 0, 1, 1, 3)
|
mode_layout.addWidget(self.mode_combo)
|
||||||
|
|
||||||
# Page range selection
|
# Mode description label
|
||||||
layout.addWidget(QLabel("Start Page:"), 1, 0)
|
self.mode_description_label = QLabel("")
|
||||||
|
self.mode_description_label.setWordWrap(True)
|
||||||
|
mode_layout.addWidget(self.mode_description_label)
|
||||||
|
|
||||||
|
mode_group.setMaximumWidth(400)
|
||||||
|
|
||||||
|
# Right side - Page range and options
|
||||||
|
config_group = QGroupBox("Configuration")
|
||||||
|
config_layout = QGridLayout(config_group)
|
||||||
|
|
||||||
|
# Page range
|
||||||
|
config_layout.addWidget(QLabel("Start Page:"), 0, 0)
|
||||||
self.start_page_spin = QSpinBox()
|
self.start_page_spin = QSpinBox()
|
||||||
self.start_page_spin.setMinimum(1)
|
self.start_page_spin.setMinimum(1)
|
||||||
self.start_page_spin.setMaximum(9999)
|
self.start_page_spin.setMaximum(9999)
|
||||||
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
||||||
layout.addWidget(self.start_page_spin, 1, 1)
|
config_layout.addWidget(self.start_page_spin, 0, 1)
|
||||||
|
|
||||||
layout.addWidget(QLabel("End Page:"), 1, 2)
|
config_layout.addWidget(QLabel("End Page:"), 0, 2)
|
||||||
self.end_page_spin = QSpinBox()
|
self.end_page_spin = QSpinBox()
|
||||||
self.end_page_spin.setMinimum(1)
|
self.end_page_spin.setMinimum(1)
|
||||||
self.end_page_spin.setMaximum(9999)
|
self.end_page_spin.setMaximum(9999)
|
||||||
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
||||||
layout.addWidget(self.end_page_spin, 1, 3)
|
config_layout.addWidget(self.end_page_spin, 0, 3)
|
||||||
|
|
||||||
# Mode description label
|
|
||||||
self.mode_description_label = QLabel("")
|
|
||||||
self.mode_description_label.setStyleSheet("color: #666; font-size: 11px; font-style: italic;")
|
|
||||||
self.mode_description_label.setWordWrap(True)
|
|
||||||
layout.addWidget(self.mode_description_label, 2, 0, 1, 4)
|
|
||||||
|
|
||||||
# Options
|
# Options
|
||||||
self.headless_checkbox = QCheckBox("Headless Mode")
|
self.headless_checkbox = QCheckBox("Headless Mode")
|
||||||
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
||||||
self.headless_checkbox.setToolTip("Run browser in background (recommended)")
|
config_layout.addWidget(self.headless_checkbox, 1, 0, 1, 2)
|
||||||
layout.addWidget(self.headless_checkbox, 3, 0, 1, 2)
|
|
||||||
|
|
||||||
self.verbose_checkbox = QCheckBox("Verbose Logging")
|
self.verbose_checkbox = QCheckBox("Verbose Logging")
|
||||||
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
||||||
self.verbose_checkbox.setToolTip("Show detailed progress information")
|
config_layout.addWidget(self.verbose_checkbox, 1, 2, 1, 2)
|
||||||
layout.addWidget(self.verbose_checkbox, 3, 2, 1, 2)
|
|
||||||
|
# Help texts
|
||||||
|
page_help = QLabel("💡 Start with 1-3 pages to test")
|
||||||
|
config_layout.addWidget(page_help, 2, 0, 1, 4)
|
||||||
|
|
||||||
|
browser_help = QLabel("🖥️ Headless = background mode (recommended) • Verbose = detailed logs")
|
||||||
|
config_layout.addWidget(browser_help, 3, 0, 1, 4)
|
||||||
|
|
||||||
|
# Add to horizontal layout
|
||||||
|
top_section.addWidget(mode_group)
|
||||||
|
top_section.addWidget(config_group)
|
||||||
|
|
||||||
|
main_layout.addLayout(top_section)
|
||||||
|
|
||||||
# Update mode description
|
# Update mode description
|
||||||
self.update_mode_description()
|
self.update_mode_description()
|
||||||
@@ -196,49 +256,286 @@ class MainWindow(QMainWindow):
|
|||||||
mode_index = self.mode_combo.currentIndex()
|
mode_index = self.mode_combo.currentIndex()
|
||||||
|
|
||||||
if mode_index == 0: # All Comics
|
if mode_index == 0: # All Comics
|
||||||
description = ("Scrapes all comics from the 'stripverhalen-alle' page. "
|
description = ("🗃️ <b>All Comics (Complete Archive)</b><br>"
|
||||||
"This is the original scraping mode with complete comic archives.")
|
"Scrapes from 'stripverhalen-alle' page containing the full comic archive. "
|
||||||
|
"Best for systematic collection of all available comics. "
|
||||||
|
"Large page counts available (1000+ pages).")
|
||||||
elif mode_index == 1: # Latest Comics
|
elif mode_index == 1: # Latest Comics
|
||||||
description = ("Scrapes latest comics from the 'laatste' page. "
|
description = ("🆕 <b>Latest Comics (Recent Additions)</b><br>"
|
||||||
"This mode gets the most recently added comics with page parameter support.")
|
"Scrapes from 'laatste' page with most recently added comics. "
|
||||||
|
"Perfect for staying up-to-date with new releases. "
|
||||||
|
"Smaller page counts but fresh content.")
|
||||||
else:
|
else:
|
||||||
description = ""
|
description = ""
|
||||||
|
|
||||||
self.mode_description_label.setText(description)
|
self.mode_description_label.setText(description)
|
||||||
|
|
||||||
|
def create_timing_section(self, parent_layout):
|
||||||
|
"""Create the timing configuration section."""
|
||||||
|
group = QGroupBox("Timing Configuration")
|
||||||
|
main_layout = QVBoxLayout(group)
|
||||||
|
|
||||||
|
# Create horizontal layout for better space usage
|
||||||
|
top_layout = QHBoxLayout()
|
||||||
|
|
||||||
|
# Left side - Quick presets
|
||||||
|
preset_group = QGroupBox("Quick Presets")
|
||||||
|
preset_layout = QVBoxLayout(preset_group)
|
||||||
|
|
||||||
|
preset_info = QLabel("Choose a predefined timing profile:")
|
||||||
|
preset_layout.addWidget(preset_info)
|
||||||
|
|
||||||
|
preset_buttons_layout = QVBoxLayout()
|
||||||
|
|
||||||
|
fast_btn = QPushButton("Fast Mode")
|
||||||
|
fast_btn.clicked.connect(lambda: self.apply_timing_preset("fast"))
|
||||||
|
preset_buttons_layout.addWidget(fast_btn)
|
||||||
|
|
||||||
|
fast_desc = QLabel("Minimal delays • Faster scraping • Higher detection risk")
|
||||||
|
fast_desc.setWordWrap(True)
|
||||||
|
preset_buttons_layout.addWidget(fast_desc)
|
||||||
|
|
||||||
|
balanced_btn = QPushButton("Balanced Mode (Recommended)")
|
||||||
|
balanced_btn.clicked.connect(lambda: self.apply_timing_preset("balanced"))
|
||||||
|
preset_buttons_layout.addWidget(balanced_btn)
|
||||||
|
|
||||||
|
balanced_desc = QLabel("Default settings • Good balance • Recommended for most users")
|
||||||
|
balanced_desc.setWordWrap(True)
|
||||||
|
preset_buttons_layout.addWidget(balanced_desc)
|
||||||
|
|
||||||
|
stealth_btn = QPushButton("Stealth Mode")
|
||||||
|
stealth_btn.clicked.connect(lambda: self.apply_timing_preset("stealth"))
|
||||||
|
preset_buttons_layout.addWidget(stealth_btn)
|
||||||
|
|
||||||
|
stealth_desc = QLabel("Maximum delays • Very human-like • Slower but undetectable")
|
||||||
|
stealth_desc.setWordWrap(True)
|
||||||
|
preset_buttons_layout.addWidget(stealth_desc)
|
||||||
|
|
||||||
|
preset_layout.addLayout(preset_buttons_layout)
|
||||||
|
preset_layout.addStretch()
|
||||||
|
preset_group.setMinimumWidth(350) # Changed from setMaximumWidth to setMinimumWidth
|
||||||
|
|
||||||
|
# Right side - Manual controls in a more compact layout
|
||||||
|
manual_group = QGroupBox("Manual Configuration")
|
||||||
|
manual_layout = QGridLayout(manual_group)
|
||||||
|
|
||||||
|
# Action delays
|
||||||
|
manual_layout.addWidget(QLabel("Action Delays (sec):"), 0, 0)
|
||||||
|
self.action_delay_min_spin = self.create_double_spinbox(0.1, 10.0, 0.1,
|
||||||
|
self.app_settings.get('action_delay_min', 0.5))
|
||||||
|
manual_layout.addWidget(self.action_delay_min_spin, 0, 1)
|
||||||
|
manual_layout.addWidget(QLabel("to"), 0, 2)
|
||||||
|
self.action_delay_max_spin = self.create_double_spinbox(0.1, 10.0, 0.1,
|
||||||
|
self.app_settings.get('action_delay_max', 2.0))
|
||||||
|
manual_layout.addWidget(self.action_delay_max_spin, 0, 3)
|
||||||
|
|
||||||
|
# Page breaks
|
||||||
|
manual_layout.addWidget(QLabel("Page Break Chance:"), 1, 0)
|
||||||
|
self.page_break_chance_spin = QSpinBox()
|
||||||
|
self.page_break_chance_spin.setRange(0, 100)
|
||||||
|
self.page_break_chance_spin.setValue(self.app_settings.get('page_break_chance', 70))
|
||||||
|
self.page_break_chance_spin.setSuffix("%")
|
||||||
|
manual_layout.addWidget(self.page_break_chance_spin, 1, 1)
|
||||||
|
|
||||||
|
manual_layout.addWidget(QLabel("Duration:"), 1, 2)
|
||||||
|
page_duration_layout = QHBoxLayout()
|
||||||
|
self.page_break_min_spin = QSpinBox()
|
||||||
|
self.page_break_min_spin.setRange(5, 300)
|
||||||
|
self.page_break_min_spin.setValue(self.app_settings.get('page_break_min', 15))
|
||||||
|
page_duration_layout.addWidget(self.page_break_min_spin)
|
||||||
|
page_duration_layout.addWidget(QLabel("-"))
|
||||||
|
self.page_break_max_spin = QSpinBox()
|
||||||
|
self.page_break_max_spin.setRange(5, 300)
|
||||||
|
self.page_break_max_spin.setValue(self.app_settings.get('page_break_max', 45))
|
||||||
|
self.page_break_max_spin.setSuffix("s")
|
||||||
|
page_duration_layout.addWidget(self.page_break_max_spin)
|
||||||
|
manual_layout.addLayout(page_duration_layout, 1, 3)
|
||||||
|
|
||||||
|
# Batch breaks
|
||||||
|
manual_layout.addWidget(QLabel("Batch Break Every:"), 2, 0)
|
||||||
|
self.batch_break_interval_spin = QSpinBox()
|
||||||
|
self.batch_break_interval_spin.setRange(1, 50)
|
||||||
|
self.batch_break_interval_spin.setValue(self.app_settings.get('batch_break_interval', 5))
|
||||||
|
self.batch_break_interval_spin.setSuffix(" comics")
|
||||||
|
manual_layout.addWidget(self.batch_break_interval_spin, 2, 1)
|
||||||
|
|
||||||
|
manual_layout.addWidget(QLabel("Duration:"), 2, 2)
|
||||||
|
batch_duration_layout = QHBoxLayout()
|
||||||
|
self.batch_break_min_spin = QSpinBox()
|
||||||
|
self.batch_break_min_spin.setRange(1, 60)
|
||||||
|
self.batch_break_min_spin.setValue(self.app_settings.get('batch_break_min', 3))
|
||||||
|
batch_duration_layout.addWidget(self.batch_break_min_spin)
|
||||||
|
batch_duration_layout.addWidget(QLabel("-"))
|
||||||
|
self.batch_break_max_spin = QSpinBox()
|
||||||
|
self.batch_break_max_spin.setRange(1, 60)
|
||||||
|
self.batch_break_max_spin.setValue(self.app_settings.get('batch_break_max', 7))
|
||||||
|
self.batch_break_max_spin.setSuffix("s")
|
||||||
|
batch_duration_layout.addWidget(self.batch_break_max_spin)
|
||||||
|
manual_layout.addLayout(batch_duration_layout, 2, 3)
|
||||||
|
|
||||||
|
# Typing speed
|
||||||
|
manual_layout.addWidget(QLabel("Typing Speed:"), 3, 0)
|
||||||
|
self.typing_delay_spin = self.create_double_spinbox(0.01, 1.0, 0.01,
|
||||||
|
self.app_settings.get('typing_delay', 0.1))
|
||||||
|
self.typing_delay_spin.setSuffix(" sec/char")
|
||||||
|
manual_layout.addWidget(self.typing_delay_spin, 3, 1)
|
||||||
|
|
||||||
|
# Bottom row with reset and help buttons
|
||||||
|
button_layout = QHBoxLayout()
|
||||||
|
|
||||||
|
self.reset_timing_btn = QPushButton("Reset to Balanced")
|
||||||
|
self.reset_timing_btn.clicked.connect(self.reset_timing_defaults)
|
||||||
|
button_layout.addWidget(self.reset_timing_btn)
|
||||||
|
|
||||||
|
help_btn = QPushButton("Help")
|
||||||
|
help_btn.clicked.connect(self.show_timing_help_dialog)
|
||||||
|
button_layout.addWidget(help_btn)
|
||||||
|
|
||||||
|
button_layout.addStretch()
|
||||||
|
manual_layout.addLayout(button_layout, 4, 0, 1, 4)
|
||||||
|
|
||||||
|
# Add to horizontal layout with proper proportions
|
||||||
|
top_layout.addWidget(preset_group, 1) # Give preset group more space
|
||||||
|
top_layout.addWidget(manual_group, 1) # Equal space for manual group
|
||||||
|
|
||||||
|
main_layout.addLayout(top_layout)
|
||||||
|
|
||||||
|
# Set the timing section to expand properly
|
||||||
|
group.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred)
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
def create_double_spinbox(self, min_val, max_val, step, value):
|
||||||
|
"""Create a double precision spinbox with specified parameters."""
|
||||||
|
spinbox = QDoubleSpinBox()
|
||||||
|
spinbox.setRange(min_val, max_val)
|
||||||
|
spinbox.setSingleStep(step)
|
||||||
|
spinbox.setDecimals(1)
|
||||||
|
spinbox.setValue(value)
|
||||||
|
return spinbox
|
||||||
|
|
||||||
|
def show_timing_help_dialog(self):
|
||||||
|
"""Show a compact help dialog for timing configuration."""
|
||||||
|
dialog = QMessageBox(self)
|
||||||
|
dialog.setWindowTitle("Timing Configuration Help")
|
||||||
|
dialog.setIcon(QMessageBox.Information)
|
||||||
|
|
||||||
|
help_text = """
|
||||||
|
<b>Timing Configuration Guide</b><br><br>
|
||||||
|
|
||||||
|
<b>🎯 Purpose:</b> Simulate human-like browsing to avoid automated detection<br><br>
|
||||||
|
|
||||||
|
<b>⚡ Action Delays:</b> Time between clicks and scrolls<br>
|
||||||
|
• Lower = Faster scraping, higher detection risk<br>
|
||||||
|
• Higher = Slower scraping, more realistic behavior<br><br>
|
||||||
|
|
||||||
|
<b>⏸️ Page Breaks:</b> Random pauses between pages<br>
|
||||||
|
• Simulates reading time and human fatigue<br>
|
||||||
|
• 70% chance with 15-45s duration is realistic<br><br>
|
||||||
|
|
||||||
|
<b>📚 Batch Breaks:</b> Pauses after processing multiple comics<br>
|
||||||
|
• Every 5 comics with 3-7s breaks simulates attention shifts<br><br>
|
||||||
|
|
||||||
|
<b>⌨️ Typing Speed:</b> Character delay when entering credentials<br>
|
||||||
|
• 0.1 sec = Average typing speed (40 WPM)<br><br>
|
||||||
|
|
||||||
|
<b>💡 Recommendations:</b><br>
|
||||||
|
• <b>Fast Mode:</b> Testing or when speed matters more than stealth<br>
|
||||||
|
• <b>Balanced Mode:</b> Recommended for regular use<br>
|
||||||
|
• <b>Stealth Mode:</b> Maximum safety but slower operation
|
||||||
|
"""
|
||||||
|
|
||||||
|
dialog.setText(help_text)
|
||||||
|
dialog.setStandardButtons(QMessageBox.Ok)
|
||||||
|
dialog.exec_()
|
||||||
|
|
||||||
|
def apply_timing_preset(self, preset_type):
|
||||||
|
"""Apply a timing preset configuration."""
|
||||||
|
if preset_type == "fast":
|
||||||
|
# Fast & Aggressive - minimal delays
|
||||||
|
self.action_delay_min_spin.setValue(0.1)
|
||||||
|
self.action_delay_max_spin.setValue(0.5)
|
||||||
|
self.page_break_chance_spin.setValue(20)
|
||||||
|
self.page_break_min_spin.setValue(5)
|
||||||
|
self.page_break_max_spin.setValue(10)
|
||||||
|
self.batch_break_interval_spin.setValue(15)
|
||||||
|
self.batch_break_min_spin.setValue(1)
|
||||||
|
self.batch_break_max_spin.setValue(2)
|
||||||
|
self.typing_delay_spin.setValue(0.05)
|
||||||
|
|
||||||
|
elif preset_type == "balanced":
|
||||||
|
# Balanced - default recommended settings
|
||||||
|
self.action_delay_min_spin.setValue(0.5)
|
||||||
|
self.action_delay_max_spin.setValue(2.0)
|
||||||
|
self.page_break_chance_spin.setValue(70)
|
||||||
|
self.page_break_min_spin.setValue(15)
|
||||||
|
self.page_break_max_spin.setValue(45)
|
||||||
|
self.batch_break_interval_spin.setValue(5)
|
||||||
|
self.batch_break_min_spin.setValue(3)
|
||||||
|
self.batch_break_max_spin.setValue(7)
|
||||||
|
self.typing_delay_spin.setValue(0.1)
|
||||||
|
|
||||||
|
elif preset_type == "stealth":
|
||||||
|
# Stealth - maximum human-like behavior
|
||||||
|
self.action_delay_min_spin.setValue(1.0)
|
||||||
|
self.action_delay_max_spin.setValue(3.0)
|
||||||
|
self.page_break_chance_spin.setValue(90)
|
||||||
|
self.page_break_min_spin.setValue(30)
|
||||||
|
self.page_break_max_spin.setValue(90)
|
||||||
|
self.batch_break_interval_spin.setValue(3)
|
||||||
|
self.batch_break_min_spin.setValue(5)
|
||||||
|
self.batch_break_max_spin.setValue(15)
|
||||||
|
self.typing_delay_spin.setValue(0.15)
|
||||||
|
|
||||||
|
# Save the new settings
|
||||||
|
self.save_current_settings()
|
||||||
|
|
||||||
|
# Show feedback
|
||||||
|
self.statusBar().showMessage(f"Applied {preset_type.title()} timing preset", 3000)
|
||||||
|
|
||||||
|
def reset_timing_defaults(self):
|
||||||
|
"""Reset all timing settings to safe defaults."""
|
||||||
|
self.apply_timing_preset("balanced")
|
||||||
|
|
||||||
def create_status_section(self, parent_layout):
|
def create_status_section(self, parent_layout):
|
||||||
"""Create the status display section."""
|
"""Create the status display section."""
|
||||||
group = QGroupBox("Status")
|
group = QGroupBox("Status & Controls")
|
||||||
layout = QVBoxLayout(group)
|
layout = QVBoxLayout(group)
|
||||||
|
|
||||||
|
# Status display
|
||||||
|
status_layout = QHBoxLayout()
|
||||||
|
|
||||||
|
status_info_layout = QVBoxLayout()
|
||||||
self.status_label = QLabel("Ready to start scraping...")
|
self.status_label = QLabel("Ready to start scraping...")
|
||||||
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57; font-size: 13px;")
|
||||||
layout.addWidget(self.status_label)
|
status_info_layout.addWidget(self.status_label)
|
||||||
|
|
||||||
# Progress bar
|
# Progress bar
|
||||||
self.progress_bar = QProgressBar()
|
self.progress_bar = QProgressBar()
|
||||||
self.progress_bar.setVisible(False)
|
self.progress_bar.setVisible(False)
|
||||||
layout.addWidget(self.progress_bar)
|
status_info_layout.addWidget(self.progress_bar)
|
||||||
|
|
||||||
parent_layout.addWidget(group)
|
status_layout.addLayout(status_info_layout)
|
||||||
|
status_layout.addStretch()
|
||||||
|
|
||||||
|
# Control buttons
|
||||||
def create_control_section(self, parent_layout):
|
button_layout = QVBoxLayout()
|
||||||
"""Create the control buttons section."""
|
|
||||||
layout = QHBoxLayout()
|
|
||||||
|
|
||||||
self.start_btn = QPushButton("Start Scraping")
|
self.start_btn = QPushButton("Start Scraping")
|
||||||
self.start_btn.clicked.connect(self.start_scraping)
|
self.start_btn.clicked.connect(self.start_scraping)
|
||||||
self.start_btn.setStyleSheet("QPushButton { background-color: #4CAF50; color: white; font-weight: bold; padding: 8px; }")
|
button_layout.addWidget(self.start_btn)
|
||||||
layout.addWidget(self.start_btn)
|
|
||||||
|
|
||||||
layout.addStretch()
|
|
||||||
|
|
||||||
self.downloads_btn = QPushButton("Open Downloads Folder")
|
self.downloads_btn = QPushButton("Open Downloads Folder")
|
||||||
self.downloads_btn.clicked.connect(self.open_downloads_folder)
|
self.downloads_btn.clicked.connect(self.open_downloads_folder)
|
||||||
layout.addWidget(self.downloads_btn)
|
button_layout.addWidget(self.downloads_btn)
|
||||||
|
|
||||||
parent_layout.addLayout(layout)
|
status_layout.addLayout(button_layout)
|
||||||
|
layout.addLayout(status_layout)
|
||||||
|
|
||||||
|
parent_layout.addWidget(group)
|
||||||
|
|
||||||
|
def create_control_section(self, parent_layout):
|
||||||
|
"""Create the control buttons section - now integrated into status section."""
|
||||||
|
pass # This is now handled by create_status_section
|
||||||
|
|
||||||
def update_credential_status(self):
|
def update_credential_status(self):
|
||||||
"""Update the credential status display."""
|
"""Update the credential status display."""
|
||||||
@@ -289,13 +586,27 @@ class MainWindow(QMainWindow):
|
|||||||
self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}")
|
self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}")
|
||||||
|
|
||||||
# Create and start scraper thread
|
# Create and start scraper thread
|
||||||
|
# Collect current timing configuration
|
||||||
|
timing_config = {
|
||||||
|
'action_delay_min': self.action_delay_min_spin.value(),
|
||||||
|
'action_delay_max': self.action_delay_max_spin.value(),
|
||||||
|
'page_break_chance': self.page_break_chance_spin.value(),
|
||||||
|
'page_break_min': self.page_break_min_spin.value(),
|
||||||
|
'page_break_max': self.page_break_max_spin.value(),
|
||||||
|
'batch_break_interval': self.batch_break_interval_spin.value(),
|
||||||
|
'batch_break_min': self.batch_break_min_spin.value(),
|
||||||
|
'batch_break_max': self.batch_break_max_spin.value(),
|
||||||
|
'typing_delay': self.typing_delay_spin.value(),
|
||||||
|
}
|
||||||
|
|
||||||
self.scraper_thread = ScraperThread(
|
self.scraper_thread = ScraperThread(
|
||||||
username=credentials['username'],
|
username=credentials['username'],
|
||||||
password=credentials['password'],
|
password=credentials['password'],
|
||||||
start_page=start_page,
|
start_page=start_page,
|
||||||
end_page=end_page,
|
end_page=end_page,
|
||||||
scraping_mode=mode_index,
|
scraping_mode=mode_index,
|
||||||
headless=self.headless_checkbox.isChecked()
|
headless=self.headless_checkbox.isChecked(),
|
||||||
|
timing_config=timing_config
|
||||||
)
|
)
|
||||||
|
|
||||||
# Connect signals
|
# Connect signals
|
||||||
@@ -398,6 +709,16 @@ class MainWindow(QMainWindow):
|
|||||||
'default_start_page': self.start_page_spin.value(),
|
'default_start_page': self.start_page_spin.value(),
|
||||||
'default_end_page': self.end_page_spin.value(),
|
'default_end_page': self.end_page_spin.value(),
|
||||||
'scraping_mode': self.mode_combo.currentIndex(),
|
'scraping_mode': self.mode_combo.currentIndex(),
|
||||||
|
# Timing settings
|
||||||
|
'action_delay_min': self.action_delay_min_spin.value(),
|
||||||
|
'action_delay_max': self.action_delay_max_spin.value(),
|
||||||
|
'page_break_chance': self.page_break_chance_spin.value(),
|
||||||
|
'page_break_min': self.page_break_min_spin.value(),
|
||||||
|
'page_break_max': self.page_break_max_spin.value(),
|
||||||
|
'batch_break_interval': self.batch_break_interval_spin.value(),
|
||||||
|
'batch_break_min': self.batch_break_min_spin.value(),
|
||||||
|
'batch_break_max': self.batch_break_max_spin.value(),
|
||||||
|
'typing_delay': self.typing_delay_spin.value(),
|
||||||
}
|
}
|
||||||
settings.update(self.app_settings) # Keep other settings
|
settings.update(self.app_settings) # Keep other settings
|
||||||
self.credential_manager.save_app_settings(settings)
|
self.credential_manager.save_app_settings(settings)
|
||||||
|
|||||||
58
gui/startup_dialog.py
Normal file
58
gui/startup_dialog.py
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
"""
|
||||||
|
Startup chooser: pick which site to scrape when the app boots.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PyQt5.QtWidgets import (
|
||||||
|
QDialog, QVBoxLayout, QHBoxLayout, QPushButton, QLabel
|
||||||
|
)
|
||||||
|
from PyQt5.QtCore import Qt
|
||||||
|
|
||||||
|
|
||||||
|
class StartupDialog(QDialog):
|
||||||
|
"""Small modal shown on boot to choose EBoek or Bookys."""
|
||||||
|
|
||||||
|
EBOEK = "eboek"
|
||||||
|
BOOKYS = "bookys"
|
||||||
|
|
||||||
|
def __init__(self, parent=None):
|
||||||
|
super().__init__(parent)
|
||||||
|
self.choice = None
|
||||||
|
self._init_ui()
|
||||||
|
|
||||||
|
def _init_ui(self):
|
||||||
|
self.setWindowTitle("Choose Scraper")
|
||||||
|
self.setModal(True)
|
||||||
|
self.setMinimumWidth(420)
|
||||||
|
|
||||||
|
layout = QVBoxLayout(self)
|
||||||
|
layout.setSpacing(18)
|
||||||
|
layout.setContentsMargins(28, 28, 28, 28)
|
||||||
|
|
||||||
|
title = QLabel("Which site do you want to scrape?")
|
||||||
|
title.setStyleSheet("font-size: 16px; font-weight: bold;")
|
||||||
|
title.setAlignment(Qt.AlignCenter)
|
||||||
|
layout.addWidget(title)
|
||||||
|
|
||||||
|
subtitle = QLabel("You can switch anytime from the menu bar.")
|
||||||
|
subtitle.setAlignment(Qt.AlignCenter)
|
||||||
|
subtitle.setStyleSheet("color: #666;")
|
||||||
|
layout.addWidget(subtitle)
|
||||||
|
|
||||||
|
buttons = QHBoxLayout()
|
||||||
|
buttons.setSpacing(14)
|
||||||
|
|
||||||
|
eboek_btn = QPushButton("EBoek.info\n\nDownload comic files")
|
||||||
|
eboek_btn.setMinimumHeight(90)
|
||||||
|
eboek_btn.clicked.connect(lambda: self._choose(self.EBOEK))
|
||||||
|
buttons.addWidget(eboek_btn)
|
||||||
|
|
||||||
|
bookys_btn = QPushButton("Bookys\n\nHarvest 1fichier links → .txt")
|
||||||
|
bookys_btn.setMinimumHeight(90)
|
||||||
|
bookys_btn.clicked.connect(lambda: self._choose(self.BOOKYS))
|
||||||
|
buttons.addWidget(bookys_btn)
|
||||||
|
|
||||||
|
layout.addLayout(buttons)
|
||||||
|
|
||||||
|
def _choose(self, choice):
|
||||||
|
self.choice = choice
|
||||||
|
self.accept()
|
||||||
37
gui_main.py
37
gui_main.py
@@ -59,7 +59,8 @@ class EBoekScraperApp(QApplication):
|
|||||||
# Handle exceptions
|
# Handle exceptions
|
||||||
sys.excepthook = self.handle_exception
|
sys.excepthook = self.handle_exception
|
||||||
|
|
||||||
self.main_window = None
|
self.main_window = None # EBoek window
|
||||||
|
self.bookys_window = None # Bookys window
|
||||||
|
|
||||||
def set_application_icon(self):
|
def set_application_icon(self):
|
||||||
"""Set the application icon if available."""
|
"""Set the application icon if available."""
|
||||||
@@ -101,9 +102,16 @@ class EBoekScraperApp(QApplication):
|
|||||||
# Check system requirements
|
# Check system requirements
|
||||||
self.check_requirements()
|
self.check_requirements()
|
||||||
|
|
||||||
# Create and show main window
|
# Ask which site to scrape, then show the matching window.
|
||||||
self.main_window = MainWindow()
|
from gui.startup_dialog import StartupDialog
|
||||||
self.main_window.show()
|
chooser = StartupDialog()
|
||||||
|
chooser.exec_()
|
||||||
|
|
||||||
|
if chooser.choice == StartupDialog.BOOKYS:
|
||||||
|
self.show_bookys()
|
||||||
|
else:
|
||||||
|
# Default to EBoek (also covers the dialog being closed).
|
||||||
|
self.show_eboek()
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -124,6 +132,27 @@ class EBoekScraperApp(QApplication):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def show_eboek(self):
|
||||||
|
"""Show the EBoek window, hiding the Bookys one if present."""
|
||||||
|
if self.main_window is None:
|
||||||
|
self.main_window = MainWindow()
|
||||||
|
if self.bookys_window is not None:
|
||||||
|
self.bookys_window.hide()
|
||||||
|
self.main_window.show()
|
||||||
|
self.main_window.raise_()
|
||||||
|
self.main_window.activateWindow()
|
||||||
|
|
||||||
|
def show_bookys(self):
|
||||||
|
"""Show the Bookys window, hiding the EBoek one if present."""
|
||||||
|
from gui.bookys_window import BookysWindow
|
||||||
|
if self.bookys_window is None:
|
||||||
|
self.bookys_window = BookysWindow()
|
||||||
|
if self.main_window is not None:
|
||||||
|
self.main_window.hide()
|
||||||
|
self.bookys_window.show()
|
||||||
|
self.bookys_window.raise_()
|
||||||
|
self.bookys_window.activateWindow()
|
||||||
|
|
||||||
def check_requirements(self):
|
def check_requirements(self):
|
||||||
"""Check system requirements and dependencies."""
|
"""Check system requirements and dependencies."""
|
||||||
errors = []
|
errors = []
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ python -m pyinstaller --onefile --windowed --name "EBoek_Scraper" ^
|
|||||||
--hidden-import "gui.login_dialog" ^
|
--hidden-import "gui.login_dialog" ^
|
||||||
--hidden-import "gui.progress_dialog" ^
|
--hidden-import "gui.progress_dialog" ^
|
||||||
--hidden-import "utils.validators" ^
|
--hidden-import "utils.validators" ^
|
||||||
|
--hidden-import "core.bookys_scraper" ^
|
||||||
|
--hidden-import "core.bookys_thread" ^
|
||||||
|
--hidden-import "gui.bookys_window" ^
|
||||||
|
--hidden-import "gui.startup_dialog" ^
|
||||||
--exclude-module "tkinter" ^
|
--exclude-module "tkinter" ^
|
||||||
--exclude-module "matplotlib" ^
|
--exclude-module "matplotlib" ^
|
||||||
..\gui_main.py
|
..\gui_main.py
|
||||||
|
|||||||
@@ -92,6 +92,11 @@ def main():
|
|||||||
"--hidden-import", "gui.login_dialog",
|
"--hidden-import", "gui.login_dialog",
|
||||||
"--hidden-import", "gui.progress_dialog",
|
"--hidden-import", "gui.progress_dialog",
|
||||||
"--hidden-import", "utils.validators",
|
"--hidden-import", "utils.validators",
|
||||||
|
# Bookys scraper modules (imported lazily, so PyInstaller can't see them)
|
||||||
|
"--hidden-import", "core.bookys_scraper",
|
||||||
|
"--hidden-import", "core.bookys_thread",
|
||||||
|
"--hidden-import", "gui.bookys_window",
|
||||||
|
"--hidden-import", "gui.startup_dialog",
|
||||||
# Exclude unnecessary modules to reduce size
|
# Exclude unnecessary modules to reduce size
|
||||||
"--exclude-module", "tkinter",
|
"--exclude-module", "tkinter",
|
||||||
"--exclude-module", "matplotlib",
|
"--exclude-module", "matplotlib",
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ a = Analysis(
|
|||||||
pathex=[],
|
pathex=[],
|
||||||
binaries=[],
|
binaries=[],
|
||||||
datas=[],
|
datas=[],
|
||||||
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators'],
|
hiddenimports=['PyQt5.QtCore', 'PyQt5.QtGui', 'PyQt5.QtWidgets', 'selenium', 'selenium.webdriver', 'selenium.webdriver.chrome', 'selenium.webdriver.chrome.options', 'selenium.webdriver.common.by', 'core.scraper', 'core.scraper_thread', 'core.credentials', 'gui.main_window', 'gui.login_dialog', 'gui.progress_dialog', 'utils.validators', 'core.bookys_scraper', 'core.bookys_thread', 'gui.bookys_window', 'gui.startup_dialog'],
|
||||||
hookspath=[],
|
hookspath=[],
|
||||||
hooksconfig={},
|
hooksconfig={},
|
||||||
runtime_hooks=[],
|
runtime_hooks=[],
|
||||||
|
|||||||
Binary file not shown.
BIN
utils/__pycache__/validators.cpython-314.pyc
Normal file
BIN
utils/__pycache__/validators.cpython-314.pyc
Normal file
Binary file not shown.
Reference in New Issue
Block a user