c6a69c6ada
Adds a second scraper for bookys-ebooks.com that harvests direct 1fichier download URLs into a .txt file. The existing EBoek scraper core is unchanged; only the GUI gained a Mode menu entry to switch between the two. Bookys specifics: - Visible (never headless) browser, since the site is behind Cloudflare. The challenge is auto-detected: the scraper polls for real listing items and continues on its own once it clears, no button needed. - Persistent Chrome profile in ~/.eboek_scraper/bookys_chrome_profile so the Cloudflare clearance cookie survives between runs. Removed the hardcoded user-agent override, which claimed Chrome/120 against a real Chrome/150 and invalidated that cookie on every launch. - Pop-up ads are avoided by never clicking host links: hrefs are read and navigated to directly, with window.open neutered via CDP as a backstop. - Host links are read with textContent rather than Selenium's .text, which returns empty for these elements because they sit in a collapsed container. - Only 1fichier hosts are followed; other hosts on a page are ignored. Build configs list the new modules as hidden imports in all four places. gui_main imports the Bookys window lazily, so PyInstaller's static analysis would otherwise miss it and the .exe would crash on switching modes. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NVJc2XuTvqBHSuqotetseS
406 lines
16 KiB
Python
406 lines
16 KiB
Python
"""
|
||
Main window for the Bookys (bookys-ebooks.com) scraper.
|
||
|
||
Harvests direct 1fichier download URLs for a category and writes them to a .txt
|
||
file. Kept entirely separate from the EBoek window so the EBoek flow is untouched.
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import subprocess
|
||
from pathlib import Path
|
||
|
||
from PyQt5.QtWidgets import (
|
||
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
||
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox, QComboBox,
|
||
QLineEdit, QProgressBar, QMessageBox, QFileDialog, QApplication, QAction
|
||
)
|
||
from PyQt5.QtCore import Qt
|
||
|
||
project_root = Path(__file__).parent.parent
|
||
sys.path.insert(0, str(project_root))
|
||
|
||
from core.credentials import CredentialManager
|
||
from core.bookys_thread import BookysThread
|
||
from core.bookys_scraper import CATEGORIES, DEFAULT_PROFILE_DIR
|
||
from utils.validators import validate_page_range, format_error_message
|
||
|
||
|
||
class BookysWindow(QMainWindow):
|
||
"""Interface for scraping 1fichier links from bookys-ebooks.com."""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.credential_manager = CredentialManager()
|
||
self.scraper_thread = None
|
||
|
||
# Bookys settings live under their own key in the shared config.
|
||
all_settings = self.credential_manager.load_app_settings() or {}
|
||
self.settings = all_settings.get('bookys', {})
|
||
|
||
self.init_ui()
|
||
self.apply_light_theme()
|
||
|
||
# ------------------------------------------------------------------- set-up
|
||
|
||
def init_ui(self):
|
||
self.setWindowTitle("Bookys Scraper — 1fichier link harvester")
|
||
self.setMinimumSize(760, 620)
|
||
self.resize(900, 720)
|
||
|
||
self.create_menu_bar()
|
||
|
||
central = QWidget()
|
||
self.setCentralWidget(central)
|
||
layout = QVBoxLayout(central)
|
||
layout.setSpacing(16)
|
||
layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
self.create_config_section(layout)
|
||
self.create_output_section(layout)
|
||
self.create_control_section(layout)
|
||
self.create_log_section(layout)
|
||
|
||
self.statusBar().showMessage("Ready")
|
||
|
||
def apply_light_theme(self):
|
||
self.setStyleSheet("QMainWindow { background-color: #ffffff; }")
|
||
|
||
def create_menu_bar(self):
|
||
menubar = self.menuBar()
|
||
|
||
file_menu = menubar.addMenu('File')
|
||
open_out = QAction('Open Output Folder', self)
|
||
open_out.triggered.connect(self.open_output_folder)
|
||
file_menu.addAction(open_out)
|
||
file_menu.addSeparator()
|
||
exit_action = QAction('Exit', self)
|
||
exit_action.triggered.connect(self.close)
|
||
file_menu.addAction(exit_action)
|
||
|
||
mode_menu = menubar.addMenu('Mode')
|
||
to_eboek = QAction('Switch to EBoek', self)
|
||
to_eboek.triggered.connect(self._switch_to_eboek)
|
||
mode_menu.addAction(to_eboek)
|
||
|
||
browser_menu = menubar.addMenu('Browser')
|
||
reset_profile = QAction('Reset Browser Profile', self)
|
||
reset_profile.triggered.connect(self.reset_browser_profile)
|
||
browser_menu.addAction(reset_profile)
|
||
|
||
help_menu = menubar.addMenu('Help')
|
||
about_action = QAction('About', self)
|
||
about_action.triggered.connect(self.show_about)
|
||
help_menu.addAction(about_action)
|
||
|
||
def create_config_section(self, parent_layout):
|
||
group = QGroupBox("Scraping Configuration")
|
||
grid = QGridLayout(group)
|
||
|
||
grid.addWidget(QLabel("Category:"), 0, 0)
|
||
self.category_combo = QComboBox()
|
||
self._category_keys = list(CATEGORIES.keys())
|
||
for key in self._category_keys:
|
||
self.category_combo.addItem(CATEGORIES[key]["name"], key)
|
||
saved_cat = self.settings.get('category', 'bd')
|
||
if saved_cat in self._category_keys:
|
||
self.category_combo.setCurrentIndex(self._category_keys.index(saved_cat))
|
||
grid.addWidget(self.category_combo, 0, 1, 1, 3)
|
||
|
||
grid.addWidget(QLabel("Start Page:"), 1, 0)
|
||
self.start_page_spin = QSpinBox()
|
||
self.start_page_spin.setRange(1, 99999)
|
||
self.start_page_spin.setValue(self.settings.get('start_page', 1))
|
||
grid.addWidget(self.start_page_spin, 1, 1)
|
||
|
||
grid.addWidget(QLabel("End Page:"), 1, 2)
|
||
self.end_page_spin = QSpinBox()
|
||
self.end_page_spin.setRange(1, 99999)
|
||
self.end_page_spin.setValue(self.settings.get('end_page', 1))
|
||
grid.addWidget(self.end_page_spin, 1, 3)
|
||
|
||
hint = QLabel("💡 The browser opens visibly. If Cloudflare appears, tick "
|
||
"its checkbox — scraping resumes automatically.")
|
||
hint.setWordWrap(True)
|
||
hint.setStyleSheet("color: #666;")
|
||
grid.addWidget(hint, 2, 0, 1, 4)
|
||
|
||
parent_layout.addWidget(group)
|
||
|
||
def create_output_section(self, parent_layout):
|
||
group = QGroupBox("Output File (.txt)")
|
||
layout = QHBoxLayout(group)
|
||
|
||
default_out = self.settings.get(
|
||
'output_file', str(Path.home() / "Downloads" / "bookys_1fichier_links.txt"))
|
||
self.output_edit = QLineEdit(default_out)
|
||
layout.addWidget(self.output_edit)
|
||
|
||
browse_btn = QPushButton("Browse…")
|
||
browse_btn.clicked.connect(self.browse_output)
|
||
layout.addWidget(browse_btn)
|
||
|
||
parent_layout.addWidget(group)
|
||
|
||
def create_control_section(self, parent_layout):
|
||
group = QGroupBox("Status & Controls")
|
||
layout = QVBoxLayout(group)
|
||
|
||
row = QHBoxLayout()
|
||
info = QVBoxLayout()
|
||
self.status_label = QLabel("Ready to start.")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||
info.addWidget(self.status_label)
|
||
|
||
self.links_label = QLabel("Links found: 0")
|
||
info.addWidget(self.links_label)
|
||
|
||
self.progress_bar = QProgressBar()
|
||
self.progress_bar.setVisible(False)
|
||
info.addWidget(self.progress_bar)
|
||
|
||
row.addLayout(info)
|
||
row.addStretch()
|
||
|
||
buttons = QVBoxLayout()
|
||
self.start_btn = QPushButton("Start Scraping")
|
||
self.start_btn.clicked.connect(self.start_scraping)
|
||
buttons.addWidget(self.start_btn)
|
||
|
||
self.stop_btn = QPushButton("Stop")
|
||
self.stop_btn.clicked.connect(self.stop_scraping)
|
||
self.stop_btn.setEnabled(False)
|
||
buttons.addWidget(self.stop_btn)
|
||
|
||
row.addLayout(buttons)
|
||
layout.addLayout(row)
|
||
parent_layout.addWidget(group)
|
||
|
||
def create_log_section(self, parent_layout):
|
||
group = QGroupBox("Activity Log")
|
||
layout = QVBoxLayout(group)
|
||
self.log_view = QTextEdit()
|
||
self.log_view.setReadOnly(True)
|
||
layout.addWidget(self.log_view)
|
||
parent_layout.addWidget(group)
|
||
|
||
# ------------------------------------------------------------------ actions
|
||
|
||
def log(self, message):
|
||
self.log_view.append(message)
|
||
|
||
def browse_output(self):
|
||
path, _ = QFileDialog.getSaveFileName(
|
||
self, "Choose output file", self.output_edit.text(),
|
||
"Text files (*.txt);;All files (*.*)")
|
||
if path:
|
||
self.output_edit.setText(path)
|
||
|
||
def _current_category(self):
|
||
return self.category_combo.currentData() or 'bd'
|
||
|
||
def save_settings(self):
|
||
all_settings = self.credential_manager.load_app_settings() or {}
|
||
all_settings['bookys'] = {
|
||
'category': self._current_category(),
|
||
'start_page': self.start_page_spin.value(),
|
||
'end_page': self.end_page_spin.value(),
|
||
'output_file': self.output_edit.text(),
|
||
}
|
||
self.credential_manager.save_app_settings(all_settings)
|
||
self.settings = all_settings['bookys']
|
||
|
||
def start_scraping(self):
|
||
start_page = self.start_page_spin.value()
|
||
end_page = self.end_page_spin.value()
|
||
|
||
validation = validate_page_range(start_page, end_page)
|
||
if not validation['valid']:
|
||
QMessageBox.warning(self, "Invalid Page Range",
|
||
format_error_message(validation['errors']))
|
||
return
|
||
|
||
output_file = self.output_edit.text().strip()
|
||
if not output_file:
|
||
QMessageBox.warning(self, "No Output File",
|
||
"Please choose where to save the links (.txt).")
|
||
return
|
||
|
||
self.save_settings()
|
||
self.log_view.clear()
|
||
self.log(f"Starting Bookys scrape — {CATEGORIES[self._current_category()]['name']}, "
|
||
f"pages {start_page}–{end_page}")
|
||
self.log(f"Writing links to: {output_file}")
|
||
|
||
self.scraper_thread = BookysThread(
|
||
category=self._current_category(),
|
||
start_page=start_page,
|
||
end_page=end_page,
|
||
output_file=output_file,
|
||
)
|
||
self._connect_signals()
|
||
self.scraper_thread.start()
|
||
|
||
self.start_btn.setEnabled(False)
|
||
self.stop_btn.setEnabled(True)
|
||
self.progress_bar.setVisible(True)
|
||
self.progress_bar.setRange(0, 0) # indeterminate until first page completes
|
||
self.status_label.setText("Opening browser…")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
||
|
||
def stop_scraping(self):
|
||
if self.scraper_thread and self.scraper_thread.is_running():
|
||
self.scraper_thread.request_stop()
|
||
self.log("Stop requested — finishing current item…")
|
||
self.stop_btn.setEnabled(False)
|
||
|
||
def _connect_signals(self):
|
||
t = self.scraper_thread
|
||
t.cloudflare_waiting.connect(self.on_cloudflare_waiting)
|
||
t.cloudflare_passed.connect(self.on_cloudflare_passed)
|
||
t.cloudflare_timeout.connect(self.on_cloudflare_timeout)
|
||
t.page_started.connect(self.on_page_started)
|
||
t.page_items_found.connect(self.on_page_items_found)
|
||
t.page_completed.connect(self.on_page_completed)
|
||
t.book_started.connect(self.on_book_started)
|
||
t.book_completed.connect(self.on_book_completed)
|
||
t.link_found.connect(self.on_link_found)
|
||
t.scraping_completed.connect(self.on_scraping_completed)
|
||
t.error_occurred.connect(lambda m: self.log(f"ERROR: {m}"))
|
||
|
||
# ------------------------------------------------------------------- signals
|
||
|
||
def on_cloudflare_waiting(self, message):
|
||
self.status_label.setText("Waiting for Cloudflare…")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
||
self.log(f"⏳ {message}")
|
||
|
||
def on_cloudflare_passed(self, item_count):
|
||
self.log(f"✅ Page loaded ({item_count} items).")
|
||
self.status_label.setText("Scraping…")
|
||
|
||
def on_cloudflare_timeout(self, timeout):
|
||
self.log(f"⚠️ Timed out after {timeout}s waiting for the page/Cloudflare.")
|
||
|
||
def on_page_started(self, page_number, page_index, total_pages, url):
|
||
self.log(f"— Page {page_number} ({page_index}/{total_pages})")
|
||
if total_pages > 0:
|
||
self.progress_bar.setRange(0, total_pages)
|
||
self.progress_bar.setValue(page_index - 1)
|
||
|
||
def on_page_items_found(self, page_number, item_count):
|
||
self.log(f" Found {item_count} items on page {page_number}.")
|
||
|
||
def on_page_completed(self, page_number, items_processed):
|
||
self.log(f" Page {page_number} done ({items_processed} items).")
|
||
self.progress_bar.setValue(self.progress_bar.value() + 1)
|
||
|
||
def on_book_started(self, page_number, book_index, total_books, url):
|
||
self.status_label.setText(f"Page {page_number}: item {book_index}/{total_books}")
|
||
|
||
def on_book_completed(self, title, links_found):
|
||
label = title if title else "(untitled)"
|
||
self.log(f" • {label} — {links_found} 1fichier link(s)")
|
||
|
||
def on_link_found(self, url, total):
|
||
self.links_label.setText(f"Links found: {total}")
|
||
|
||
def on_scraping_completed(self, summary):
|
||
self.start_btn.setEnabled(True)
|
||
self.stop_btn.setEnabled(False)
|
||
self.progress_bar.setRange(0, 1)
|
||
self.progress_bar.setValue(1)
|
||
|
||
total = summary.get('total_links_found', 0)
|
||
if summary.get('cancelled'):
|
||
self.status_label.setText("Cancelled")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
||
elif summary.get('success'):
|
||
self.status_label.setText("Completed")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
||
else:
|
||
self.status_label.setText("Completed with errors")
|
||
self.status_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
||
|
||
self.log(f"Done. {total} link(s) written to {summary.get('output_file', '')}")
|
||
errors = summary.get('errors') or []
|
||
if errors:
|
||
self.log(f"{len(errors)} error(s) occurred:")
|
||
for e in errors[:10]:
|
||
self.log(f" ! {e}")
|
||
|
||
# --------------------------------------------------------------------- misc
|
||
|
||
def open_output_folder(self):
|
||
folder = Path(self.output_edit.text()).parent
|
||
try:
|
||
if sys.platform == "win32":
|
||
os.startfile(folder)
|
||
elif sys.platform == "darwin":
|
||
subprocess.run(["open", str(folder)])
|
||
else:
|
||
subprocess.run(["xdg-open", str(folder)])
|
||
except Exception as e:
|
||
QMessageBox.information(self, "Output Folder",
|
||
f"Links are saved to:\n{folder}\n\n"
|
||
f"Could not open folder automatically: {e}")
|
||
|
||
def reset_browser_profile(self):
|
||
"""Delete the persistent Chrome profile (forces a fresh Cloudflare pass)."""
|
||
if self.scraper_thread and self.scraper_thread.is_running():
|
||
QMessageBox.warning(self, "Scraping in Progress",
|
||
"Stop the current scrape before resetting the profile.")
|
||
return
|
||
|
||
reply = QMessageBox.question(
|
||
self, "Reset Browser Profile",
|
||
f"Delete the saved browser profile?\n\n{DEFAULT_PROFILE_DIR}\n\n"
|
||
"You'll need to pass the Cloudflare check once more on the next run. "
|
||
"Use this if the browser stops loading pages correctly.",
|
||
QMessageBox.Yes | QMessageBox.No)
|
||
|
||
if reply != QMessageBox.Yes:
|
||
return
|
||
|
||
import shutil
|
||
try:
|
||
if DEFAULT_PROFILE_DIR.exists():
|
||
shutil.rmtree(DEFAULT_PROFILE_DIR)
|
||
self.log("Browser profile reset.")
|
||
QMessageBox.information(self, "Profile Reset",
|
||
"The browser profile has been cleared.")
|
||
except Exception as e:
|
||
QMessageBox.warning(self, "Reset Failed", f"Could not reset profile:\n{e}")
|
||
|
||
def show_about(self):
|
||
QMessageBox.about(self, "About Bookys Scraper",
|
||
"Bookys Scraper\n\n"
|
||
"Harvests direct 1fichier download URLs from "
|
||
"bookys-ebooks.com and saves them to a .txt file.\n\n"
|
||
"• Visible browser (Cloudflare-friendly)\n"
|
||
"• Auto-detects when the challenge clears\n"
|
||
"• Ignores pop-up ads\n\n"
|
||
"Built with Python and PyQt5.")
|
||
|
||
def _switch_to_eboek(self):
|
||
app = QApplication.instance()
|
||
if hasattr(app, 'show_eboek'):
|
||
app.show_eboek()
|
||
|
||
def closeEvent(self, event):
|
||
if self.scraper_thread and self.scraper_thread.is_running():
|
||
reply = QMessageBox.question(
|
||
self, "Scraping in Progress",
|
||
"Scraping is running. Stop and continue?",
|
||
QMessageBox.Yes | QMessageBox.No)
|
||
if reply == QMessageBox.Yes:
|
||
self.scraper_thread.request_stop()
|
||
self.scraper_thread.wait(3000)
|
||
event.accept()
|
||
else:
|
||
event.ignore()
|
||
return
|
||
else:
|
||
self.save_settings()
|
||
event.accept()
|