ea4cab15c3
- Created `install_and_run.bat` for Windows installation and setup. - Created `install_and_run.sh` for Unix-based systems installation and setup. - Removed `main.py` as it is no longer needed. - Updated `requirements.txt` to specify package versions and added PyQt5. - Deleted `start.bat` as it is redundant. - Added unit tests for core functionality and scraping modes. - Implemented input validation utilities in `utils/validators.py`. - Added support for dual scraping modes in the scraper.
510 lines
20 KiB
Python
510 lines
20 KiB
Python
"""
|
|
Main application window for the EBoek.info scraper GUI.
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from PyQt5.QtWidgets import (
|
|
QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
|
|
QPushButton, QLabel, QSpinBox, QTextEdit, QGroupBox,
|
|
QCheckBox, QProgressBar, QMessageBox, QFileDialog, QMenuBar, QMenu, QAction,
|
|
QComboBox
|
|
)
|
|
from PyQt5.QtCore import Qt, QTimer, pyqtSignal
|
|
from PyQt5.QtGui import QFont, QIcon
|
|
|
|
# Import our custom modules
|
|
import os
|
|
import sys
|
|
|
|
# Add the project root directory to Python path so we can import our modules
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
from core.credentials import CredentialManager
|
|
from core.scraper_thread import ScraperThread
|
|
from utils.validators import validate_page_range, format_error_message
|
|
from gui.login_dialog import LoginDialog
|
|
from gui.progress_dialog import ProgressDialog
|
|
|
|
|
|
class MainWindow(QMainWindow):
|
|
"""
|
|
Main application window for the EBoek.info scraper.
|
|
|
|
This window provides the primary interface for:
|
|
- Managing credentials
|
|
- Setting scraping parameters
|
|
- Starting/stopping scraping operations
|
|
- Monitoring progress and logs
|
|
"""
|
|
|
|
# Custom signals
|
|
scraping_requested = pyqtSignal(str, str, int, int, bool) # username, password, start_page, end_page, headless
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.credential_manager = CredentialManager()
|
|
self.scraper_thread = None
|
|
self.progress_dialog = None
|
|
|
|
# Load application settings
|
|
self.app_settings = self.credential_manager.load_app_settings()
|
|
if not self.app_settings:
|
|
self.app_settings = self.credential_manager.get_default_settings()
|
|
|
|
self.init_ui()
|
|
self.update_credential_status()
|
|
|
|
def init_ui(self):
|
|
"""Initialize the user interface."""
|
|
self.setWindowTitle("EBoek.info Scraper")
|
|
self.setMinimumSize(600, 500)
|
|
self.resize(700, 600)
|
|
|
|
# Create menu bar
|
|
self.create_menu_bar()
|
|
|
|
# Create central widget
|
|
central_widget = QWidget()
|
|
self.setCentralWidget(central_widget)
|
|
|
|
# Main layout
|
|
layout = QVBoxLayout(central_widget)
|
|
|
|
# Create sections
|
|
self.create_credential_section(layout)
|
|
self.create_scraping_section(layout)
|
|
self.create_status_section(layout)
|
|
self.create_control_section(layout)
|
|
|
|
# Status bar
|
|
self.statusBar().showMessage("Ready")
|
|
|
|
def create_menu_bar(self):
|
|
"""Create the menu bar."""
|
|
menubar = self.menuBar()
|
|
|
|
# File menu
|
|
file_menu = menubar.addMenu('File')
|
|
|
|
export_action = QAction('Export Settings', self)
|
|
export_action.triggered.connect(self.export_settings)
|
|
file_menu.addAction(export_action)
|
|
|
|
import_action = QAction('Import Settings', self)
|
|
import_action.triggered.connect(self.import_settings)
|
|
file_menu.addAction(import_action)
|
|
|
|
file_menu.addSeparator()
|
|
|
|
exit_action = QAction('Exit', self)
|
|
exit_action.triggered.connect(self.close)
|
|
file_menu.addAction(exit_action)
|
|
|
|
# Settings menu
|
|
settings_menu = menubar.addMenu('Settings')
|
|
|
|
clear_creds_action = QAction('Clear Saved Credentials', self)
|
|
clear_creds_action.triggered.connect(self.clear_credentials)
|
|
settings_menu.addAction(clear_creds_action)
|
|
|
|
# Help menu
|
|
help_menu = menubar.addMenu('Help')
|
|
|
|
about_action = QAction('About', self)
|
|
about_action.triggered.connect(self.show_about)
|
|
help_menu.addAction(about_action)
|
|
|
|
def create_credential_section(self, parent_layout):
|
|
"""Create the credential management section."""
|
|
group = QGroupBox("Credentials")
|
|
layout = QHBoxLayout(group)
|
|
|
|
self.credential_status_label = QLabel("No credentials configured")
|
|
layout.addWidget(self.credential_status_label)
|
|
|
|
layout.addStretch()
|
|
|
|
self.change_credentials_btn = QPushButton("Change Credentials")
|
|
self.change_credentials_btn.clicked.connect(self.show_login_dialog)
|
|
layout.addWidget(self.change_credentials_btn)
|
|
|
|
parent_layout.addWidget(group)
|
|
|
|
def create_scraping_section(self, parent_layout):
|
|
"""Create the scraping configuration section."""
|
|
group = QGroupBox("Scraping Configuration")
|
|
layout = QGridLayout(group)
|
|
|
|
# Scraping mode selection
|
|
layout.addWidget(QLabel("Mode:"), 0, 0)
|
|
self.mode_combo = QComboBox()
|
|
self.mode_combo.addItems([
|
|
"All Comics (stripverhalen-alle)",
|
|
"Latest Comics (laatste)"
|
|
])
|
|
self.mode_combo.setCurrentIndex(self.app_settings.get('scraping_mode', 0))
|
|
self.mode_combo.setToolTip("Select which page type to scrape")
|
|
self.mode_combo.currentIndexChanged.connect(self.on_mode_changed)
|
|
layout.addWidget(self.mode_combo, 0, 1, 1, 3)
|
|
|
|
# Page range selection
|
|
layout.addWidget(QLabel("Start Page:"), 1, 0)
|
|
self.start_page_spin = QSpinBox()
|
|
self.start_page_spin.setMinimum(1)
|
|
self.start_page_spin.setMaximum(9999)
|
|
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
|
layout.addWidget(self.start_page_spin, 1, 1)
|
|
|
|
layout.addWidget(QLabel("End Page:"), 1, 2)
|
|
self.end_page_spin = QSpinBox()
|
|
self.end_page_spin.setMinimum(1)
|
|
self.end_page_spin.setMaximum(9999)
|
|
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
|
layout.addWidget(self.end_page_spin, 1, 3)
|
|
|
|
# Mode description label
|
|
self.mode_description_label = QLabel("")
|
|
self.mode_description_label.setStyleSheet("color: #666; font-size: 11px; font-style: italic;")
|
|
self.mode_description_label.setWordWrap(True)
|
|
layout.addWidget(self.mode_description_label, 2, 0, 1, 4)
|
|
|
|
# Options
|
|
self.headless_checkbox = QCheckBox("Headless Mode")
|
|
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
|
self.headless_checkbox.setToolTip("Run browser in background (recommended)")
|
|
layout.addWidget(self.headless_checkbox, 3, 0, 1, 2)
|
|
|
|
self.verbose_checkbox = QCheckBox("Verbose Logging")
|
|
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
|
self.verbose_checkbox.setToolTip("Show detailed progress information")
|
|
layout.addWidget(self.verbose_checkbox, 3, 2, 1, 2)
|
|
|
|
# Update mode description
|
|
self.update_mode_description()
|
|
|
|
parent_layout.addWidget(group)
|
|
|
|
def on_mode_changed(self):
|
|
"""Handle scraping mode selection change."""
|
|
self.update_mode_description()
|
|
self.save_current_settings()
|
|
|
|
def update_mode_description(self):
|
|
"""Update the mode description text based on current selection."""
|
|
mode_index = self.mode_combo.currentIndex()
|
|
|
|
if mode_index == 0: # All Comics
|
|
description = ("Scrapes all comics from the 'stripverhalen-alle' page. "
|
|
"This is the original scraping mode with complete comic archives.")
|
|
elif mode_index == 1: # Latest Comics
|
|
description = ("Scrapes latest comics from the 'laatste' page. "
|
|
"This mode gets the most recently added comics with page parameter support.")
|
|
else:
|
|
description = ""
|
|
|
|
self.mode_description_label.setText(description)
|
|
|
|
def create_status_section(self, parent_layout):
|
|
"""Create the status display section."""
|
|
group = QGroupBox("Status")
|
|
layout = QVBoxLayout(group)
|
|
|
|
self.status_label = QLabel("Ready to start scraping...")
|
|
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
|
layout.addWidget(self.status_label)
|
|
|
|
# Progress bar
|
|
self.progress_bar = QProgressBar()
|
|
self.progress_bar.setVisible(False)
|
|
layout.addWidget(self.progress_bar)
|
|
|
|
parent_layout.addWidget(group)
|
|
|
|
|
|
def create_control_section(self, parent_layout):
|
|
"""Create the control buttons section."""
|
|
layout = QHBoxLayout()
|
|
|
|
self.start_btn = QPushButton("Start Scraping")
|
|
self.start_btn.clicked.connect(self.start_scraping)
|
|
self.start_btn.setStyleSheet("QPushButton { background-color: #4CAF50; color: white; font-weight: bold; padding: 8px; }")
|
|
layout.addWidget(self.start_btn)
|
|
|
|
layout.addStretch()
|
|
|
|
self.downloads_btn = QPushButton("Open Downloads Folder")
|
|
self.downloads_btn.clicked.connect(self.open_downloads_folder)
|
|
layout.addWidget(self.downloads_btn)
|
|
|
|
parent_layout.addLayout(layout)
|
|
|
|
def update_credential_status(self):
|
|
"""Update the credential status display."""
|
|
username = self.credential_manager.get_saved_username()
|
|
if username:
|
|
self.credential_status_label.setText(f"Logged in as: {username}")
|
|
self.credential_status_label.setStyleSheet("color: #2E8B57; font-weight: bold;")
|
|
else:
|
|
self.credential_status_label.setText("No credentials configured")
|
|
self.credential_status_label.setStyleSheet("color: #FF6B35; font-weight: bold;")
|
|
|
|
def show_login_dialog(self):
|
|
"""Show the login dialog for credential input."""
|
|
dialog = LoginDialog(self, self.credential_manager)
|
|
if dialog.exec_() == dialog.Accepted:
|
|
self.update_credential_status()
|
|
self.log_message("Credentials updated successfully.")
|
|
|
|
def start_scraping(self):
|
|
"""Start the scraping process."""
|
|
# Validate credentials
|
|
credentials = self.credential_manager.load_credentials()
|
|
if not credentials:
|
|
QMessageBox.warning(self, "No Credentials",
|
|
"Please configure your EBoek.info credentials first.")
|
|
self.show_login_dialog()
|
|
return
|
|
|
|
# Validate page range
|
|
start_page = self.start_page_spin.value()
|
|
end_page = self.end_page_spin.value()
|
|
|
|
validation = validate_page_range(start_page, end_page)
|
|
if not validation['valid']:
|
|
QMessageBox.warning(self, "Invalid Page Range",
|
|
format_error_message(validation['errors']))
|
|
return
|
|
|
|
# Save current settings
|
|
self.save_current_settings()
|
|
|
|
# Get scraping mode
|
|
mode_index = self.mode_combo.currentIndex()
|
|
mode_names = ["All Comics", "Latest Comics"]
|
|
mode_name = mode_names[mode_index] if mode_index < len(mode_names) else "Unknown"
|
|
|
|
# Start scraping
|
|
self.log_message(f"Starting scraping: {mode_name} mode, pages {start_page} to {end_page}")
|
|
|
|
# Create and start scraper thread
|
|
self.scraper_thread = ScraperThread(
|
|
username=credentials['username'],
|
|
password=credentials['password'],
|
|
start_page=start_page,
|
|
end_page=end_page,
|
|
scraping_mode=mode_index,
|
|
headless=self.headless_checkbox.isChecked()
|
|
)
|
|
|
|
# Connect signals
|
|
self.connect_scraper_signals()
|
|
|
|
# Show progress dialog
|
|
self.progress_dialog = ProgressDialog(self, self.scraper_thread)
|
|
self.progress_dialog.show()
|
|
|
|
# Start the thread
|
|
self.scraper_thread.start()
|
|
|
|
# Update UI state
|
|
self.start_btn.setEnabled(False)
|
|
self.status_label.setText("Scraping in progress...")
|
|
self.status_label.setStyleSheet("font-weight: bold; color: #FF8C00;")
|
|
|
|
def connect_scraper_signals(self):
|
|
"""Connect signals from the scraper thread to UI updates."""
|
|
if not self.scraper_thread:
|
|
return
|
|
|
|
# Login signals
|
|
self.scraper_thread.login_started.connect(self.on_login_started)
|
|
self.scraper_thread.login_success.connect(self.on_login_success)
|
|
self.scraper_thread.login_failed.connect(self.on_login_failed)
|
|
|
|
# Scraping completion
|
|
self.scraper_thread.scraping_completed.connect(self.on_scraping_completed)
|
|
|
|
# Status updates
|
|
self.scraper_thread.status_update.connect(self.log_message)
|
|
self.scraper_thread.error_occurred.connect(self.on_error_occurred)
|
|
|
|
# Page progress
|
|
self.scraper_thread.page_started.connect(self.on_page_started)
|
|
self.scraper_thread.page_completed.connect(self.on_page_completed)
|
|
|
|
def on_login_started(self, username):
|
|
"""Handle login started event."""
|
|
self.log_message(f"Logging in as {username}...")
|
|
|
|
def on_login_success(self, username):
|
|
"""Handle successful login."""
|
|
self.log_message(f"Login successful for {username}")
|
|
|
|
def on_login_failed(self, username, error):
|
|
"""Handle failed login."""
|
|
self.log_message(f"Login failed for {username}: {error}")
|
|
QMessageBox.critical(self, "Login Failed",
|
|
f"Could not log in as {username}.\n\n{error}\n\nPlease check your credentials.")
|
|
|
|
def on_page_started(self, page_number, page_index, total_pages, url):
|
|
"""Handle page started event."""
|
|
self.log_message(f"Processing page {page_number} ({page_index}/{total_pages})")
|
|
|
|
def on_page_completed(self, page_number, comics_processed):
|
|
"""Handle page completed event."""
|
|
self.log_message(f"Page {page_number} completed - {comics_processed} comics processed")
|
|
|
|
def on_scraping_completed(self, summary):
|
|
"""Handle scraping completion."""
|
|
self.start_btn.setEnabled(True)
|
|
|
|
if summary.get('cancelled'):
|
|
self.status_label.setText("Scraping cancelled")
|
|
self.status_label.setStyleSheet("font-weight: bold; color: #FF6B35;")
|
|
self.log_message("Scraping was cancelled by user")
|
|
elif summary.get('success'):
|
|
self.status_label.setText("Scraping completed")
|
|
self.status_label.setStyleSheet("font-weight: bold; color: #2E8B57;")
|
|
self.log_message(f"Scraping completed! Processed {summary.get('total_comics_processed', 0)} comics, "
|
|
f"triggered {summary.get('total_downloads_triggered', 0)} downloads")
|
|
else:
|
|
self.status_label.setText("Scraping failed")
|
|
self.status_label.setStyleSheet("font-weight: bold; color: #f44336;")
|
|
self.log_message("Scraping failed - see errors above")
|
|
|
|
# Show summary
|
|
if summary.get('errors'):
|
|
error_count = len(summary['errors'])
|
|
QMessageBox.warning(self, "Scraping Completed with Errors",
|
|
f"Scraping completed but {error_count} errors occurred.\n"
|
|
f"Check the log for details.")
|
|
|
|
def on_error_occurred(self, error_message):
|
|
"""Handle error events."""
|
|
self.log_message(f"ERROR: {error_message}")
|
|
|
|
def log_message(self, message):
|
|
"""Log message - removed from main window, only shown in progress dialog."""
|
|
# Activity log now only appears in the scraping progress dialog
|
|
pass
|
|
|
|
def save_current_settings(self):
|
|
"""Save current UI settings to configuration."""
|
|
settings = {
|
|
'headless_mode': self.headless_checkbox.isChecked(),
|
|
'verbose_logging': self.verbose_checkbox.isChecked(),
|
|
'default_start_page': self.start_page_spin.value(),
|
|
'default_end_page': self.end_page_spin.value(),
|
|
'scraping_mode': self.mode_combo.currentIndex(),
|
|
}
|
|
settings.update(self.app_settings) # Keep other settings
|
|
self.credential_manager.save_app_settings(settings)
|
|
self.app_settings = settings
|
|
|
|
def open_downloads_folder(self):
|
|
"""Open the downloads folder in the system file manager."""
|
|
downloads_path = Path.home() / "Downloads"
|
|
|
|
import sys
|
|
import subprocess
|
|
|
|
try:
|
|
if sys.platform == "win32":
|
|
os.startfile(downloads_path)
|
|
elif sys.platform == "darwin":
|
|
subprocess.run(["open", str(downloads_path)])
|
|
else: # linux
|
|
subprocess.run(["xdg-open", str(downloads_path)])
|
|
except Exception as e:
|
|
QMessageBox.information(self, "Downloads Folder",
|
|
f"Downloads are saved to:\n{downloads_path}\n\n"
|
|
f"Could not open folder automatically: {e}")
|
|
|
|
def clear_credentials(self):
|
|
"""Clear saved credentials."""
|
|
reply = QMessageBox.question(self, "Clear Credentials",
|
|
"Are you sure you want to clear the saved credentials?",
|
|
QMessageBox.Yes | QMessageBox.No)
|
|
|
|
if reply == QMessageBox.Yes:
|
|
if self.credential_manager.clear_credentials():
|
|
self.update_credential_status()
|
|
self.log_message("Saved credentials cleared.")
|
|
QMessageBox.information(self, "Credentials Cleared",
|
|
"Saved credentials have been cleared.")
|
|
else:
|
|
QMessageBox.warning(self, "Error", "Could not clear credentials.")
|
|
|
|
def export_settings(self):
|
|
"""Export application settings to a file."""
|
|
file_path, _ = QFileDialog.getSaveFileName(
|
|
self, "Export Settings",
|
|
"eboek_scraper_settings.json",
|
|
"JSON files (*.json);;All files (*.*)"
|
|
)
|
|
|
|
if file_path:
|
|
if self.credential_manager.export_settings(file_path):
|
|
QMessageBox.information(self, "Export Successful",
|
|
f"Settings exported to:\n{file_path}")
|
|
else:
|
|
QMessageBox.warning(self, "Export Failed",
|
|
"Could not export settings.")
|
|
|
|
def import_settings(self):
|
|
"""Import application settings from a file."""
|
|
file_path, _ = QFileDialog.getOpenFileName(
|
|
self, "Import Settings",
|
|
"",
|
|
"JSON files (*.json);;All files (*.*)"
|
|
)
|
|
|
|
if file_path:
|
|
if self.credential_manager.import_settings(file_path):
|
|
self.app_settings = self.credential_manager.load_app_settings()
|
|
# Update UI with imported settings
|
|
self.headless_checkbox.setChecked(self.app_settings.get('headless_mode', True))
|
|
self.verbose_checkbox.setChecked(self.app_settings.get('verbose_logging', False))
|
|
self.start_page_spin.setValue(self.app_settings.get('default_start_page', 1))
|
|
self.end_page_spin.setValue(self.app_settings.get('default_end_page', 1))
|
|
|
|
QMessageBox.information(self, "Import Successful",
|
|
f"Settings imported from:\n{file_path}")
|
|
self.log_message("Settings imported successfully.")
|
|
else:
|
|
QMessageBox.warning(self, "Import Failed",
|
|
"Could not import settings.")
|
|
|
|
def show_about(self):
|
|
"""Show the about dialog."""
|
|
QMessageBox.about(self, "About EBoek.info Scraper",
|
|
"EBoek.info Scraper\n\n"
|
|
"A GUI application for downloading comic strips from eboek.info.\n\n"
|
|
"Features:\n"
|
|
"• Automated login and scraping\n"
|
|
"• Real-time progress monitoring\n"
|
|
"• Human-like behavior simulation\n"
|
|
"• Secure credential storage\n\n"
|
|
"Built with Python and PyQt5.")
|
|
|
|
def closeEvent(self, event):
|
|
"""Handle application close event."""
|
|
if self.scraper_thread and self.scraper_thread.isRunning():
|
|
reply = QMessageBox.question(self, "Scraping in Progress",
|
|
"Scraping is currently in progress. "
|
|
"Do you want to stop and exit?",
|
|
QMessageBox.Yes | QMessageBox.No)
|
|
|
|
if reply == QMessageBox.Yes:
|
|
self.scraper_thread.request_stop()
|
|
# Give it a moment to stop gracefully
|
|
self.scraper_thread.wait(3000) # Wait up to 3 seconds
|
|
event.accept()
|
|
else:
|
|
event.ignore()
|
|
else:
|
|
# Save settings before closing
|
|
self.save_current_settings()
|
|
event.accept() |