import javax.swing.*;
import javax.swing.border.EmptyBorder;
import javax.swing.text.DefaultCaret;
import java.awt.*;
import java.awt.datatransfer.StringSelection;
import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.List;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.function.Consumer;
import java.util.regex.*;

/**
 * Magnetic Pagination - crawler + link processor in ONE Swing application.
 * Java 8 compatible, no external dependencies.
 *
 * What it does
 * ------------
 * The crawler part fetches the start URLs (and, depending on "Max depth", the
 * pages they link to) and collects magnet links and/or http(s) URLs.
 *
 * The moment a NEW link is found (one that is not already stored), it goes
 * through the whole processing pipeline automatically:
 *
 *   1) it is appended to links.txt (links_1.txt, links_2.txt, ... once a file
 *      reaches 10 MB). Links are stored in discovery order, so the id of a
 *      link is simply its position (line number) across those files;
 *   2) it is saved as id_files/<id>.txt (the file content is the link itself);
 *   3) its categories are computed and the id is added to the HTML page of
 *      every category (html_pages/<category>.html). When a page would grow
 *      past the configured size, ids continue on <category>_2.html,
 *      <category>_3.html, ... and every page of that category gets
 *      Previous/Next buttons and a "Page X of Y" indicator;
 *   4) index.html (landing page: first 5 categories + search box) is created
 *      and kept up to date.
 *
 * Modes (Options... button)
 * -------------------------
 *   - simple:   ids are shown as "Id N" labels that open id_files/N.txt.
 *               Categories are whole words.
 *   - full:     (DEFAULT) the page fetches id_files/N.txt at runtime and shows
 *               / opens the real link. Categories are whole words. Needs the
 *               pages to be served over http(s), e.g. "python -m http.server",
 *               because browsers block fetch() on file:// pages.
 *   - advanced: like full, but categories are every 3..26 letter substring of
 *               each word (many more pages).
 *
 * The Options dialog also sets the maximum size (KB) of every generated HTML
 * page. Changing mode/size can make already generated pages inconsistent, so
 * the program offers to rebuild them; "Rebuild HTML..." does the same on
 * demand (this is what the old stand-alone LinkProcessor did, and it also
 * indexes links.txt files created before the merge).
 *
 * Output (relative to the working directory):
 *   links.txt, links_1.txt, ...   sites.txt, sites_1.txt, ...
 *   index.html
 *   id_files/1.txt, 2.txt, ...
 *   html_pages/common.js, common.css, <category>.html, <category>_2.html, ...
 *   crawler-settings.properties   (remembers the mode and page size)
 */
public class CrawlerGUI extends JFrame {

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new Runnable() {
            @Override
            public void run() {
                try {
                    UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
                } catch (Exception ignored) {
                    // fall back to default look and feel
                }
                new CrawlerGUI().setVisible(true);
            }
        });
    }

    // ---------------- UI fields ----------------
    private final JTextArea urlsArea = new JTextArea();
    private final JCheckBox magnetCheck = new JCheckBox("Collect magnet: links", true);
    private final JCheckBox urlsCheck = new JCheckBox("Collect valid http(s) URLs", true);
    private final JSpinner depthSpinner = new JSpinner(new SpinnerNumberModel(0, 0, 50, 1));
    private final JSpinner timeoutSpinner = new JSpinner(new SpinnerNumberModel(8, 1, 120, 1));
    private final JLabel htmlInfoLabel = new JLabel();
    private final JButton optionsButton = new JButton("Options...");
    private final JButton rebuildButton = new JButton("Rebuild HTML...");
    private final JButton loadUrlsButton = new JButton("Load urls.txt...");
    private final JButton startButton = new JButton("Start Crawl");
    private final JButton stopButton = new JButton("Stop");
    private final JButton clearLogButton = new JButton("Clear Log");
    private final JButton openFolderButton = new JButton("Open Output Folder");
    private final JTextArea logArea = new JTextArea();
    private final DefaultListModel<String> resultsModel = new DefaultListModel<String>();
    private final JList<String> resultsList = new JList<String>(resultsModel);
    private final JTabbedPane tabs = new JTabbedPane();
    private final JProgressBar progressBar = new JProgressBar();
    private final JLabel statusLabel = new JLabel("Idle.");

    /** Current HTML settings (mode + page size). Only touched on the EDT. */
    private IndexSettings settings = IndexSettings.load();
    private JobWorker activeWorker;

    public CrawlerGUI() {
        super("Simple Crawler");
        setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        setSize(980, 700);
        setMinimumSize(new Dimension(760, 500));
        setLocationRelativeTo(null);

        setLayout(new BorderLayout(8, 8));
        add(buildTopPanel(), BorderLayout.NORTH);
        add(buildCenterPanel(), BorderLayout.CENTER);
        add(buildBottomPanel(), BorderLayout.SOUTH);

        wireActions();
        refreshHtmlInfo();
    }

    private JComponent buildTopPanel() {
        JPanel panel = new JPanel();
        panel.setBorder(new EmptyBorder(10, 10, 0, 10));
        panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS));

        JLabel title = new JLabel("Magnetic Pagination");
        title.setFont(title.getFont().deriveFont(Font.BOLD, 18f));
        title.setAlignmentX(Component.LEFT_ALIGNMENT);

        JLabel subtitle = new JLabel("Link crawler + HTML index generator - zero external dependencies");
        subtitle.setForeground(Color.GRAY);
        subtitle.setAlignmentX(Component.LEFT_ALIGNMENT);

        JPanel options = new JPanel(new FlowLayout(FlowLayout.LEFT, 12, 6));
        options.add(magnetCheck);
        options.add(urlsCheck);
        options.add(new JLabel("Max depth:"));
        depthSpinner.setPreferredSize(new Dimension(60, 24));
        options.add(depthSpinner);
        options.add(new JLabel("Timeout (s):"));
        timeoutSpinner.setPreferredSize(new Dimension(60, 24));
        options.add(timeoutSpinner);
        options.setAlignmentX(Component.LEFT_ALIGNMENT);

        JPanel htmlRow = new JPanel(new FlowLayout(FlowLayout.LEFT, 12, 0));
        htmlRow.add(new JLabel("HTML pages:"));
        htmlInfoLabel.setFont(htmlInfoLabel.getFont().deriveFont(Font.BOLD));
        htmlRow.add(htmlInfoLabel);
        htmlRow.add(optionsButton);
        htmlRow.add(rebuildButton);
        htmlRow.setAlignmentX(Component.LEFT_ALIGNMENT);

        panel.add(title);
        panel.add(subtitle);
        panel.add(Box.createVerticalStrut(6));
        panel.add(options);
        panel.add(htmlRow);
        return panel;
    }

    private JComponent buildCenterPanel() {
        // Left: start URLs
        urlsArea.setLineWrap(false);
        urlsArea.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
        JScrollPane urlsScroll = new JScrollPane(urlsArea);
        urlsScroll.setBorder(BorderFactory.createTitledBorder("Start URLs (one per line)"));

        JPanel urlsButtons = new JPanel(new FlowLayout(FlowLayout.LEFT));
        urlsButtons.add(loadUrlsButton);

        JPanel leftPanel = new JPanel(new BorderLayout());
        leftPanel.add(urlsScroll, BorderLayout.CENTER);
        leftPanel.add(urlsButtons, BorderLayout.SOUTH);

        // Right: tabbed pane with Log and Found Links
        logArea.setEditable(false);
        logArea.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
        DefaultCaret caret = (DefaultCaret) logArea.getCaret();
        caret.setUpdatePolicy(DefaultCaret.ALWAYS_UPDATE);
        JScrollPane logScroll = new JScrollPane(logArea);

        resultsList.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
        JScrollPane resultsScroll = new JScrollPane(resultsList);
        JPanel resultsPanel = new JPanel(new BorderLayout());
        resultsPanel.add(resultsScroll, BorderLayout.CENTER);
        JButton copyAllButton = new JButton("Copy All Found Links");
        copyAllButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                StringBuilder sb = new StringBuilder();
                for (int i = 0; i < resultsModel.size(); i++) {
                    sb.append(resultsModel.get(i)).append('\n');
                }
                StringSelection sel = new StringSelection(sb.toString());
                Toolkit.getDefaultToolkit().getSystemClipboard().setContents(sel, null);
                statusLabel.setText("Copied " + resultsModel.size() + " link(s) to clipboard.");
            }
        });
        JPanel resultsBottom = new JPanel(new FlowLayout(FlowLayout.LEFT));
        resultsBottom.add(copyAllButton);
        resultsPanel.add(resultsBottom, BorderLayout.SOUTH);

        tabs.addTab("Log", logScroll);
        tabs.addTab("Found Links", resultsPanel);

        JSplitPane split = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, leftPanel, tabs);
        split.setResizeWeight(0.35);
        split.setBorder(new EmptyBorder(6, 10, 0, 10));
        return split;
    }

    private JComponent buildBottomPanel() {
        JPanel panel = new JPanel(new BorderLayout());
        panel.setBorder(new EmptyBorder(6, 10, 10, 10));

        JPanel buttons = new JPanel(new FlowLayout(FlowLayout.LEFT));
        stopButton.setEnabled(false);
        buttons.add(startButton);
        buttons.add(stopButton);
        buttons.add(clearLogButton);
        buttons.add(openFolderButton);

        progressBar.setIndeterminate(false);
        progressBar.setPreferredSize(new Dimension(160, 20));

        JPanel statusPanel = new JPanel(new BorderLayout(8, 0));
        statusPanel.add(statusLabel, BorderLayout.CENTER);
        statusPanel.add(progressBar, BorderLayout.EAST);

        panel.add(buttons, BorderLayout.WEST);
        panel.add(statusPanel, BorderLayout.SOUTH);
        return panel;
    }

    private void wireActions() {
        loadUrlsButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                JFileChooser chooser = new JFileChooser();
                chooser.setDialogTitle("Load urls.txt");
                if (chooser.showOpenDialog(CrawlerGUI.this) == JFileChooser.APPROVE_OPTION) {
                    File f = chooser.getSelectedFile();
                    try {
                        List<String> lines = Files.readAllLines(f.toPath(), StandardCharsets.UTF_8);
                        urlsArea.setText(joinLines(lines));
                    } catch (IOException ex) {
                        JOptionPane.showMessageDialog(CrawlerGUI.this,
                                "Could not read file: " + ex.getMessage(),
                                "Error", JOptionPane.ERROR_MESSAGE);
                    }
                }
            }
        });

        clearLogButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                logArea.setText("");
            }
        });

        openFolderButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                try {
                    File dir = new File(".").getAbsoluteFile().getParentFile();
                    if (Desktop.isDesktopSupported()) {
                        Desktop.getDesktop().open(dir);
                    } else {
                        JOptionPane.showMessageDialog(CrawlerGUI.this,
                                "Output folder: " + dir.getAbsolutePath());
                    }
                } catch (Exception ex) {
                    JOptionPane.showMessageDialog(CrawlerGUI.this,
                            "Could not open folder: " + ex.getMessage(),
                            "Error", JOptionPane.ERROR_MESSAGE);
                }
            }
        });

        optionsButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                showOptionsDialog();
            }
        });

        rebuildButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                startRebuild(true);
            }
        });

        startButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                startCrawl();
            }
        });

        stopButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                if (activeWorker != null) {
                    activeWorker.requestStop();
                    statusLabel.setText("Stopping...");
                    stopButton.setEnabled(false);
                }
            }
        });
    }

    private static String joinLines(List<String> lines) {
        StringBuilder sb = new StringBuilder();
        for (String l : lines) {
            sb.append(l).append('\n');
        }
        return sb.toString();
    }

    private void refreshHtmlInfo() {
        htmlInfoLabel.setText(settings.describe());
    }

    /** Enables/disables the controls that must not be used while a job is running. */
    private void setBusy(boolean busy) {
        startButton.setEnabled(!busy);
        loadUrlsButton.setEnabled(!busy);
        optionsButton.setEnabled(!busy);
        rebuildButton.setEnabled(!busy);
        stopButton.setEnabled(busy);
        progressBar.setIndeterminate(busy);
    }

    // ------------------------------------------------------------------
    // Options dialog (mode + page size)
    // ------------------------------------------------------------------

    private void showOptionsDialog() {
        final JDialog dialog = new JDialog(this, "HTML Options", true);
        final OptionsPanel optionsPanel = new OptionsPanel(settings);
        final IndexSettings[] result = new IndexSettings[1];

        final JButton ok = new JButton("OK");
        JButton cancel = new JButton("Cancel");
        ok.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                result[0] = optionsPanel.readSettings();
                dialog.dispose();
            }
        });
        cancel.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                dialog.dispose();
            }
        });
        JPanel buttons = new JPanel(new FlowLayout(FlowLayout.RIGHT));
        buttons.add(ok);
        buttons.add(cancel);
        buttons.setAlignmentX(Component.LEFT_ALIGNMENT);

        JPanel content = new JPanel();
        content.setLayout(new BoxLayout(content, BoxLayout.Y_AXIS));
        content.setBorder(new EmptyBorder(12, 12, 8, 12));
        optionsPanel.setAlignmentX(Component.LEFT_ALIGNMENT);
        content.add(optionsPanel);
        content.add(Box.createVerticalStrut(4));
        content.add(buttons);

        dialog.setContentPane(content);
        dialog.getRootPane().setDefaultButton(ok);
        dialog.pack();
        dialog.setLocationRelativeTo(this);
        dialog.setVisible(true); // modal: returns when the dialog is closed

        if (result[0] != null && !result[0].equals(settings)) {
            applyNewSettings(result[0]);
        }
    }

    private void applyNewSettings(IndexSettings updated) {
        if (LinkStore.hasAny()) {
            Object[] choices = {"Rebuild now", "Only new links", "Cancel"};
            int choice = JOptionPane.showOptionDialog(this,
                    "<html>New settings: <b>" + updated.describe() + "</b><br><br>"
                            + "Links that were already processed have pages built with the old settings.<br>"
                            + "<b>Rebuild now</b> regenerates id_files and every page in html_pages so everything matches.<br>"
                            + "<b>Only new links</b> keeps the existing pages and applies the new settings from now on.</html>",
                    "Apply HTML settings", JOptionPane.DEFAULT_OPTION, JOptionPane.QUESTION_MESSAGE,
                    null, choices, choices[0]);
            if (choice == 2 || choice == JOptionPane.CLOSED_OPTION) {
                return; // cancelled: keep the old settings
            }
            settings = updated;
            settings.save();
            refreshHtmlInfo();
            if (choice == 0) {
                startRebuild(false);
            }
        } else {
            settings = updated;
            settings.save();
            refreshHtmlInfo();
        }
    }

    // ------------------------------------------------------------------
    // Starting / finishing jobs
    // ------------------------------------------------------------------

    private void startCrawl() {
        List<String> startUrls = new ArrayList<String>();
        for (String line : urlsArea.getText().split("\\r?\\n")) {
            String trimmed = line.trim();
            if (!trimmed.isEmpty()) {
                startUrls.add(trimmed);
            }
        }

        if (startUrls.isEmpty()) {
            String entered = JOptionPane.showInputDialog(this,
                    "Enter start URL (e.g. https://example.com):", "No URLs provided",
                    JOptionPane.QUESTION_MESSAGE);
            if (entered == null) {
                return;
            }
            entered = entered.trim();
            if (entered.isEmpty() || !(entered.startsWith("http://") || entered.startsWith("https://"))) {
                JOptionPane.showMessageDialog(this,
                        "URL must start with http:// or https://", "Invalid URL",
                        JOptionPane.ERROR_MESSAGE);
                return;
            }
            startUrls.add(entered);
            urlsArea.setText(entered);
        }

        if (!magnetCheck.isSelected() && !urlsCheck.isSelected()) {
            JOptionPane.showMessageDialog(this,
                    "Enable at least one of \"Collect magnet links\" or \"Collect valid URLs\".",
                    "Nothing to collect", JOptionPane.WARNING_MESSAGE);
            return;
        }

        tabs.setSelectedIndex(0);
        logArea.setText("");
        resultsModel.clear();
        setBusy(true);
        statusLabel.setText("Crawling...");

        boolean collectMagnets = magnetCheck.isSelected();
        boolean collectUrls = urlsCheck.isSelected();
        int maxDepth = (Integer) depthSpinner.getValue();
        int timeoutMs = ((Integer) timeoutSpinner.getValue()) * 1000;

        activeWorker = new JobWorker(settings, startUrls, collectMagnets, collectUrls, maxDepth, timeoutMs);
        activeWorker.execute();
    }

    private void startRebuild(boolean askConfirmation) {
        if (!LinkStore.hasAny()) {
            JOptionPane.showMessageDialog(this,
                    "There are no saved links yet (links.txt is missing or empty).",
                    "Nothing to rebuild", JOptionPane.INFORMATION_MESSAGE);
            return;
        }
        if (askConfirmation) {
            int answer = JOptionPane.showConfirmDialog(this,
                    "<html>This regenerates id_files and <b>replaces every .html page in html_pages</b><br>"
                            + "from links*.txt using: <b>" + settings.describe() + "</b>.<br>"
                            + "The links files themselves are not modified.<br><br>Continue?</html>",
                    "Rebuild HTML", JOptionPane.YES_NO_OPTION, JOptionPane.WARNING_MESSAGE);
            if (answer != JOptionPane.YES_OPTION) {
                return;
            }
        }
        tabs.setSelectedIndex(0);
        logArea.setText("");
        setBusy(true);
        statusLabel.setText("Rebuilding HTML...");

        activeWorker = new JobWorker(settings);
        activeWorker.execute();
    }

    private void onJobFinished(boolean rebuild, boolean stopped) {
        setBusy(false);
        if (stopped) {
            statusLabel.setText("Stopped by user.");
        } else {
            statusLabel.setText(rebuild ? "Rebuild complete." : "Crawl complete.");
        }
        activeWorker = null;
    }

    /**
     * Carries one (kind, value) pair from the background thread to the EDT.
     * A small typed holder is used instead of passing an Object[] to publish():
     * publish() is varargs, so an Object[] argument would be spread into
     * separate chunks and the cast in process() would fail silently on the EDT.
     * (It has to be a static member of the outer class: CrawlWorker-style inner
     * classes cannot declare static classes before Java 16.)
     */
    private static final class Chunk {
        final int kind;
        final String value;
        Chunk(int kind, String value) {
            this.kind = kind;
            this.value = value;
        }
    }

    /**
     * Runs a job (crawl or HTML rebuild) off the Event Dispatch Thread and
     * streams log lines / found links back into the GUI.
     */
    private class JobWorker extends SwingWorker<Void, Chunk> {
        private static final int LOG = 0;
        private static final int LINK = 1;

        private final boolean rebuild;
        private final IndexSettings jobSettings;
        private final List<String> startUrls;
        private final boolean collectMagnets;
        private final boolean collectUrls;
        private final int maxDepth;
        private final int timeoutMs;
        private volatile boolean stopRequested = false;

        /** Crawl job. */
        JobWorker(IndexSettings jobSettings, List<String> startUrls, boolean collectMagnets,
                  boolean collectUrls, int maxDepth, int timeoutMs) {
            this.rebuild = false;
            this.jobSettings = jobSettings;
            this.startUrls = startUrls;
            this.collectMagnets = collectMagnets;
            this.collectUrls = collectUrls;
            this.maxDepth = maxDepth;
            this.timeoutMs = timeoutMs;
        }

        /** Rebuild-HTML job. */
        JobWorker(IndexSettings jobSettings) {
            this.rebuild = true;
            this.jobSettings = jobSettings;
            this.startUrls = Collections.<String>emptyList();
            this.collectMagnets = false;
            this.collectUrls = false;
            this.maxDepth = 0;
            this.timeoutMs = 0;
        }

        void requestStop() {
            stopRequested = true;
        }

        private void logLine(String line) {
            publish(new Chunk(LOG, line));
        }

        private final Callable<Boolean> cancelSupplier = new Callable<Boolean>() {
            @Override
            public Boolean call() {
                return stopRequested || isCancelled();
            }
        };

        private final Consumer<String> logger = new Consumer<String>() {
            @Override
            public void accept(String line) {
                logLine(line);
            }
        };

        @Override
        protected Void doInBackground() {
            try {
                if (rebuild) {
                    runRebuild();
                } else {
                    runCrawl();
                }
            } catch (Throwable t) {
                // Never let a background failure vanish silently.
                logLine("[!] Unexpected error: " + t);
            }
            return null;
        }

        private void runCrawl() {
            HtmlIndexer indexer = new HtmlIndexer(Paths.get(""), jobSettings);
            CrawlerEngine engine = new CrawlerEngine(collectMagnets, collectUrls, maxDepth, timeoutMs,
                    indexer, jobSettings);
            engine.setLogger(logger);
            engine.setLinkListener(new LinkListener() {
                @Override
                public void onLink(int id, String link) {
                    publish(new Chunk(LINK, link));
                }
            });
            engine.setCancelSupplier(cancelSupplier);
            engine.crawl(startUrls);
        }

        private void runRebuild() {
            logLine("==============================================");
            logLine("  Rebuild HTML - " + jobSettings.describe());
            logLine("==============================================");
            try {
                List<String> lines = LinkStore.readAll();
                logLine("Read " + lines.size() + " link(s) from links*.txt");
                if (lines.isEmpty()) {
                    logLine("Nothing to do.");
                    return;
                }
                HtmlIndexer indexer = new HtmlIndexer(Paths.get(""), jobSettings);
                boolean completed = indexer.rebuildAll(lines, logger, cancelSupplier);
                if (completed) {
                    logLine("Rebuild complete. Landing page: " + indexer.landingPagePath());
                } else {
                    logLine("Rebuild stopped - the pages may be incomplete. Run it again to finish.");
                }
            } catch (IOException e) {
                logLine("[!] Rebuild failed: " + e.getMessage());
            }
        }

        @Override
        protected void process(List<Chunk> chunks) {
            for (Chunk c : chunks) {
                if (c.kind == LOG) {
                    logArea.append(c.value);
                    logArea.append("\n");
                } else {
                    resultsModel.addElement(c.value);
                }
            }
        }

        @Override
        protected void done() {
            onJobFinished(rebuild, stopRequested);
        }
    }

    /** Receives every NEW link right after it has been stored and indexed. */
    interface LinkListener {
        void onLink(int id, String link);
    }

    // =====================================================================
    // HTML settings
    // =====================================================================

    /** The three ways the HTML pages can be generated. */
    enum Mode {
        SIMPLE("simple", "Simple",
                "<html><b>Simple</b><br>"
                        + "Each id is a plain \"Id N\" label that opens id_files/N.txt.<br>"
                        + "Categories are whole words.</html>"),
        FULL("full", "Full",
                "<html><b>Full</b> (default)<br>"
                        + "Shows and opens the real link: the page loads id_files/N.txt at runtime.<br>"
                        + "Categories are whole words. Pages must be served over http(s).</html>"),
        ADVANCED("advanced", "Advanced",
                "<html><b>Advanced</b><br>"
                        + "Like Full, but categories are every 3-26 letter substring of each word.<br>"
                        + "Generates many more pages.</html>");

        final String id;
        final String label;
        final String htmlDescription;

        Mode(String id, String label, String htmlDescription) {
            this.id = id;
            this.label = label;
            this.htmlDescription = htmlDescription;
        }

        /** "advanced" categorises by substring; "simple"/"full" by whole word. */
        boolean usesSubstrings() {
            return this == ADVANCED;
        }

        /** Value written to "var linkStyle" in the pages. "advanced" reuses full's runtime behaviour. */
        String runtimeStyle() {
            return this == ADVANCED ? FULL.id : id;
        }

        static Mode fromId(String value, Mode fallback) {
            if (value != null) {
                for (Mode m : values()) {
                    if (m.id.equalsIgnoreCase(value.trim())) {
                        return m;
                    }
                }
            }
            return fallback;
        }
    }

    /** The contents of the Options dialog: mode radio buttons + maximum page size. */
    @SuppressWarnings("serial")
    static final class OptionsPanel extends JPanel {
        private final Map<Mode, JRadioButton> radios = new EnumMap<Mode, JRadioButton>(Mode.class);
        private final JSpinner sizeSpinner;
        private final Mode fallbackMode;

        OptionsPanel(IndexSettings current) {
            this.fallbackMode = current.mode;
            setLayout(new BoxLayout(this, BoxLayout.Y_AXIS));

            ButtonGroup group = new ButtonGroup();
            JPanel modePanel = new JPanel();
            modePanel.setLayout(new BoxLayout(modePanel, BoxLayout.Y_AXIS));
            modePanel.setBorder(BorderFactory.createTitledBorder("Mode"));
            for (Mode m : Mode.values()) {
                JRadioButton rb = new JRadioButton(m.htmlDescription);
                rb.setSelected(m == current.mode);
                rb.setVerticalTextPosition(SwingConstants.TOP); // radio dot next to the first line
                rb.setAlignmentX(Component.LEFT_ALIGNMENT);
                group.add(rb);
                radios.put(m, rb);
                modePanel.add(rb);
                modePanel.add(Box.createVerticalStrut(6));
            }
            modePanel.setAlignmentX(Component.LEFT_ALIGNMENT);

            sizeSpinner = new JSpinner(new SpinnerNumberModel(
                    current.pageSizeKb, IndexSettings.MIN_PAGE_SIZE_KB, IndexSettings.MAX_PAGE_SIZE_KB, 10));
            sizeSpinner.setPreferredSize(new Dimension(90, 24));
            JPanel sizeRow = new JPanel(new FlowLayout(FlowLayout.LEFT, 8, 4));
            sizeRow.add(new JLabel("Maximum size of each HTML page:"));
            sizeRow.add(sizeSpinner);
            sizeRow.add(new JLabel("KB"));
            sizeRow.setAlignmentX(Component.LEFT_ALIGNMENT);
            JLabel sizeNote = new JLabel("<html>When a category has more ids than fit, the rest continue on<br>"
                    + "&lt;category&gt;_2.html, _3.html, ... with Previous/Next buttons.<br>"
                    + "Smaller pages load faster in Full mode (one fetch per id).</html>");
            sizeNote.setForeground(Color.GRAY);
            sizeNote.setBorder(new EmptyBorder(0, 8, 4, 0));
            sizeNote.setAlignmentX(Component.LEFT_ALIGNMENT);

            JPanel sizePanel = new JPanel();
            sizePanel.setLayout(new BoxLayout(sizePanel, BoxLayout.Y_AXIS));
            sizePanel.setBorder(BorderFactory.createTitledBorder("Pagination"));
            sizePanel.add(sizeRow);
            sizePanel.add(sizeNote);
            sizePanel.setAlignmentX(Component.LEFT_ALIGNMENT);

            add(modePanel);
            add(Box.createVerticalStrut(8));
            add(sizePanel);
        }

        /** The settings currently chosen in the panel. */
        IndexSettings readSettings() {
            try {
                sizeSpinner.commitEdit();
            } catch (java.text.ParseException ignored) {
                // keep the last valid value
            }
            Mode chosen = fallbackMode;
            for (Map.Entry<Mode, JRadioButton> en : radios.entrySet()) {
                if (en.getValue().isSelected()) {
                    chosen = en.getKey();
                }
            }
            return new IndexSettings(chosen, ((Number) sizeSpinner.getValue()).intValue());
        }
    }

    /** Immutable mode + page-size pair. Persisted in crawler-settings.properties. */
    static final class IndexSettings {
        static final int DEFAULT_PAGE_SIZE_KB = 100;
        static final int MIN_PAGE_SIZE_KB = 2;
        static final int MAX_PAGE_SIZE_KB = 10240;
        private static final String FILE_NAME = "crawler-settings.properties";

        final Mode mode;
        final int pageSizeKb;

        IndexSettings(Mode mode, int pageSizeKb) {
            this.mode = mode;
            this.pageSizeKb = Math.max(MIN_PAGE_SIZE_KB, Math.min(MAX_PAGE_SIZE_KB, pageSizeKb));
        }

        /** Maximum page size in bytes (binary kilobytes, as in the original processor). */
        int maxPageBytes() {
            return pageSizeKb * 1024;
        }

        String describe() {
            return mode.label + " mode, " + pageSizeKb + " KB pages";
        }

        @Override
        public boolean equals(Object o) {
            if (!(o instanceof IndexSettings)) {
                return false;
            }
            IndexSettings other = (IndexSettings) o;
            return mode == other.mode && pageSizeKb == other.pageSizeKb;
        }

        @Override
        public int hashCode() {
            return mode.hashCode() * 31 + pageSizeKb;
        }

        static IndexSettings defaults() {
            return new IndexSettings(Mode.FULL, DEFAULT_PAGE_SIZE_KB);
        }

        static IndexSettings load() {
            Properties props = new Properties();
            Path file = Paths.get(FILE_NAME);
            if (Files.exists(file)) {
                try (InputStream in = Files.newInputStream(file)) {
                    props.load(in);
                } catch (IOException ignored) {
                    return defaults();
                }
            }
            Mode mode = Mode.fromId(props.getProperty("html.mode"), Mode.FULL);
            int kb = DEFAULT_PAGE_SIZE_KB;
            try {
                kb = Integer.parseInt(props.getProperty("html.pageSizeKb", "").trim());
            } catch (NumberFormatException ignored) {
                // keep the default
            }
            return new IndexSettings(mode, kb);
        }

        void save() {
            Properties props = new Properties();
            props.setProperty("html.mode", mode.id);
            props.setProperty("html.pageSizeKb", String.valueOf(pageSizeKb));
            try (OutputStream out = Files.newOutputStream(Paths.get(FILE_NAME))) {
                props.store(out, "Magnetic Pagination settings");
            } catch (IOException ignored) {
                // not fatal: the settings just won't be remembered
            }
        }
    }

    // =====================================================================
    // links.txt storage helper - shared by the crawler and the rebuild job.
    // A link's id is its position (1-based) across links.txt, links_1.txt, ...
    // Blank lines are ignored and never consume an id.
    // =====================================================================
    static final class LinkStore {
        private static final String BASE_NAME = "links";
        private static final String EXTENSION = ".txt";
        static final long MAX_FILE_SIZE_BYTES = 10L * 1024 * 1024; // 10 MB

        private LinkStore() {
        }

        static String fileName(int index) {
            return (index == 0) ? BASE_NAME + EXTENSION : BASE_NAME + "_" + index + EXTENSION;
        }

        /** Index of the last links file that exists (0 if there is none). */
        static int lastIndex() {
            int index = 0;
            int last = 0;
            while (Files.exists(Paths.get(fileName(index)))) {
                last = index;
                index++;
            }
            return last;
        }

        static boolean hasAny() {
            Path first = Paths.get(fileName(0));
            try {
                return Files.exists(first) && Files.size(first) > 0;
            } catch (IOException e) {
                return false;
            }
        }

        /** Every stored link in id order (id = list position + 1). */
        static List<String> readAll() throws IOException {
            List<String> out = new ArrayList<String>();
            int index = 0;
            while (true) {
                Path path = Paths.get(fileName(index));
                if (!Files.exists(path)) {
                    break;
                }
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(Files.newInputStream(path), StandardCharsets.UTF_8));
                try {
                    String line;
                    while ((line = reader.readLine()) != null) {
                        String trimmed = line.trim();
                        if (!trimmed.isEmpty()) {
                            out.add(trimmed);
                        }
                    }
                } finally {
                    reader.close();
                }
                index++;
            }
            return out;
        }
    }

    // =====================================================================
    // HTML indexer - the former LinkProcessor, made incremental.
    //
    // addLink() is called once per NEW link. It writes id_files/<id>.txt and
    // adds the id to every category page, creating overflow pages when a page
    // would exceed the configured size. Appending to an existing page only
    // patches the tail of the file (the ids array is the last thing before a
    // constant suffix), so a very common category such as "www" is not
    // rewritten in full for every link.
    //
    // rebuildAll() regenerates everything from scratch out of a list of links
    // (the old batch behaviour). Both paths produce identical files.
    // =====================================================================
    static final class HtmlIndexer {

        static final String ID_FILES_DIR = "id_files";
        static final String HTML_DIR = "html_pages";

        // "advanced" mode: every substring of length 3..26 of each piece is a category.
        static final int MIN_SUBSTRING_LENGTH = 3;
        static final int MAX_SUBSTRING_LENGTH = 26;
        // Words shorter than this never become categories (e.g. "a", "to", "of", "id").
        static final int MIN_WORD_LENGTH = 3;
        // The landing page only embeds this many categories (first-seen order); any other
        // category is still reachable through the search box's "Open" button.
        static final int LANDING_CATEGORIES_LIMIT = 5;
        static final int SUGGESTION_LIMIT = 10;
        // Category page states kept in memory (they mirror the files, so evicting is safe).
        private static final int STATE_CACHE_LIMIT = 50000;

        // Strips a leading scheme such as "http://" or "https://"
        private static final Pattern PROTOCOL = Pattern.compile("^[a-zA-Z][a-zA-Z0-9+.-]*://");
        // Anything that is not a letter or a digit becomes a separator
        private static final Pattern SPECIAL_CHARS = Pattern.compile("[^a-zA-Z0-9]+");
        // Words that never get their own page: too generic/noisy to be useful.
        private static final Set<String> STOPWORDS =
                new HashSet<String>(Arrays.asList("http", "https", "com"));
        // Windows refuses to create files with these base names ("con.html" is the console
        // device), so page 1 of these categories is stored as "<name>_1.html" instead.
        // common.js applies the same rule when it builds a page URL.
        private static final Set<String> RESERVED_PAGE_NAMES =
                new HashSet<String>(Arrays.asList("con", "prn", "aux", "nul"));

        private static final String IDS_MARKER = "var ids = [";
        // Everything that follows the ids list in a category page - constant text.
        private static final String PAGE_SUFFIX =
                ";\n</script>\n<script src=\"common.js\"></script>\n</body>\n</html>\n";
        private static final byte[] PAGE_TAIL = ("]" + PAGE_SUFFIX).getBytes(StandardCharsets.UTF_8);

        /** What we need to know about a category to append to it without re-reading its pages. */
        private static final class CategoryState {
            int totalPages;
            int lastIdsBytes;     // bytes of the comma-joined ids on the last page
            int overheadBytes;    // page size without any ids (worst-case nav buttons)
        }

        private final Path baseDir;
        private final Path idFilesDir;
        private final Path htmlDir;
        private final IndexSettings settings;
        private final String runtimeStyle;
        private final List<String> starterWords = new ArrayList<String>();

        @SuppressWarnings("serial")
        private final Map<String, CategoryState> cache =
                new LinkedHashMap<String, CategoryState>(1024, 0.75f, true) {
                    @Override
                    protected boolean removeEldestEntry(Map.Entry<String, CategoryState> eldest) {
                        return size() > STATE_CACHE_LIMIT;
                    }
                };

        HtmlIndexer(Path baseDir, IndexSettings settings) {
            this.baseDir = baseDir.toAbsolutePath().normalize();
            this.idFilesDir = this.baseDir.resolve(ID_FILES_DIR);
            this.htmlDir = this.baseDir.resolve(HTML_DIR);
            this.settings = settings;
            this.runtimeStyle = settings.mode.runtimeStyle();
        }

        String landingPagePath() {
            return baseDir.resolve("index.html").toString();
        }

        // ------------------------------------------------------------------
        // Incremental path
        // ------------------------------------------------------------------

        /** Creates the folders and shared files; loads the landing page's category list. */
        void init() throws IOException {
            Files.createDirectories(idFilesDir);
            Files.createDirectories(htmlDir);
            writeCommonFiles();
            loadStarterWords();
            if (!Files.exists(baseDir.resolve("index.html"))) {
                writeLandingPage();
            }
        }

        boolean hasIdFile(int id) {
            return Files.exists(idFilesDir.resolve(id + ".txt"));
        }

        /**
         * Processes one new link: id file first (so a page never points at a missing
         * file), then every category page. Returns how many categories the link is in.
         * If some category fails, the others are still updated and the first error is
         * thrown at the end.
         */
        int addLink(int id, String link) throws IOException {
            writeFile(idFilesDir.resolve(id + ".txt"), link);

            Set<String> categories = categoriesFor(link);
            IOException firstError = null;
            for (String category : categories) {
                try {
                    addToCategory(category, id);
                } catch (IOException e) {
                    cache.remove(category); // force a fresh look at the disk next time
                    if (firstError == null) {
                        firstError = e;
                    }
                }
            }
            if (firstError != null) {
                throw firstError;
            }
            return categories.size();
        }

        private void addToCategory(String category, int id) throws IOException {
            CategoryState st = stateOf(category);
            String idText = String.valueOf(id);
            int idBytes = idText.length();

            if (st.totalPages == 0) {
                // brand-new category
                writeCategoryPage(category, 1, 1, "[" + idText + "]");
                st.totalPages = 1;
                st.lastIdsBytes = idBytes;
                onNewCategory(category);
                return;
            }

            int addedBytes = idBytes + (st.lastIdsBytes == 0 ? 0 : 1); // +1 for the joining comma
            boolean fits = st.lastIdsBytes == 0
                    || (long) st.overheadBytes + st.lastIdsBytes + addedBytes <= settings.maxPageBytes();

            if (fits) {
                appendIdToLastPage(pagePath(category, st.totalPages), idText, st.lastIdsBytes == 0);
                st.lastIdsBytes += addedBytes;
            } else {
                // Same greedy rule as the batch splitter: start a new page. Every existing
                // page of the category now needs its "Page X of Y" / Next button refreshed.
                int newTotal = st.totalPages + 1;
                writeCategoryPage(category, newTotal, newTotal, "[" + idText + "]");
                for (int page = 1; page < newTotal; page++) {
                    refreshPageNavigation(category, page, newTotal);
                }
                st.totalPages = newTotal;
                st.lastIdsBytes = idBytes;
            }
        }

        /** Loads (and caches) how many pages a category has and how full its last page is. */
        private CategoryState stateOf(String category) throws IOException {
            CategoryState st = cache.get(category);
            if (st != null) {
                return st;
            }
            st = new CategoryState();
            st.overheadBytes = utf8ByteLength(buildCategoryPageHtml(category, 2, 9999, runtimeStyle, "[]"));

            int total = 0;
            while (Files.exists(pagePath(category, total + 1))) {
                total++;
            }
            st.totalPages = total;
            if (total > 0) {
                Path last = pagePath(category, total);
                String ids = extractIds(readFile(last));
                if (ids == null) {
                    throw new IOException("Unexpected format in " + last.getFileName()
                            + " - use \"Rebuild HTML\" to regenerate the pages");
                }
                st.lastIdsBytes = ids.length();
            }
            cache.put(category, st);
            return st;
        }

        /**
         * Adds ",id" to the ids array of a page by rewriting only the end of the file.
         * The tail is verified first, so a page that was edited by hand is never corrupted.
         */
        private void appendIdToLastPage(Path page, String idText, boolean listWasEmpty) throws IOException {
            RandomAccessFile raf = new RandomAccessFile(page.toFile(), "rw");
            try {
                long tailStart = raf.length() - PAGE_TAIL.length;
                if (tailStart < 0) {
                    throw new IOException("Unexpected format in " + page.getFileName()
                            + " - use \"Rebuild HTML\" to regenerate the pages");
                }
                byte[] existing = new byte[PAGE_TAIL.length];
                raf.seek(tailStart);
                raf.readFully(existing);
                if (!Arrays.equals(existing, PAGE_TAIL)) {
                    throw new IOException("Unexpected format in " + page.getFileName()
                            + " - use \"Rebuild HTML\" to regenerate the pages");
                }
                String addition = (listWasEmpty ? "" : ",") + idText + new String(PAGE_TAIL, StandardCharsets.UTF_8);
                raf.seek(tailStart);
                raf.write(addition.getBytes(StandardCharsets.UTF_8));
            } finally {
                raf.close();
            }
        }

        /** Re-renders an existing page (same ids) with an updated total page count. */
        private void refreshPageNavigation(String category, int pageNumber, int totalPages) throws IOException {
            Path page = pagePath(category, pageNumber);
            String ids = extractIds(readFile(page));
            if (ids == null) {
                throw new IOException("Unexpected format in " + page.getFileName()
                        + " - use \"Rebuild HTML\" to regenerate the pages");
            }
            writeCategoryPage(category, pageNumber, totalPages, "[" + ids + "]");
        }

        /** The landing page lists the first categories ever created, so it only changes while it has room. */
        private void onNewCategory(String category) throws IOException {
            if (starterWords.size() < LANDING_CATEGORIES_LIMIT && !starterWords.contains(category)) {
                starterWords.add(category);
                writeLandingPage();
            }
        }

        private void loadStarterWords() {
            starterWords.clear();
            Path index = baseDir.resolve("index.html");
            if (!Files.exists(index)) {
                return;
            }
            try {
                String html = readFile(index);
                int open = html.indexOf("var wordsData = [");
                if (open < 0) {
                    return;
                }
                int start = open + "var wordsData = [".length();
                int end = html.indexOf(']', start);
                if (end < 0) {
                    return;
                }
                Matcher m = Pattern.compile("\"([^\"]*)\"").matcher(html.substring(start, end));
                while (m.find() && starterWords.size() < LANDING_CATEGORIES_LIMIT) {
                    starterWords.add(m.group(1));
                }
            } catch (IOException ignored) {
                // start with an empty list; new categories will repopulate it
            }
        }

        // ------------------------------------------------------------------
        // Batch path (Rebuild HTML)
        // ------------------------------------------------------------------

        /**
         * Regenerates id_files, every category page, the shared files and the landing page
         * from scratch. Old *.html files in html_pages are deleted first so nothing stale
         * (e.g. substring pages from a previous "advanced" run) is left behind.
         *
         * @return false if it was cancelled part-way through
         */
        boolean rebuildAll(List<String> lines, Consumer<String> log, Callable<Boolean> cancel) throws IOException {
            Files.createDirectories(idFilesDir);
            Files.createDirectories(htmlDir);
            cache.clear();
            starterWords.clear();

            int removed = deleteGeneratedPages();
            if (removed > 0) {
                log.accept("Removed " + removed + " old page(s) from " + HTML_DIR);
            }
            writeCommonFiles();

            log.accept("Writing id files and building the category index...");
            Map<String, List<Integer>> index = new LinkedHashMap<String, List<Integer>>();
            for (int i = 0; i < lines.size(); i++) {
                if (isCancelled(cancel)) {
                    return false;
                }
                int id = i + 1;
                String line = lines.get(i);
                writeFile(idFilesDir.resolve(id + ".txt"), line);
                for (String category : categoriesFor(line)) {
                    List<Integer> ids = index.get(category);
                    if (ids == null) {
                        ids = new ArrayList<Integer>();
                        index.put(category, ids);
                    }
                    ids.add(id);
                }
                if (id % 5000 == 0) {
                    log.accept("  " + id + " / " + lines.size() + " links processed");
                }
            }

            log.accept("Writing pages for " + index.size() + " categor" + (index.size() == 1 ? "y" : "ies") + "...");
            int written = 0;
            for (Map.Entry<String, List<Integer>> entry : index.entrySet()) {
                if (isCancelled(cancel)) {
                    return false;
                }
                writeCategoryPages(entry.getKey(), entry.getValue());
                written++;
                if (written % 2000 == 0) {
                    log.accept("  " + written + " / " + index.size() + " categories written");
                }
            }

            for (String category : index.keySet()) {
                if (starterWords.size() >= LANDING_CATEGORIES_LIMIT) {
                    break;
                }
                starterWords.add(category);
            }
            writeLandingPage();
            log.accept("Created " + lines.size() + " id file(s) and " + index.size()
                    + " categor" + (index.size() == 1 ? "y" : "ies") + " in " + htmlDir);
            return true;
        }

        private int deleteGeneratedPages() throws IOException {
            int count = 0;
            if (!Files.isDirectory(htmlDir)) {
                return 0;
            }
            DirectoryStream<Path> stream = Files.newDirectoryStream(htmlDir, "*.html");
            try {
                for (Path p : stream) {
                    Files.delete(p);
                    count++;
                }
            } finally {
                stream.close();
            }
            return count;
        }

        /** Writes all pages of one category, splitting ids across pages at the size limit. */
        private void writeCategoryPages(String category, List<Integer> ids) throws IOException {
            int overheadBytes = utf8ByteLength(buildCategoryPageHtml(category, 2, 9999, runtimeStyle, "[]"));
            List<List<Integer>> pages = splitIntoPages(ids, overheadBytes);
            int totalPages = pages.size();
            for (int i = 0; i < totalPages; i++) {
                writeCategoryPage(category, i + 1, totalPages, buildIdsJson(pages.get(i)));
            }
        }

        /** Greedily groups bare ids so that each page's estimated byte size stays under the limit. */
        private List<List<Integer>> splitIntoPages(List<Integer> ids, int overheadBytes) {
            List<List<Integer>> pages = new ArrayList<List<Integer>>();
            List<Integer> current = new ArrayList<Integer>();
            int currentJsonBytes = 0; // bytes used by ids already in 'current', joined by commas

            for (Integer id : ids) {
                int jsonBytes = String.valueOf(id).length();
                int addedBytes = jsonBytes + (current.isEmpty() ? 0 : 1); // +1 for the joining comma

                if (!current.isEmpty()
                        && (long) overheadBytes + currentJsonBytes + addedBytes > settings.maxPageBytes()) {
                    pages.add(current);
                    current = new ArrayList<Integer>();
                    currentJsonBytes = 0;
                    addedBytes = jsonBytes; // first item of the new page, no leading comma
                }
                current.add(id);
                currentJsonBytes += addedBytes;
            }
            pages.add(current); // always at least one page
            return pages;
        }

        private static String buildIdsJson(List<Integer> ids) {
            StringBuilder sb = new StringBuilder("[");
            for (int i = 0; i < ids.size(); i++) {
                if (i > 0) {
                    sb.append(",");
                }
                sb.append(ids.get(i));
            }
            sb.append("]");
            return sb.toString();
        }

        // ------------------------------------------------------------------
        // Categories
        // ------------------------------------------------------------------

        /**
         * The categories a link belongs to, in first-seen order.
         *   simple/full: whole words, skipping any word with a digit, a stopword, or
         *                fewer than 3 letters.
         *   advanced:    every digit-free substring of length 3..26 of each piece.
         * The link's scheme is stripped and every non-alphanumeric run acts as a separator.
         */
        Set<String> categoriesFor(String line) {
            Set<String> out = new LinkedHashSet<String>();
            String withoutProtocol = PROTOCOL.matcher(line).replaceFirst("");
            String cleaned = SPECIAL_CHARS.matcher(withoutProtocol).replaceAll(" ").trim();
            if (cleaned.isEmpty()) {
                return out;
            }
            boolean substrings = settings.mode.usesSubstrings();
            for (String piece : cleaned.split("\\s+")) {
                if (piece.isEmpty()) {
                    continue;
                }
                if (substrings) {
                    addSubstringCategories(piece.toLowerCase(Locale.ROOT), out);
                } else {
                    addWordCategory(piece, out);
                }
            }
            return out;
        }

        private static void addWordCategory(String rawWord, Set<String> out) {
            if (rawWord.length() < MIN_WORD_LENGTH || hasDigit(rawWord)) {
                return;
            }
            String word = rawWord.toLowerCase(Locale.ROOT);
            if (!STOPWORDS.contains(word)) {
                out.add(word);
            }
        }

        /** Every substring of length 3..26 that contains no digit (digits split the piece into runs). */
        private static void addSubstringCategories(String piece, Set<String> out) {
            int len = piece.length();
            int runStart = 0;
            while (runStart < len) {
                if (isDigit(piece.charAt(runStart))) {
                    runStart++;
                    continue;
                }
                int runEnd = runStart;
                while (runEnd < len && !isDigit(piece.charAt(runEnd))) {
                    runEnd++;
                }
                for (int start = runStart; start < runEnd; start++) {
                    int maxSubLen = Math.min(MAX_SUBSTRING_LENGTH, runEnd - start);
                    for (int subLen = MIN_SUBSTRING_LENGTH; subLen <= maxSubLen; subLen++) {
                        out.add(piece.substring(start, start + subLen));
                    }
                }
                runStart = runEnd;
            }
        }

        private static boolean isDigit(char c) {
            return c >= '0' && c <= '9';
        }

        private static boolean hasDigit(String s) {
            for (int i = 0; i < s.length(); i++) {
                if (isDigit(s.charAt(i))) {
                    return true;
                }
            }
            return false;
        }

        // ------------------------------------------------------------------
        // Page / file generation
        // ------------------------------------------------------------------

        /** "cat.html" for page 1, "cat_2.html", "cat_3.html", ... for later pages. */
        static String pageFileName(String category, int pageNumber) {
            if (pageNumber == 1) {
                return RESERVED_PAGE_NAMES.contains(category) ? category + "_1.html" : category + ".html";
            }
            return category + "_" + pageNumber + ".html";
        }

        private Path pagePath(String category, int pageNumber) {
            return htmlDir.resolve(pageFileName(category, pageNumber));
        }

        private void writeCategoryPage(String category, int pageNumber, int totalPages, String idsJson)
                throws IOException {
            writeFile(pagePath(category, pageNumber),
                    buildCategoryPageHtml(category, pageNumber, totalPages, runtimeStyle, idsJson));
        }

        /** The text between "var ids = [" and its closing bracket, or null if the page has no ids array. */
        private static String extractIds(String html) {
            int open = html.indexOf(IDS_MARKER);
            if (open < 0) {
                return null;
            }
            int start = open + IDS_MARKER.length();
            int end = html.indexOf(']', start);
            return end < 0 ? null : html.substring(start, end);
        }

        private static String buildCategoryPageHtml(String word, int pageNumber, int totalPages,
                                                    String style, String idsJson) {
            StringBuilder html = new StringBuilder();
            html.append("<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n");
            html.append("<meta charset=\"UTF-8\">\n");
            html.append("<title>").append(escapeHtml(word));
            if (totalPages > 1) {
                html.append(" (page ").append(pageNumber).append(" of ").append(totalPages).append(")");
            }
            html.append("</title>\n");
            html.append("<link rel=\"stylesheet\" href=\"common.css\">\n");
            html.append("</head>\n<body>\n");
            html.append("<a class=\"back\" href=\"../index.html\">&larr; Back to all words</a>\n");
            html.append("<h1 id=\"word-title\">").append(escapeHtml(word)).append("</h1>\n");
            if (totalPages > 1) {
                html.append("<p class=\"page-indicator\">Page ").append(pageNumber).append(" of ").append(totalPages).append("</p>\n");
            }
            html.append("<div id=\"links\"></div>\n");

            if (totalPages > 1) {
                html.append("<div class=\"pagination\">\n");
                if (pageNumber > 1) {
                    html.append("  <a class=\"page-btn\" href=\"").append(pageFileName(word, pageNumber - 1))
                        .append("\">&laquo; Previous</a>\n");
                } else {
                    html.append("  <span class=\"page-btn disabled\">&laquo; Previous</span>\n");
                }
                html.append("  <span class=\"page-info\">Page ").append(pageNumber).append(" of ").append(totalPages).append("</span>\n");
                if (pageNumber < totalPages) {
                    html.append("  <a class=\"page-btn\" href=\"").append(pageFileName(word, pageNumber + 1))
                        .append("\">Next &raquo;</a>\n");
                } else {
                    html.append("  <span class=\"page-btn disabled\">Next &raquo;</span>\n");
                }
                html.append("</div>\n");
            }

            html.append("<script>\n");
            html.append("  var wordName = ").append(jsonString(word)).append(";\n");
            html.append("  var linkStyle = ").append(jsonString(style)).append(";\n");
            html.append("  var ids = ").append(idsJson).append(PAGE_SUFFIX);
            return html.toString();
        }

        /** Landing page: the first few categories plus a live search box with a direct "Open" button. */
        private void writeLandingPage() throws IOException {
            StringBuilder wordsJson = new StringBuilder("[");
            for (int i = 0; i < starterWords.size(); i++) {
                if (i > 0) {
                    wordsJson.append(",");
                }
                wordsJson.append(jsonString(starterWords.get(i)));
            }
            wordsJson.append("]");

            StringBuilder html = new StringBuilder();
            html.append("<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n");
            html.append("<meta charset=\"UTF-8\">\n");
            html.append("<title>Links index</title>\n");
            html.append("<link rel=\"stylesheet\" href=\"").append(HTML_DIR).append("/common.css\">\n");
            html.append("</head>\n<body class=\"landing\">\n");
            html.append("<h1>Links index</h1>\n");
            html.append("<div class=\"search-box\">\n");
            html.append("  <div class=\"search-row\">\n");
            html.append("    <input type=\"text\" id=\"search-input\" placeholder=\"Search or type an exact word...\" autocomplete=\"off\">\n");
            html.append("    <button type=\"button\" id=\"search-go\">Open</button>\n");
            html.append("  </div>\n");
            html.append("  <div id=\"suggestions\"></div>\n");
            html.append("</div>\n");
            html.append("<h2>First ").append(LANDING_CATEGORIES_LIMIT).append(" categories found</h2>\n");
            html.append("<div id=\"top-words\"></div>\n");
            html.append("<script>\n");
            html.append("  var wordsData = ").append(wordsJson).append(";\n");
            html.append("</script>\n");
            html.append("<script src=\"").append(HTML_DIR).append("/common.js\"></script>\n");
            html.append("</body>\n</html>\n");

            writeFile(baseDir.resolve("index.html"), html.toString());
        }

        private void writeCommonFiles() throws IOException {
            writeFile(htmlDir.resolve("common.js"), buildCommonJs());
            writeFile(htmlDir.resolve("common.css"), buildCommonCss());
        }

        private static String text(String... lines) {
            StringBuilder sb = new StringBuilder();
            for (String l : lines) {
                sb.append(l).append('\n');
            }
            return sb.toString();
        }

        /** Single JS file shared by every generated page (category pages + landing page). */
        private static String buildCommonJs() {
            return text(
                "// Shared script used by every generated html page.",
                "//",
                "// Category pages define:",
                "//   var wordName = '...';",
                "//   var linkStyle = 'simple' | 'full';",
                "//   var ids = [1, 7, 42, ...];              (bare id numbers)",
                "//",
                "// 'simple' style: each id becomes a plain 'Id N' label that opens",
                "//   ../" + ID_FILES_DIR + "/<id>.txt in a new tab.",
                "// 'full' style: for each id, the content of ../" + ID_FILES_DIR + "/<id>.txt is",
                "//   fetched at runtime and used as both the label and the href, so",
                "//   clicking opens the real original link in a new tab. Requires the",
                "//   page to be served over http(s) (fetch() can't read file:// URLs).",
                "//   If the fetch fails, the link falls back to opening the local",
                "//   " + ID_FILES_DIR + "/<id>.txt file so it's never left dead.",
                "//",
                "// The landing page (index.html) defines:",
                "//   var wordsData = ['word1', 'word2', ...];  (first " + LANDING_CATEGORIES_LIMIT + " categories found, no counts)",
                "document.addEventListener('DOMContentLoaded', function () {",
                "    if (window.ids) {",
                "        renderIdLinks();",
                "    }",
                "    if (window.wordsData) {",
                "        renderTopWords();",
                "        initSearch();",
                "    }",
                "});",
                "",
                "// Local " + ID_FILES_DIR + "/<id>.txt path, relative to a category page in '" + HTML_DIR + "'.",
                "function idFilePath(id) {",
                "    return '../" + ID_FILES_DIR + "/' + id + '.txt';",
                "}",
                "",
                "// Renders the list of clickable ids on a category page.",
                "// 'simple' style: label/href are computed immediately, no fetch needed.",
                "// 'full' style: each link starts as a placeholder, then " + ID_FILES_DIR + "/<id>.txt is",
                "// fetched and its content becomes both the label and the href - so",
                "// clicking opens the real link, not the .txt file.",
                "function renderIdLinks() {",
                "    var container = document.getElementById('links');",
                "    if (!container) {",
                "        return;",
                "    }",
                "",
                "    var title = document.getElementById('word-title');",
                "    if (title && window.wordName) {",
                "        title.textContent = window.wordName;",
                "    }",
                "",
                "    var isFull = window.linkStyle === 'full';",
                "",
                "    window.ids.forEach(function (id) {",
                "        var a = document.createElement('a');",
                "        a.target = '_blank';",
                "        a.rel = 'noopener noreferrer';",
                "        a.className = 'link-item';",
                "",
                "        if (!isFull) {",
                "            a.href = idFilePath(id);",
                "            a.textContent = 'Id ' + id;",
                "            container.appendChild(a);",
                "            return;",
                "        }",
                "",
                "        a.href = idFilePath(id);",
                "        a.textContent = 'Id ' + id + ' (loading...)';",
                "        container.appendChild(a);",
                "",
                "        fetch(idFilePath(id))",
                "            .then(function (res) {",
                "                if (!res.ok) {",
                "                    throw new Error('HTTP ' + res.status);",
                "                }",
                "                return res.text();",
                "            })",
                "            .then(function (text) {",
                "                var link = text.trim();",
                "                a.href = link;",
                "                a.textContent = link;",
                "            })",
                "            .catch(function () {",
                "                // Fetch can fail (e.g. page opened via file:// instead of a",
                "                // real http server). Fall back to opening the local .txt file",
                "                // itself so the link is never left dead.",
                "                a.href = idFilePath(id);",
                "                a.textContent = 'Id ' + id + ' (could not load link - opens local file)';",
                "            });",
                "    });",
                "}",
                "",
                "// Renders the small set of starter categories embedded in wordsData.",
                "function renderTopWords() {",
                "    var container = document.getElementById('top-words');",
                "    if (!container) {",
                "        return;",
                "    }",
                "    window.wordsData.forEach(function (word) {",
                "        container.appendChild(buildWordLink(word));",
                "    });",
                "}",
                "",
                "// Page 1 of these categories is stored as <name>_1.html because Windows",
                "// does not allow files named con/prn/aux/nul (keep in sync with the generator).",
                "var RESERVED_PAGE_NAMES = ['con', 'prn', 'aux', 'nul'];",
                "",
                "// The landing page (index.html) lives one level above '" + HTML_DIR + "',",
                "// so category pages are reached with '" + HTML_DIR + "/<word>.html'.",
                "function wordPageUrl(word) {",
                "    var file = RESERVED_PAGE_NAMES.indexOf(word) !== -1 ? word + '_1' : word;",
                "    return '" + HTML_DIR + "/' + file + '.html';",
                "}",
                "",
                "function buildWordLink(word) {",
                "    var a = document.createElement('a');",
                "    a.href = wordPageUrl(word);",
                "    a.className = 'word-item';",
                "    a.textContent = word;",
                "    return a;",
                "}",
                "",
                "// Live search box with up to " + SUGGESTION_LIMIT + " suggestions (matched only against the",
                "// small starter set in wordsData), plus a direct 'Open' button/Enter fallback that",
                "// jumps straight to the typed word's page even if it isn't in that starter set or",
                "// doesn't exist at all - the browser will simply show its usual not-found page.",
                "function initSearch() {",
                "    var input = document.getElementById('search-input');",
                "    var suggestions = document.getElementById('suggestions');",
                "    var goButton = document.getElementById('search-go');",
                "    if (!input || !suggestions) {",
                "        return;",
                "    }",
                "",
                "    function openTypedWord() {",
                "        var word = input.value.trim().toLowerCase();",
                "        if (!word) {",
                "            return;",
                "        }",
                "        suggestions.style.display = 'none';",
                "        window.location.href = wordPageUrl(word);",
                "    }",
                "",
                "    function updateSuggestions() {",
                "        var query = input.value.trim().toLowerCase();",
                "        suggestions.innerHTML = '';",
                "        if (!query) {",
                "            suggestions.style.display = 'none';",
                "            return;",
                "        }",
                "        var matches = window.wordsData.filter(function (word) {",
                "            return word.indexOf(query) !== -1;",
                "        }).slice(0, " + SUGGESTION_LIMIT + ");",
                "",
                "        if (matches.length === 0) {",
                "            var empty = document.createElement('div');",
                "            empty.className = 'suggestion-empty';",
                "            empty.textContent = 'No matching words in the starter list - click Open to try that exact page anyway';",
                "            suggestions.appendChild(empty);",
                "            suggestions.style.display = 'block';",
                "            return;",
                "        }",
                "",
                "        matches.forEach(function (word) {",
                "            var div = document.createElement('div');",
                "            div.className = 'suggestion-item';",
                "            div.textContent = word;",
                "            div.addEventListener('click', function () {",
                "                window.location.href = wordPageUrl(word);",
                "            });",
                "            suggestions.appendChild(div);",
                "        });",
                "        suggestions.style.display = 'block';",
                "    }",
                "",
                "    // 'input' fires on every kind of user input: typing, pasting, cutting, dragging text in, etc.",
                "    input.addEventListener('input', updateSuggestions);",
                "    input.addEventListener('focus', updateSuggestions);",
                "",
                "    input.addEventListener('keydown', function (e) {",
                "        if (e.key === 'Enter') {",
                "            var first = suggestions.querySelector('.suggestion-item');",
                "            if (first) {",
                "                first.click();",
                "            } else {",
                "                // No matching suggestion - try opening the typed word's page directly.",
                "                openTypedWord();",
                "            }",
                "        } else if (e.key === 'Escape') {",
                "            suggestions.style.display = 'none';",
                "        }",
                "    });",
                "",
                "    if (goButton) {",
                "        goButton.addEventListener('click', openTypedWord);",
                "    }",
                "",
                "    document.addEventListener('click', function (e) {",
                "        if (e.target !== input && e.target !== goButton && e.target.parentNode !== suggestions) {",
                "            suggestions.style.display = 'none';",
                "        }",
                "    });",
                "}");
        }

        /** Single stylesheet shared by every generated page. */
        private static String buildCommonCss() {
            return text(
                "/* Shared stylesheet used by every generated html page. */",
                "body { font-family: Arial, sans-serif; margin: 40px; }",
                "h1 { color: #333; margin-bottom: 4px; }",
                "h2 { color: #333; margin-top: 40px; }",
                "a.back { display: inline-block; margin-bottom: 16px; color: #555; text-decoration: none; }",
                "a.back:hover { text-decoration: underline; }",
                ".page-indicator { color: #777; margin: 0 0 20px 0; font-size: 14px; }",
                ".link-item { display: block; margin: 6px 0; color: #0645ad; text-decoration: none; word-break: break-all; }",
                ".link-item:hover { text-decoration: underline; }",
                ".pagination { display: flex; align-items: center; gap: 16px; margin-top: 28px; }",
                ".page-btn { padding: 8px 14px; background: #f0f4ff; color: #0645ad; text-decoration: none; border-radius: 6px; font-size: 14px; }",
                ".page-btn:hover { background: #dbe6ff; }",
                ".page-btn.disabled { color: #aaa; background: #f2f2f2; cursor: default; }",
                ".page-info { color: #555; font-size: 14px; }",
                "/* Landing page only */",
                "body.landing { max-width: 720px; }",
                ".search-box { position: relative; margin-top: 12px; }",
                ".search-row { display: flex; gap: 8px; }",
                "#search-input { flex: 1; min-width: 0; box-sizing: border-box; padding: 10px 12px; font-size: 16px; border: 1px solid #ccc; border-radius: 6px; }",
                "#search-go { padding: 10px 18px; font-size: 15px; background: #0645ad; color: #fff; border: none; border-radius: 6px; cursor: pointer; white-space: nowrap; }",
                "#search-go:hover { background: #033a8c; }",
                "#suggestions { display: none; position: absolute; top: 100%; left: 0; right: 0; background: #fff; border: 1px solid #ccc; border-top: none; border-radius: 0 0 6px 6px; max-height: 260px; overflow-y: auto; z-index: 10; }",
                ".suggestion-item { padding: 8px 12px; cursor: pointer; }",
                ".suggestion-item:hover { background: #f0f4ff; }",
                ".suggestion-empty { padding: 8px 12px; color: #888; }",
                "#top-words { display: flex; flex-wrap: wrap; gap: 8px; }",
                ".word-item { display: inline-block; padding: 6px 12px; background: #f0f4ff; color: #0645ad; text-decoration: none; border-radius: 16px; font-size: 14px; }",
                ".word-item:hover { background: #dbe6ff; }");
        }

        // ------------------------------------------------------------------
        // Small helpers
        // ------------------------------------------------------------------

        private static boolean isCancelled(Callable<Boolean> cancel) {
            try {
                return cancel != null && Boolean.TRUE.equals(cancel.call());
            } catch (Exception e) {
                return false;
            }
        }

        private static String jsonString(String s) {
            return "\"" + s.replace("\\", "\\\\").replace("\"", "\\\"")
                    .replace("\r", "\\r").replace("\n", "\\n") + "\"";
        }

        private static int utf8ByteLength(String s) {
            return s.getBytes(StandardCharsets.UTF_8).length;
        }

        private static String escapeHtml(String s) {
            return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;");
        }

        private static String readFile(Path path) throws IOException {
            return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
        }

        private static void writeFile(Path path, String content) throws IOException {
            Files.write(path, content.getBytes(StandardCharsets.UTF_8));
        }
    }

    // =====================================================================
    // Crawling engine - the original crawler logic, driven by GUI-selected
    // options and reporting through callbacks. Every NEW link is stored in
    // links*.txt and immediately handed to the HtmlIndexer.
    // =====================================================================
    static class CrawlerEngine {

        private final boolean collectMagnetLinks;
        private final boolean collectValidUrls;
        private final int maxDepth;
        private final int timeoutMs;
        private final HtmlIndexer indexer;
        private final IndexSettings settings;

        private static final String SITES_BASE_NAME = "sites";
        private static final String SITES_EXTENSION = ".txt";
        private static final int MAX_URL_LENGTH = 1000;

        private static final String USER_AGENT = "Mozilla/5.0 (compatible; JavaWebCrawler/1.0)";

        private static final Pattern LINK_PATTERN =
                Pattern.compile("(?:href|src)=[\"']([^\"'#>]+)[\"']", Pattern.CASE_INSENSITIVE);

        // apibay.org (The Pirate Bay's JSON backend) - used for sites whose magnet
        // links are injected by JavaScript instead of being present in raw HTML.
        private static final String APIBAY_SEARCH_URL = "https://apibay.org/q.php?q=%s&cat=0";
        private static final Pattern APIBAY_ENTRY_PATTERN = Pattern.compile(
                "\"info_hash\"\\s*:\\s*\"([a-fA-F0-9]{40})\"[^}]*?\"name\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"");
        private static final Pattern APIBAY_NAME_FIRST_PATTERN = Pattern.compile(
                "\"name\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"[^}]*?\"info_hash\"\\s*:\\s*\"([a-fA-F0-9]{40})\"");
        // Tracker URLs appended to every magnet link built from apibay results.
        // Empty entries are skipped, so add your trackers here if you want them.
        private static final String[] DEFAULT_TRACKERS = {
                "",
                "",
                "",
                "",
                ""
        };

        private final Set<String> visited = new LinkedHashSet<String>();
        private final Set<String> savedLinks = new LinkedHashSet<String>();
        private int currentFileIndex = 0;
        private int linkCount = 0; // links stored so far; the next link gets id linkCount + 1

        private final Set<String> savedSites = new LinkedHashSet<String>();
        private int currentSitesFileIndex = 0;

        private Consumer<String> logger = new Consumer<String>() {
            @Override
            public void accept(String s) { /* no-op default */ }
        };
        private LinkListener linkListener = new LinkListener() {
            @Override
            public void onLink(int id, String link) { /* no-op default */ }
        };
        private Callable<Boolean> cancelSupplier = new Callable<Boolean>() {
            @Override
            public Boolean call() { return false; }
        };

        CrawlerEngine(boolean collectMagnetLinks, boolean collectValidUrls, int maxDepth, int timeoutMs,
                      HtmlIndexer indexer, IndexSettings settings) {
            this.collectMagnetLinks = collectMagnetLinks;
            this.collectValidUrls = collectValidUrls;
            this.maxDepth = maxDepth;
            this.timeoutMs = timeoutMs;
            this.indexer = indexer;
            this.settings = settings;
        }

        void setLogger(Consumer<String> logger) { this.logger = logger; }
        void setLinkListener(LinkListener listener) { this.linkListener = listener; }
        void setCancelSupplier(Callable<Boolean> supplier) { this.cancelSupplier = supplier; }

        private void log(String s) { logger.accept(s); }

        private boolean cancelled() {
            try {
                return cancelSupplier.call();
            } catch (Exception e) {
                return false;
            }
        }

        void crawl(List<String> startUrls) {
            log("==============================================");
            log("             Magnetic Pagination");
            log("==============================================");

            try {
                loadExistingLinks();
            } catch (IOException e) {
                log("[!] Cannot read the existing links files, aborting: " + e.getMessage());
                return;
            }
            loadExistingSites();
            try {
                indexer.init();
            } catch (IOException e) {
                log("[!] Cannot prepare the HTML output folders, aborting: " + e.getMessage());
                return;
            }

            log("Loaded " + linkCount + " existing link(s). HTML: " + settings.describe());
            if (linkCount > 0 && !indexer.hasIdFile(linkCount)) {
                log("[i] Existing links are not fully indexed in HTML yet - use \"Rebuild HTML...\" to include them.");
            }
            log("Starting with " + startUrls.size() + " URL(s), max depth=" + maxDepth);
            log("----------------------------------------------");

            List<String> currentLevel = new ArrayList<String>(startUrls);
            int depth = 0;
            while (!currentLevel.isEmpty()) {
                if (cancelled()) {
                    log("Crawl stopped by user.");
                    return;
                }
                log("--- Depth " + depth + ": " + currentLevel.size() + " URL(s) ---");
                Set<String> nextLevel = new LinkedHashSet<String>();

                for (String url : currentLevel) {
                    if (cancelled()) {
                        log("Crawl stopped by user.");
                        return;
                    }
                    Set<String> foundUrls = processUrl(url);
                    if (depth < maxDepth) {
                        nextLevel.addAll(foundUrls);
                    }
                }

                depth++;
                if (depth > maxDepth) {
                    break;
                }
                currentLevel = new ArrayList<String>(nextLevel);
            }

            log("----------------------------------------------");
            log("Crawl complete. " + linkCount + " link(s) in total, saved to "
                    + LinkStore.fileName(currentFileIndex) + " (and any earlier files).");
        }

        private String sitesFileName(int index) {
            return (index == 0)
                    ? SITES_BASE_NAME + SITES_EXTENSION
                    : SITES_BASE_NAME + "_" + index + SITES_EXTENSION;
        }

        private void loadExistingSites() {
            int index = 0;
            boolean anyFound = false;
            while (true) {
                Path path = Paths.get(sitesFileName(index));
                if (!Files.exists(path)) {
                    break;
                }
                anyFound = true;
                try {
                    List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
                    for (String line : lines) {
                        String trimmed = line.trim();
                        if (!trimmed.isEmpty()) {
                            savedSites.add(trimmed);
                        }
                    }
                } catch (IOException e) {
                    // ignore unreadable file, keep scanning
                }
                currentSitesFileIndex = index;
                index++;
            }
            if (!anyFound) {
                currentSitesFileIndex = 0;
            }
        }

        private void saveSite(String url) {
            if (savedSites.contains(url)) {
                return;
            }

            Path currentPath = Paths.get(sitesFileName(currentSitesFileIndex));
            long currentSize = 0;
            try {
                if (Files.exists(currentPath)) {
                    currentSize = Files.size(currentPath);
                }
            } catch (IOException e) {
                // assume 0 if we can't stat it
            }

            int lineBytes = (url + System.lineSeparator()).getBytes(StandardCharsets.UTF_8).length;
            if (currentSize + lineBytes > LinkStore.MAX_FILE_SIZE_BYTES) {
                currentSitesFileIndex++;
                currentPath = Paths.get(sitesFileName(currentSitesFileIndex));
            }

            savedSites.add(url);
            try {
                BufferedWriter writer = Files.newBufferedWriter(
                        currentPath, StandardCharsets.UTF_8,
                        StandardOpenOption.CREATE, StandardOpenOption.APPEND);
                try {
                    writer.write(url);
                    writer.newLine();
                } finally {
                    writer.close();
                }
            } catch (IOException e) {
                log("  [!] Failed to save site: " + e.getMessage());
            }
        }

        private void loadExistingLinks() throws IOException {
            List<String> all = LinkStore.readAll();
            linkCount = all.size();
            savedLinks.addAll(all);
            currentFileIndex = LinkStore.lastIndex();
        }

        private Set<String> processUrl(String url) {
            if (url == null || url.length() > MAX_URL_LENGTH) {
                log("  [!] Skipping URL (exceeds " + MAX_URL_LENGTH + " characters).");
                return Collections.emptySet();
            }

            String normalisedUrl = normalise(url);
            if (normalisedUrl == null || visited.contains(normalisedUrl)) {
                return Collections.emptySet();
            }
            visited.add(normalisedUrl);

            if (savedSites.contains(normalisedUrl)) {
                log("  [i] Already crawled, skipping: " + url);
                return Collections.emptySet();
            }
            saveSite(normalisedUrl);

            log("Fetching: " + url);

            if (collectMagnetLinks) {
                String searchQuery = extractSearchQuery(url);
                if (searchQuery != null) {
                    fetchMagnetsFromApibay(searchQuery);
                }
            }

            String html = fetchPage(url);
            if (html == null) {
                return Collections.emptySet();
            }

            Set<String> magnetLinks = new LinkedHashSet<String>();
            Set<String> validUrls = new LinkedHashSet<String>();
            extractLinks(html, url, magnetLinks, validUrls);

            Iterator<String> it = validUrls.iterator();
            while (it.hasNext()) {
                if (it.next().length() > MAX_URL_LENGTH) {
                    it.remove();
                }
            }

            if (collectMagnetLinks) {
                for (String link : magnetLinks) {
                    saveLink(link);
                }
            }
            if (collectValidUrls) {
                for (String link : validUrls) {
                    saveLink(link);
                }
            }

            return validUrls;
        }

        private String extractSearchQuery(String url) {
            try {
                URL u = new URL(url);
                String query = u.getQuery();
                if (query == null) return null;
                for (String param : query.split("&")) {
                    int eq = param.indexOf('=');
                    if (eq <= 0) continue;
                    String key = param.substring(0, eq);
                    String value = param.substring(eq + 1);
                    if (key.equals("q") && !value.isEmpty()) {
                        return URLDecoder.decode(value, "UTF-8");
                    }
                }
            } catch (Exception e) {
                // ignore, not a search URL
            }
            return null;
        }

        private void fetchMagnetsFromApibay(String searchQuery) {
            String apiUrl;
            try {
                apiUrl = String.format(APIBAY_SEARCH_URL, URLEncoder.encode(searchQuery, "UTF-8"));
            } catch (Exception e) {
                return;
            }

            log("  [i] Querying apibay API for: " + searchQuery);
            String json = fetchRaw(apiUrl);
            if (json == null) {
                log("  [!] apibay API request failed.");
                return;
            }

            int found = 0;
            Matcher m1 = APIBAY_ENTRY_PATTERN.matcher(json);
            while (m1.find()) {
                String infoHash = m1.group(1);
                String name = unescapeJson(m1.group(2));
                if (isValidHash(infoHash)) {
                    saveLink(buildMagnetLink(infoHash, name));
                    found++;
                }
            }
            Matcher m2 = APIBAY_NAME_FIRST_PATTERN.matcher(json);
            while (m2.find()) {
                String name = unescapeJson(m2.group(1));
                String infoHash = m2.group(2);
                if (isValidHash(infoHash)) {
                    saveLink(buildMagnetLink(infoHash, name));
                    found++;
                }
            }

            if (found == 0) {
                log("  [!] No results found on apibay for: " + searchQuery);
            }
        }

        private boolean isValidHash(String hash) {
            return hash != null && !hash.matches("0+");
        }

        private String buildMagnetLink(String infoHash, String name) {
            StringBuilder sb = new StringBuilder();
            sb.append("magnet:?xt=urn:btih:").append(infoHash.toLowerCase(Locale.ROOT));
            try {
                sb.append("&dn=").append(URLEncoder.encode(name, "UTF-8"));
            } catch (Exception e) {
                // skip display name if encoding fails
            }
            for (String tracker : DEFAULT_TRACKERS) {
                if (tracker == null || tracker.isEmpty()) {
                    continue;
                }
                try {
                    sb.append("&tr=").append(URLEncoder.encode(tracker, "UTF-8"));
                } catch (Exception e) {
                    // ignore
                }
            }
            return sb.toString();
        }

        private String unescapeJson(String s) {
            return s.replace("\\\"", "\"")
                    .replace("\\\\", "\\")
                    .replace("\\/", "/")
                    .replace("\\n", " ")
                    .replace("\\t", " ")
                    .trim();
        }

        private String fetchRaw(String url) {
            HttpURLConnection conn = null;
            try {
                URL u = new URL(url);
                conn = (HttpURLConnection) u.openConnection();
                conn.setRequestMethod("GET");
                conn.setConnectTimeout(timeoutMs);
                conn.setReadTimeout(timeoutMs);
                conn.setInstanceFollowRedirects(true);
                conn.setRequestProperty("User-Agent", USER_AGENT);
                conn.setRequestProperty("Accept", "application/json,text/plain,*/*");

                int status = conn.getResponseCode();
                if (status < 200 || status >= 400) {
                    return null;
                }

                InputStream is = conn.getInputStream();
                try {
                    BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
                    StringBuilder sb = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        sb.append(line).append('\n');
                    }
                    return sb.toString();
                } finally {
                    is.close();
                }
            } catch (Exception e) {
                // Keep going quietly if transport anomalies occur
            } finally {
                if (conn != null) conn.disconnect();
            }
            return null;
        }

        /**
         * Stores a link if it is new: append to links*.txt (rotating at 10 MB), then run the
         * HTML pipeline for it (id file + category pages + landing page).
         */
        private void saveLink(String link) {
            if (savedLinks.contains(link)) {
                return;
            }

            Path currentPath = Paths.get(LinkStore.fileName(currentFileIndex));
            long currentSize = 0;
            try {
                if (Files.exists(currentPath)) {
                    currentSize = Files.size(currentPath);
                }
            } catch (IOException e) {
                // assume 0 if we can't stat it
            }

            int lineBytes = (link + System.lineSeparator()).getBytes(StandardCharsets.UTF_8).length;
            if (currentSize + lineBytes > LinkStore.MAX_FILE_SIZE_BYTES) {
                currentFileIndex++;
                currentPath = Paths.get(LinkStore.fileName(currentFileIndex));
            }

            savedLinks.add(link);
            try {
                BufferedWriter writer = Files.newBufferedWriter(
                        currentPath, StandardCharsets.UTF_8,
                        StandardOpenOption.CREATE, StandardOpenOption.APPEND);
                try {
                    writer.write(link);
                    writer.newLine();
                } finally {
                    writer.close();
                }
            } catch (IOException e) {
                log("  [!] Failed to save link: " + e.getMessage());
                return;
            }

            int id = ++linkCount; // the link's position across links*.txt
            String summary = "  [+] #" + id + " " + currentPath.getFileName();
            try {
                int categories = indexer.addLink(id, link);
                log(summary + " (" + categories + " categor" + (categories == 1 ? "y" : "ies") + "): " + link);
            } catch (IOException e) {
                log(summary + ": " + link);
                log("      [!] HTML index update problem for #" + id + ": " + e.getMessage());
            }
            linkListener.onLink(id, link);
        }

        private String fetchPage(String url) {
            HttpURLConnection conn = null;
            try {
                URL u = new URL(url);
                conn = (HttpURLConnection) u.openConnection();
                conn.setRequestMethod("GET");
                conn.setConnectTimeout(timeoutMs);
                conn.setReadTimeout(timeoutMs);
                conn.setInstanceFollowRedirects(true);
                conn.setRequestProperty("User-Agent", USER_AGENT);
                conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");

                int status = conn.getResponseCode();
                if (status < 200 || status >= 400) {
                    return null;
                }

                String contentType = conn.getContentType();
                if (contentType != null && !contentType.contains("text/html") && !contentType.contains("application/xhtml")) {
                    return null;
                }

                String charset = StandardCharsets.UTF_8.name();
                if (contentType != null) {
                    Pattern p = Pattern.compile("charset=([^;\\s]+)", Pattern.CASE_INSENSITIVE);
                    Matcher m = p.matcher(contentType);
                    if (m.find()) {
                        charset = m.group(1).replace("\"", "").trim();
                    }
                }

                InputStream is = conn.getInputStream();
                try {
                    BufferedReader reader = new BufferedReader(new InputStreamReader(is, charset));
                    StringBuilder sb = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        sb.append(line).append('\n');
                    }
                    return sb.toString();
                } finally {
                    is.close();
                }
            } catch (Exception e) {
                // Keep going quietly if transport anomalies occur
            } finally {
                if (conn != null) conn.disconnect();
            }
            return null;
        }

        private void extractLinks(String html, String baseUrl, Set<String> magnetLinksOut, Set<String> validUrlsOut) {
            Matcher m = LINK_PATTERN.matcher(html);
            while (m.find()) {
                String raw = m.group(1).trim();
                if (raw.isEmpty()) continue;

                if (raw.toLowerCase(Locale.ROOT).startsWith("magnet:")) {
                    magnetLinksOut.add(raw);
                    continue;
                }

                String resolved = resolveUrl(baseUrl, raw);
                if (resolved != null) {
                    validUrlsOut.add(resolved);
                }
            }
        }

        private String resolveUrl(String base, String raw) {
            try {
                String lower = raw.toLowerCase(Locale.ROOT);
                if (lower.startsWith("mailto:") || lower.startsWith("javascript:")
                        || lower.startsWith("data:") || lower.startsWith("tel:")) {
                    return null;
                }
                URL baseUrl = new URL(base);
                URL resolved = new URL(baseUrl, raw);
                String scheme = resolved.getProtocol();
                if (!"http".equals(scheme) && !"https".equals(scheme)) {
                    return null;
                }
                String result = resolved.toExternalForm();
                int hashIdx = result.indexOf('#');
                if (hashIdx >= 0) {
                    result = result.substring(0, hashIdx);
                }
                return result;
            } catch (MalformedURLException e) {
                return null;
            }
        }

        private String normalise(String url) {
            if (url == null || url.isEmpty()) return null;
            try {
                URL u = new URL(url);
                String path = u.getPath();
                if (path.endsWith("/")) {
                    path = path.substring(0, path.length() - 1);
                }
                String query = u.getQuery() == null ? "" : "?" + u.getQuery();
                return u.getProtocol().toLowerCase(Locale.ROOT) + "://" + u.getHost().toLowerCase(Locale.ROOT)
                        + (u.getPort() == -1 ? "" : ":" + u.getPort()) + path + query;
            } catch (MalformedURLException e) {
                return null;
            }
        }
    }
}