我想为一个我心中的项目抓取Silksong维基中“needolin dialogue”部分。我参考了一些在线教程，写的代码似乎大体可用，但它保存的是维基中的Location一节，而不是needolin，我实在搞不清楚为什么。
有人能帮忙吗？我在这上面已经花了几个小时，也不想放弃这个项目（我从未做过编程）。

import requests
from bs4 import BeautifulSoup
import csv
from urllib.parse import urljoin
import time

class SilksongEnemiesScraper:
    def __init__(self, base_url, output_file="needolin_dialogue.csv"):
        self.base_url = base_url
        self.output_file = output_file
        self.session = requests.Session()
        self.session.headers.update({"User-Agent": "Mozilla/5.0"})

    def fetch_page(self, url):
        response = self.session.get(url, timeout=10)
        response.raise_for_status()
        return BeautifulSoup(response.text, "html.parser")

    def get_enemy_links(self):
        soup = self.fetch_page(self.base_url)
        links = []
        content_div = soup.select_one("#mw-content-text")
        if content_div:
            for a in content_div.select("a"):
                href = a.get("href")
                name = a.get_text(strip=True)
                # Only keep links that look like enemy pages
                if href and href.startswith("/w/") and name and ":" not in href:
                    full = urljoin(self.base_url, href)
                    if full not in links:
                        links.append((name, full))
        return links

    def scrape_enemy_dialogue(self, url):
        soup = self.fetch_page(url)
        dialogue = []

        # Look for any header that contains "Needolin Dialogue"
        for header in soup.find_all(["h2"]):
            if "Dialogue" in header.get_text(strip=True):
                # Next sibling <ul> has the lines
                ul = header.find_next_sibling("ul")
                if ul:
                    dialogue = [li.get_text(strip=True) for li in ul.find_all("li")]
                    print("collected", dialogue)
                break

        return dialogue

    def scrape_all(self):
        enemies = self.get_enemy_links()
        results = []
        for name, url in enemies:
            print(f"Scraping {name}...")
            dia = self.scrape_enemy_dialogue(url)
            results.append({
                "enemy_name": name,
                "needolin_dialogue": " | ".join(dia) if dia else ""
            })
            time.sleep(1)
        return results

    def save_csv(self, data):
        with open(self.output_file, "w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=["enemy_name", "needolin_dialogue"])
            writer.writeheader()
            writer.writerows(data)
        print(f"Saved", len(data), "entries to", self.output_file)

# Start scraping
scraper = SilksongEnemiesScraper("https://hollowknight.wiki/w/Enemies_(Silksong)")
all_data = scraper.scrape_all()
scraper.save_csv(all_data)

解决方案

你对 ul 的搜索方式错了。

你用文本 Dialogue 搜索 h2，接着你搜索同级节点 ul，但 ul 与对话框相关的并不是同级，因为它位于 div 里面。

存在一个同级元素 ul，但在 Location 之后，你得到的是来自位置的文本。

你必须在 <div id="needolin-dialogues> 中搜索 ul（不要带有“sibling”）。

你甚至可以直接在这个 div 中搜索 li——也就是使用CSS选择器。

soup.select("div#needolin-dialogues li")

可用的函数：

    def scrape_enemy_dialogue(self, url):
        #print("url:", url)
        soup = self.fetch_page(url)

        dialogue = [
            li.get_text(strip=True) 
            for li in soup.select("div#needolin-dialogues li")
        ]
        #print("dialogue:", dialogue)

        return dialogue

输出的一部分：

Scraping Mossgrub...
url: https://hollowknight.wiki/w/Mossgrub
dialogue: ['Protect us, mother!', 'Call for danger, hide away...', 'Young must eat, grow or die...', 'Sleep and change, have no fear...']

Scraping Massive Mossgrub...
url: https://hollowknight.wiki/w/Massive_Mossgrub
dialogue: ["Mother's voice... distant...", 'Little sisters... hide away...', 'Eat and grow... larger...', 'Change... hidden change...']

Scraping Mossmir...
url: https://hollowknight.wiki/w/Mossmir
dialogue: ['Protect us, mother!', 'Call for danger, hide away...', 'Young must eat, grow or die...', 'Sleep and change, have no fear...']

顺便说一句：

同样适用于 find() find_all()

    def scrape_enemy_dialogue(self, url):
        soup = self.fetch_page(url)

        dialogue = []

        div = soup.find("div", id="needolin-dialogues")
        # div = soup.find("div", {"id": "needolin-dialogues"})

        if div:
            dialogue = [li.get_text(strip=True) for li in div.find_all("li")]

        return dialogue