How to Scrape Wikipedia Data: Ultimate Tutorial

touch main.py

USERNAME = 'yourUsername'
PASSWORD = 'yourPassword'

# Structure payload.
payload = {
   'source': 'universal',
   'url': 'https://en.wikipedia.org/wiki/Michael_Phelps',
}

response = requests.request(
   'POST',
   'https://realtime.oxylabs.io/v1/queries',
   auth=(USERNAME, PASSWORD),
   json=payload,
)

print(response.status_code)

import requests
import json
import pandas

def extract_links():
    links = []
    for link in result_set['content']['links']:
        if link and not link.startswith(('https://')):
            if link.startswith('//'):
                link = 'https:' + link
            else:
                link = 'https://en.wikipedia.org' + link
            links.append(link)
    return links


def extract_images():
    images = []
    for img in result_set['content']['images']:
        if 'https://' not in img:
            if '/static/images/' in img:
                img = 'https://en.wikipedia.org' + img
            else:
                img = 'https:' + img
            images.append(img)
        else:
            continue
    return images


def extract_tables(html):
   list_of_df = pandas.read_html(html)
   tables = []
   for df in list_of_df:
       tables.append(df.to_json(orient='table'))
   return tables

# Structure payload.
payload = {
   'source': 'universal',
   'url': 'https://en.wikipedia.org/wiki/Michael_Phelps',
   "parse": 'true',
   "parsing_instructions": {
       "paragraph_text": {
           "_fns": [
               {"_fn": "css", "_args": ["p"]},
               {"_fn": "element_text"}
           ]
       },
       "table": {
           "_fns": [
               {"_fn": "css", "_args": ["table.wikitable"]},
           ]
       },
        "links": {
           "_fns": [
               {"_fn": "xpath", "_args": ["//a/@href"]},
           ]
       },
        "images": {
            "_fns": [
                {"_fn": "xpath", "_args": ["//img/@src"]},
            ]
        }
   }
}

# Create and send the request
USERNAME = 'USERNAME'
PASSWORD = 'PASSWORD'

response = requests.request(
   'POST',
   'https://realtime.oxylabs.io/v1/queries',
   auth=(USERNAME, PASSWORD),
   json=payload,
)

raw_scraped_data = response.json()
processed_data = {}

for result_set in raw_scraped_data['results']:

    processed_data['links'] = extract_links()

    processed_data['tables'] = list(map(extract_tables, result_set['content']['table'])) if result_set['content']['table'] is not None else []

    processed_data['images'] = extract_images()

    processed_data['paragraphs'] = result_set['content']['paragraph_text']

json_file_path = 'data.json'

with open(json_file_path, 'w') as json_file:
   json.dump(processed_data, json_file, indent=4) 

print(f'Data saved as {json_file_path}')

proxies = {
    "http": "https://USERNAME:PASSWORD@pr.oxylabs.io:7777",
    "https": "https://USERNAME:PASSWORD@pr.oxylabs.io:7777"
}

import csv
import requests
from bs4 import BeautifulSoup


# Residential Proxies config
proxies = {
    'http': 'https://USERNAME:PASSWORD@pr.oxylabs.io:7777',
    'https': 'https://USERNAME:PASSWORD@pr.oxylabs.io:7777'
}

# Add more headers if needed
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/135.0.0.0 Safari/537.36'
}

def scrape_wikipedia(url):
    '''
    Scrape a Wikipedia page, extract paragraphs and their links separately.
    '''
    # Send a request and parse raw HTML
    response = requests.get(
        url,
        headers=headers,
        proxies=proxies
    )
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # Find main content div and paragraphs
    content = soup.find('div', {'class': 'mw-content-ltr'})
    paragraphs = content.find_all('p', recursive=True) if content else []
    
    results = []
    for paragraph in paragraphs:
        # Skip empty paragraphs
        if not paragraph.text.strip():
            continue
        
        # Remove citation references from the paragraph
        for citation in paragraph.find_all('sup', {'class': 'reference'}):
            citation.decompose()
        text = paragraph.get_text().strip()
        
        # Extract wiki hyperlinks
        links = []
        for link in paragraph.find_all('a'):
            href = link.get('href', '')
            if (href and href.startswith('/wiki/') and 
                not href.startswith('/wiki/File:') and
                not (link.get('class') and 'reference' in link.get('class'))):
                full_url = f'https://en.wikipedia.org{href}'
                links.append(full_url)
        
        if text:
            results.append({
                'text': text,
                'links': links
            })
    
    return results


def save_to_csv(data, filename):
    '''Save the scraped data to a CSV file.'''
    with open(filename, 'w', newline='', encoding='utf-8') as file:
        writer = csv.writer(file)
        writer.writerow(['Paragraph', 'Links'])
        
        for item in data:
            links_text = ',\n'.join(item['links']) if item['links'] else ''
            writer.writerow([item['text'], links_text])


if __name__ == '__main__':
    data = scrape_wikipedia('https://en.wikipedia.org/wiki/Michael_Phelps')
    
    if data:
        save_to_csv(data, 'wikipedia_data.csv')
        print(f'Successfully scraped {len(data)} paragraphs')
    else:
        print('No data was scraped.')