Data Collection

Part 1: Web-Scrap Rotton Tomatos movie reviews

- Data Included: fan movie reviews, rating score, review date 

In the following code, we employed Selenium to scrape movie reviews, ratings, and review dates for Disney Princess movies from Rotten Tomatoes. First, we set up the Selenium WebDriver using ChromeDriverManager and configured it with a page load timeout to ensure smooth handling of webpage interactions. We navigated to the specified URL and waited for the review section to load using WebDriverWait, ensuring the necessary elements were present before proceeding. To handle paginated reviews, we implemented a loop to click the “Load More” button multiple times, allowing us to access as many reviews as possible within a defined limit.

After loading the reviews, we extracted data for each review element, including the review text, review date, and rating score. The review text was cleaned using regex to remove unwanted characters, while the score was retrieved directly from the rating attribute. Each review was stored in a dictionary and added to a list, ensuring duplicates were avoided. Finally, the collected data was saved into a Pandas DataFrame and exported as a CSV file for further analysis. This automated process efficiently handled dynamic content and pagination, enabling the retrieval of comprehensive review data for analysis.

Show Code
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import pandas as pd
import time
import re
from webdriver_manager.chrome import ChromeDriverManager

def scrape_reviews(url, output_csv_path="./data/scraped_reviews.csv", max_load_more_clicks=100, review_limit=2000):
    # # Step 1: Set up Chrome Service with extended read timeout
    # # Install and set up ChromeDriver using webdriver_manager
    # chrome_service = Service(ChromeDriverManager().install())
    # chrome_service.startup_timeout = 600  # Set startup timeout to 600 seconds

    # # Configure Chrome options and initialize the browser
    # options = Options()
    # browser = webdriver.Chrome(service=chrome_service, options=options)

    # # Step 2: Set page load timeout
    # # Set a page load timeout of 600 seconds
    # browser.set_page_load_timeout(600)

    # # Open the specified URL
    # browser.get(url)

    browser = webdriver.Chrome()  
    browser.set_page_load_timeout(600) 
    browser.get(url)

    # Step 3: Wait for the page to load
    try:
        # Wait for the audience reviews to be present on the page (max 10 seconds)
        WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, "audience-reviews__review"))
        )
    except Exception as e:
        # Handle any errors during page load
        print("Error loading reviews:", e)
        browser.quit()
        return

    # Step 4: Initialize variables and lists
    all_reviews = []  # List to store all scraped reviews
    load_more_clicks = 0  # Counter for number of 'Load More' button clicks

    # Step 5: Click "Load More" buttons up to the maximum number of clicks
    while load_more_clicks < max_load_more_clicks:
        try:
            # Wait for the 'Load More' button to be clickable (max 30 seconds)
            load_more_button = WebDriverWait(browser, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, "rt-button[data-qa='load-more-btn']"))
            )
            load_more_button.click()  # Click the 'Load More' button
            load_more_clicks += 1
            print(f"Clicked 'Load More' button {load_more_clicks} times.")
            time.sleep(5)  # Additional wait time to ensure complete loading
        except Exception as e:
            # Handle any errors or when no more 'Load More' button is found
            print("No more 'Load More' button found or an error occurred:", e)
            break

    # Step 6: Scrape reviews
    try:
        # Locate all review elements on the current page
        reviews = browser.find_elements(By.CLASS_NAME, "review-text-container")

        # Extract review text, date, and score from each review block
        for review in reviews:
            if len(all_reviews) >= review_limit:
                break  # Stop if we've reached the review limit

            try:
                # Extract the review text
                review_text = review.find_element(By.CSS_SELECTOR, "p.audience-reviews__review").text.strip()
                # Preprocess the review text to keep only specified characters
                review_text = re.sub(r"[^a-zA-Z0-9!?\"'() ]", "", review_text)

                # Extract the review date
                review_date = review.find_element(By.CSS_SELECTOR, "span.audience-reviews__duration").text.strip()

                # Extract the review score
                score_element = review.find_element(By.TAG_NAME, "rating-stars-group")
                score = score_element.get_attribute("score")

                # Create a dictionary for the review data
                review_data = {
                    "review_text": review_text,
                    "date": review_date,
                    "score": score
                }

                # Avoid duplicates and add the review to the list
                if review_data not in all_reviews:
                    all_reviews.append(review_data)

            except Exception as e:
                # Handle any errors during review processing
                print(f"Error processing review: {e}")

    except Exception as e:
        # Handle any errors during scraping
        print("Error during scraping:", e)

    # Step 7: Close the browser
    browser.quit()

    # Step 8: Save reviews to a CSV file
    # Create a DataFrame from the scraped reviews and save to CSV
    df = pd.DataFrame(all_reviews)
    df.to_csv(output_csv_path, index=False)

    print(f"Scraped {len(all_reviews)} reviews and saved to '{output_csv_path}'")

Show Code
url = "https://www.rottentomatoes.com/m/frozen_2013/reviews?type=user"
url = "https://www.rottentomatoes.com/m/cinderella/reviews?type=user"
url = "https://www.rottentomatoes.com/m/1019187-sleeping_beauty/reviews?type=user"
url = "https://www.rottentomatoes.com/m/the_little_mermaid_1989/reviews?type=user"
url = "https://www.rottentomatoes.com/m/beauty_and_the_beast_1991/reviews?type=user"
url = "https://www.rottentomatoes.com/m/disneys_aladdin/reviews?type=user"
url = "https://www.rottentomatoes.com/m/1063452-pocahontas/reviews?type=user"
url = "https://www.rottentomatoes.com/m/mulan/reviews?type=user"
url = "https://www.rottentomatoes.com/m/1196003-princess_and_the_frog/reviews?type=user"
url = "https://www.rottentomatoes.com/m/tangled/reviews?type=user"
url = "https://www.rottentomatoes.com/m/brave/reviews?type=user"
url = "https://www.rottentomatoes.com/m/moana_2016/reviews?type=user"
Show Code

scrape_reviews("https://www.rottentomatoes.com/m/moana_2/reviews?type=user",
               output_csv_path="../../data/raw-data/moana_2.csv")
Show Code

scrape_reviews("https://www.rottentomatoes.com/m/snow_white_and_the_seven_dwarfs/reviews?type=user",
               output_csv_path="./data/snow_white_and_the_seven_dwarfs.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=50547): Read timed out. (read timeout=120)
Scraped 580 reviews and saved to './data/snow_white_and_the_seven_dwarfs.csv'
Show Code

scrape_reviews("https://www.rottentomatoes.com/m/cinderella/reviews?type=user",
               output_csv_path="./data/cinderella.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=47181): Read timed out. (read timeout=120)
Scraped 918 reviews and saved to './data/cinderella.csv'
Show Code

scrape_reviews("https://www.rottentomatoes.com/m/1019187-sleeping_beauty/reviews?type=user",
               output_csv_path="./data/1019187-sleeping_beauty.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
Clicked 'Load More' button 47 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=48627): Read timed out. (read timeout=120)
Scraped 955 reviews and saved to './data/1019187-sleeping_beauty.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/the_little_mermaid_1989/reviews?type=user",
               output_csv_path="./data/the_little_mermaid_1989.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=49904): Read timed out. (read timeout=120)
Scraped 918 reviews and saved to './data/the_little_mermaid_1989.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/beauty_and_the_beast_1991/reviews?type=user",
               output_csv_path="./data/beauty_and_the_beast_1991.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
No more 'Load More' button found or an error occurred: Message: 

Scraped 90 reviews and saved to './data/beauty_and_the_beast_1991.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/disneys_aladdin/reviews?type=user",
               output_csv_path="./data/disneys_aladdin.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
No more 'Load More' button found or an error occurred: Message: element click intercepted: Element <rt-button data-loadmoremanager="btnLoadMore:click" class="" data-qa="load-more-btn" disabled="">...</rt-button> is not clickable at point (746, 717). Other element would receive the click: <div class="load-more-container">...</div>
  (Session info: chrome=131.0.6778.86)
Stacktrace:
    GetHandleVerifier [0x00007FF7AE046CB5+28821]
    (No symbol) [0x00007FF7ADFB3840]
    (No symbol) [0x00007FF7ADE5578A]
    (No symbol) [0x00007FF7ADEB0E8E]
    (No symbol) [0x00007FF7ADEAE92C]
    (No symbol) [0x00007FF7ADEABAF6]
    (No symbol) [0x00007FF7ADEAAA51]
    (No symbol) [0x00007FF7ADE9CBA0]
    (No symbol) [0x00007FF7ADECF2FA]
    (No symbol) [0x00007FF7ADE9C3F6]
    (No symbol) [0x00007FF7ADECF510]
    (No symbol) [0x00007FF7ADEEF412]
    (No symbol) [0x00007FF7ADECF0A3]
    (No symbol) [0x00007FF7ADE9A778]
    (No symbol) [0x00007FF7ADE9B8E1]
    GetHandleVerifier [0x00007FF7AE37FCAD+3408013]
    GetHandleVerifier [0x00007FF7AE39741F+3504127]
    GetHandleVerifier [0x00007FF7AE38B5FD+3455453]
    GetHandleVerifier [0x00007FF7AE10BDBB+835995]
    (No symbol) [0x00007FF7ADFBEB5F]
    (No symbol) [0x00007FF7ADFBA814]
    (No symbol) [0x00007FF7ADFBA9AD]
    (No symbol) [0x00007FF7ADFAA199]
    BaseThreadInitThunk [0x00007FFCE1227374+20]
    RtlUserThreadStart [0x00007FFCE155CC91+33]

Scraped 678 reviews and saved to './data/disneys_aladdin.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/1063452-pocahontas/reviews?type=user",
               output_csv_path="./data/1063452-pocahontas.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=57817): Read timed out. (read timeout=120)
Scraped 518 reviews and saved to './data/1063452-pocahontas.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/mulan/reviews?type=user",
               output_csv_path="./data/mulan.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=58957): Read timed out. (read timeout=120)
Scraped 938 reviews and saved to './data/mulan.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/1196003-princess_and_the_frog/reviews?type=user",
               output_csv_path="./data/1196003-princess_and_the_frog.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
Clicked 'Load More' button 47 times.
Clicked 'Load More' button 48 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=1972): Read timed out. (read timeout=120)
Scraped 979 reviews and saved to './data/1196003-princess_and_the_frog.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/tangled/reviews?type=user",
               output_csv_path="./data/tangled.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
Clicked 'Load More' button 47 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=3984): Read timed out. (read timeout=120)
Scraped 960 reviews and saved to './data/tangled.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/brave/reviews?type=user",
               output_csv_path="./data/brave.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=6048): Read timed out. (read timeout=120)
Scraped 658 reviews and saved to './data/brave.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/frozen_2013/reviews?type=user",
               output_csv_path="./data/frozen_2013.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
Clicked 'Load More' button 47 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=7049): Read timed out. (read timeout=120)
Scraped 958 reviews and saved to './data/frozen_2013.csv'
Show Code
scrape_reviews("https://www.rottentomatoes.com/m/moana_2016/reviews?type=user",
               output_csv_path="./data/moana_2016.csv")
Clicked 'Load More' button 1 times.
Clicked 'Load More' button 2 times.
Clicked 'Load More' button 3 times.
Clicked 'Load More' button 4 times.
Clicked 'Load More' button 5 times.
Clicked 'Load More' button 6 times.
Clicked 'Load More' button 7 times.
Clicked 'Load More' button 8 times.
Clicked 'Load More' button 9 times.
Clicked 'Load More' button 10 times.
Clicked 'Load More' button 11 times.
Clicked 'Load More' button 12 times.
Clicked 'Load More' button 13 times.
Clicked 'Load More' button 14 times.
Clicked 'Load More' button 15 times.
Clicked 'Load More' button 16 times.
Clicked 'Load More' button 17 times.
Clicked 'Load More' button 18 times.
Clicked 'Load More' button 19 times.
Clicked 'Load More' button 20 times.
Clicked 'Load More' button 21 times.
Clicked 'Load More' button 22 times.
Clicked 'Load More' button 23 times.
Clicked 'Load More' button 24 times.
Clicked 'Load More' button 25 times.
Clicked 'Load More' button 26 times.
Clicked 'Load More' button 27 times.
Clicked 'Load More' button 28 times.
Clicked 'Load More' button 29 times.
Clicked 'Load More' button 30 times.
Clicked 'Load More' button 31 times.
Clicked 'Load More' button 32 times.
Clicked 'Load More' button 33 times.
Clicked 'Load More' button 34 times.
Clicked 'Load More' button 35 times.
Clicked 'Load More' button 36 times.
Clicked 'Load More' button 37 times.
Clicked 'Load More' button 38 times.
Clicked 'Load More' button 39 times.
Clicked 'Load More' button 40 times.
Clicked 'Load More' button 41 times.
Clicked 'Load More' button 42 times.
Clicked 'Load More' button 43 times.
Clicked 'Load More' button 44 times.
Clicked 'Load More' button 45 times.
Clicked 'Load More' button 46 times.
Clicked 'Load More' button 47 times.
No more 'Load More' button found or an error occurred: HTTPConnectionPool(host='localhost', port=7021): Read timed out. (read timeout=120)
Scraped 960 reviews and saved to './data/moana_2016.csv'

Part 2: IMDBPY Movie Details Data

- includes: movie release year, director, casts, IMDB rating, movie plot, box offices etc. 

In the following code, we utilized the Cinemagoer library to programmatically retrieve detailed information about Disney Princess movies from IMDb. We began by creating an IMDb instance to facilitate movie searches and data retrieval. A predefined list of Disney Princess movies with their respective release years was used to ensure accurate matching. For each movie in the list, we searched IMDb’s database and identified the correct movie by strictly matching both the title and release year. Once a match was found, we used the movie’s unique ID to fetch detailed information, which included metadata such as genres, directors, cast members, ratings, and votes.

The retrieved movie details were then structured into a dictionary for each title, extracting relevant fields such as top cast members, runtime, plot summary, box office performance, and production details like countries and languages. This structured data was converted into a Pandas DataFrame for further analysis and visualization. Finally, the DataFrame was exported as a CSV file, allowing for easy integration into subsequent stages of our analysis pipeline. This process efficiently automated the extraction of rich metadata for Disney Princess movies, providing a solid foundation for further exploration and insights.

Show Code
from imdb import Cinemagoer
# Create an IMDb instance to search and retrieve movie data:
ia = Cinemagoer()
Show Code
# List of movies and their release years
movies = [
    {"title": "Snow White and the Seven Dwarfs", "year": 1937},
    {"title": "Cinderella", "year": 1950},
    {"title": "Sleeping Beauty", "year": 1959},
    {"title": "The Little Mermaid", "year": 1989},
    {"title": "Beauty and the Beast", "year": 1991},
    {"title": "Aladdin", "year": 1992},
    {"title": "Pocahontas", "year": 1995},
    {"title": "Mulan", "year": 1998},
    {"title": "The Princess and the Frog", "year": 2009},
    {"title": "Tangled", "year": 2010},
    {"title": "Brave", "year": 2012},
    {"title": "Frozen", "year": 2013},
    {"title": "Frozen II", "year": 2019},  # Adding the second Frozen movie
    {"title": "Moana", "year": 2016},
]

# Dictionary to store movie data
movie_data = {}

# Loop through the movies to fetch data
for movie in movies:
    search_results = ia.search_movie(movie["title"])
    for result in search_results:
        # Match title and year strictly
        if (
            result["title"].lower() == movie["title"].lower()
            and "year" in result
            and int(result["year"]) == movie["year"]
        ):
            # Get detailed movie information
            movie_id = result.movieID
            movie_details = ia.get_movie(movie_id)
            movie_data[movie["title"]] = movie_details
            print(f"Retrieved details for {movie['title']} ({movie['year']})")
            break
Retrieved details for Snow White and the Seven Dwarfs (1937)
Retrieved details for Cinderella (1950)
Retrieved details for Sleeping Beauty (1959)
Retrieved details for The Little Mermaid (1989)
Retrieved details for Beauty and the Beast (1991)
Retrieved details for Aladdin (1992)
Retrieved details for Pocahontas (1995)
Retrieved details for Mulan (1998)
Retrieved details for The Princess and the Frog (2009)
Retrieved details for Tangled (2010)
Retrieved details for Brave (2012)
Retrieved details for Frozen (2013)
Retrieved details for Frozen II (2019)
Retrieved details for Moana (2016)
Show Code
import pandas as pd
# Extract relevant fields and create a DataFrame
movie_details_list = []

for title, details in movie_data.items():
    movie_details_list.append({
        "title": title,
        "year": details.get('year'),
        "genres": details.get('genres'),
        "directors": [director['name'] for director in details.get('directors', [])],
        "cast": [person['name'] for person in details.get('cast', [])[:5]],  # Top 5 cast members
        "rating": details.get('rating'),
        "votes": details.get('votes'),
        "plot": details.get('plot outline'),
        "box_office": details.get("box office", {}),
        "countries": details.get("countries", []),
        "languages": details.get("languages", []),
        "runtimes": details.get("runtimes", [])

    })

df_movies = pd.DataFrame(movie_details_list)
Show Code
# Save to CSV
df_movies.to_csv("../../data/raw-data/imdb_movie_details.csv", index=False)