fun

# I just copied the workflow from

# https://stackoverflow.com/questions/1080411/retrieve-links-from-web-page-using-python-and-beautifulsoup

# What this file should do, is get all the links that are present in an HTML

# file I'm giving it, and then tell me all the links it could find. Those links

# are then searched and scraped in their entirety, and copied to a directory of

# my choice.

import httplib2

from bs4 import BeautifulSoup, SoupStrainer

import requests

import time

import os

from os.path import isfile, join

url_prefix = "https://verkiezingsuitgaven.vlaanderen.be/"

root_path = "/tmp/verkiezingen/"

base_html_files = [f for f in os.listdir(root_path) if isfile(join(root_path, f))]

for file_path in base_html_files:

    useful_links = []

    with open(root_path + file_path, 'r') as f:

        content = f.read()

        for link in BeautifulSoup(content, 'html.parser', parse_only=SoupStrainer('a')):

            if link.has_attr('href'):

                if link['href'].startswith(url_prefix) and link['href'] != url_prefix and "publieke-opvraging" not in link['href']:

                    useful_links.append(link['href'])

                    print(link['href'])

        #print(useful_links)

    # We have now collected all URLs that we're interested in. Next up is

    # creating the directory (if it doesn't already exist) for this

    # municipality/province/district, and add the HTML files (if they don't

    # already exist)

    subdir_name = file_path[:-5]  # remove trailing ".html"

    if subdir_name not in os.listdir(root_path):  # Create directory if not existing

        os.mkdir(root_path + subdir_name)

        print("CREATED!")

    for useful_link in useful_links:

        # Sleeping for 1 second as to not trigger the Cloudflare shit

        time.sleep(1)

        http = httplib2.Http()

        #_, response = http.request(useful_link)

        # Using a bogus header is necessary because they decided to use

        # Cloudflare which cakes its pants without one.

        header = {"user-agent":"Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/119.0"}

        response = requests.get(useful_link, headers=header).text

        print(f"VISITED {useful_link}")

        # Decide if this is a party expense or a candidate expense:

        if useful_link.count('/') == 3:  # Party expense

            party_name = useful_link.rpartition('/')[2]

            if party_name not in os.listdir(root_path + subdir_name):  # Create directory if not existing

                os.mkdir(root_path + subdir_name +'/'+ party_name)

            with open(root_path + subdir_name +'/'+ party_name +".html", 'w') as g:

                g.write(response)

        elif useful_link.count('/') == 4:  # Candidate expense

            cand_name = useful_link.split('/')[-1]

            party_name = useful_link.split('/')[-2]

            if party_name not in os.listdir(root_path + subdir_name):  # Create directory if not existing

                os.mkdir(root_path + subdir_name +'/'+ party_name)

            with open(root_path + subdir_name +'/'+ party_name +'/'+ cand_name +".html", 'w') as g:

                g.write(response)