python – Extract flight arrival data from web

I’m trying to scrape arrival data from Flight Radar 24

My script takes extremely long time to scrape the data. Is there any way I can speed up the scraping process?

Here’s my script:

import time
from selenium import webdriver
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from import Options
import pandas as pd
import requests

def getArrivals():
    urls = {
    'Alor Island Airport': '',
    'Ambon Pattimura Airport': '',
    'Atambua Haliwen Airport': '',
    'Babo Airport': '',
    'Bajawa Turelelo Soa Airport': '',
    'Balikpapan Sepinggan Airport': '',
    'Banda Aceh International Airport': '',
    'Bandar Lampung Radin Inten II Airport': '',
    'Bandung Husein Sastranegara International Airport': '',
    'Banjarmasin Syamsudin Noor Airport': '',
    'Batam Hang Nadim Airport': '',
    'Batu Licin Airport': '',
    'Bau-Bau Betoambari Airport': '',
    'Bengkulu Fatmawati Soekarno Airport': '',
    'Biak Frans Kaisiepo Airport': '',
    'Bima Sultan Muhammad Salahudin Airport': '',
    'Blimbingsari Airport': '',
    'Buol Airport': '',
    'Dekai Nop Goliat Airport': '',
    'Denpasar Ngurah Rai International Airport': '',
    'Dumai Pinang Kampai Airport': '',
    'Ende H. Hasan Aroeboesman Airport': '',
    'Fakfak Torea Airport': '',
    'Gorontalo Jalaluddin Airport': '',
    'Gunung Sitoli Binaka Airport': '',
    'Jakarta Halim Perdanakusuma Airport': '',
    'Jakarta Soekarno Hatta International Airport': '',
    'Jambi Sultan Thaha Airport': '',
    'Jayapura Sentani Airport': '',
    'Jember Notohadinegoro Airport': '',
    'Kaimana Utarom Airport': '',
    'Kalimarau Airport': '',
    'Kebar Airport': '',
    'Kendari Haluoleo Airport': '',
    'Ketapang Airport': '',
    'Kotabaru Gusti Syamsir Alam Airport': '',
    'Kupang El Tari Airport': '',
    'Labuan Bajo Komodo Airport': '',
    'Labuha Oesman Sadik Airport': '',
    'Langgur Karel Sadsuitubun Airport': '',
    'Larantuka Gewayantana Airport': '',
    'Lewoleba Wunopito Airport': '',
    'Lhokseumawe Malikus Saleh Airport': '',
    'Lombok International Airport': '',
    'Lubuklinggau Silampari Airport': '',
    'Luwuk Bubung Airport': '',
    'Majalengka Kertajati International Airport': '',
    'Makassar Sultan Hasanuddin International Airport': '',
    'Malang Abdul Rachman Saleh Airport': '',
    'Malinau Kolonel RA Bessing Airport': '',
    'Mamuju Tampa Padang Airport': '',
    'Manado Sam Ratulangi International Airport': '',
    'Manokwari Rendani Airport': '',
    'Matak Tarempa Airport': '',
    'Maumere Frans Seda Airport': '',
    'Medan Kuala Namu International Airport': '',
    'Melangguane Airport': '',
    'Merauke Mopah International Airport': '',
    'Muara Bungo Airport': '',
    'Nabire Airport': '',
    'Nanga Pinoh Airport': '',
    'Nunukan Airport': '',
    'Oksibil Airport': '',
    'Padang Minangkabau International Airport': '',
    'Palangkaraya Tjilik Riwut Airport': '',
    'Palembang International Airport': '',
    'Palopo Lagaligo Airport': '',
    'Palu Mutiara Airport': '',
    'Pangkal Pinang Airport': '',
    'Pangkalan Bun Iskandar Airport': '',
    'Pekanbaru Sultan Syarif Kasim II Airport': '',
    'Pomala Airport': '',
    'Pontianak Supadio Airport': '',
    'Putussibau Pangsuma Airport': '',
    'Raha Sugimanuru Airport': '',
    'Ranai Airport': '',
    'Rengat Japura Airport': '',
    'Roti David C. Saudale Airport': '',
    'Ruteng Frans Sales Lega Airport': '',
    'Sabang Maimun Saleh Airport': '',
    'Samarinda AP Tumenggung Pranoto Airport': '',
    'Sampit Airport': '',
    'Saumlaki Mathilda Batlayeri Airport': '',
    'Selayar Islands H. Aroeppala Airport': '',
    'Semarang Achmad Yani International Airport': '',
    'Sibolga Ferdinand Lumban Tobing Airport': '',
    'Siborong-Borong Silangit Airport': '',
    'Sintang Airport': '',
    'Sorong Dominique Edward Osok Airport': '',
    'Sumbawa Besar Airport': '',
    'Sumenep Trunojoyo Airport': '',
    'Surabaya Juanda International Airport': '',
    'Surakarta Adisumarmo International Airport': '',
    'Tahuna Naha Airport': '',
    'Tambolaka Airport': '',
    'Tana Toraja Pongtiku Airport': '',
    'Tanahmerah Airport': '',
    'Tanjung Pandan Buluh Tumbang Airport': '',
    'Tanjung Pinang Raja Haji Fisabilillah Airport': '',
    'Tanjung Selor Tanjung Harapan Airport': '',
    'Tarakan Juwata International Airport': '',
    'Tasikmalaya Cibeureum Airport': '',
    'Ternate Babullah Airport': '',
    'Timika Airport': '',
    'Waingapu Mau Hau Airport': '',
    'Wamena Airport': '',
    'Yogyakarta Adisucipto International Airport': '',
    'Yogyakarta International Airport': ''
    options = Options()
    driver = webdriver.Chrome(options=options)
    driver = webdriver.Chrome()
    airport_name = ()
    times = ()
    dates = ()
    flightfrom = ()
    airlines = ()
    aircrafts = ()
    flight_status = ()
    abbrs = ()
    for key, value in urls.items():
        scroll_pause_time = 1 # You can set your own pause time. My laptop is a bit slow so I use 1 sec
        screen_height = driver.execute_script("return window.screen.height;")   # get the screen height of the web
        i = 1
        while True:
            # scroll one screen height each time
            driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))  
            i += 1
            # update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
            scroll_height = driver.execute_script("return document.body.scrollHeight;")  
            # Break the loop when the height we need to scroll to is larger than the total scroll height
            if (screen_height) * i > scroll_height:
        soup = BeautifulSoup(driver.page_source, "html.parser")
        time_div=soup.find_all('div',{'class':'col-xs-3 col-sm-3 p-xxs'})
        for a in time_div:
            time_span = a.find_all('span',{'class':'ng-binding'})
            for b in time_span:
                time_text = b.text.strip()
        date_tr=soup.find_all('tr',{'class':'hidden-xs hidden-sm ng-scope'})
        for d in date_tr:
            date = d.get('data-date')
        flight_td=soup.find_all('td',{'class':'p-l-s cell-flight-number'})
        for a in flight_td:
            aflight = a.find_all('a',{'class':'notranslate ng-binding'})
            for b in aflight:
                flight_text = b.get('title')
        asal = (link.get_text().strip() for link in soup.find_all("span", {"class": "hide-mobile-only ng-binding"}))
        for a in abbreviation:
            abbr_a = a.find_all('a',{'class':'fs-10 fbold notranslate ng-binding'})
            for b in abbr_a:
                abbr_text = b.text.strip()
        for a in airline_td:
            airline_a = a.find_all('a',{'class':'notranslate ng-binding'})
            for b in airline_a:
                airline = b.get('title')
        aircraft_td = soup.find_all('td')
        for a in aircraft_td:
            aircraft_span = a.find_all('span',{'class':'notranslate ng-binding'})
            for b in aircraft_span:
                aircraft = b.text.strip()
        status_td = soup.find_all('td',{'class':'ng-binding'})
        for a in status_td:
            status_span = a.find_all('span',{'class':'ng-binding'})
            for b in status_span:
                status = b.text.strip()
        airport_tmp = (key) * len(asal)

        df = pd.DataFrame()
        df("Dates") = dates
        df("Time") = times
        df("Flight") = flight
        df("From") = list(map(' '.join, zip(flightfrom, abbrs)))
        df("Airlines") = airlines
        df("Aircrafts") = aircrafts
        df("Status") = flight_status
        df("Airport") = airport_name