Advertisement
elena1234

scrolling infinite pages with Selenium

Oct 5th, 2022 (edited)
1,017
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.36 KB | Source Code | 0 0
  1. job_titles_ = []
  2. informations_ = []
  3. companies_ = []
  4.  
  5. # Scrolling
  6. last_height = driver.execute_script('return document.documentElement.scrollTop')
  7. while True:
  8.     # Soup Object
  9.     soup = BeautifulSoup(driver.page_source, 'lxml')
  10.    
  11.     dates = soup.findAll('div', class_ = 'card-date')
  12.     for date in dates:
  13.         try:
  14.             dates_.append(date.text.split()[0])
  15.         except:
  16.             dates_.append('n/a')
  17.            
  18.     titles = soup.findAll('div', class_ = 'card-title')
  19.     for title in titles:
  20.         try:
  21.             job_titles_.append(title.text.replace('star','').strip())
  22.         except:
  23.             job_titles_.append('n/a')
  24.        
  25.     informations = soup.findAll('div', class_ = 'card-info')
  26.     for information in informations:
  27.         try:
  28.             informations_.append(information.text)
  29.         except:
  30.             informations_.append('n/a')
  31.        
  32.     companies = soup.findAll('div', class_ = 'secondary-text')
  33.     for company in companies:
  34.         try:
  35.             companies_.append(company.text)
  36.         except:
  37.             companies_.append('n/a')
  38.            
  39.     driver.execute_script("window.scrollBy(0, 2000);")
  40.     time.sleep(2)
  41.     new_height = driver.execute_script('return document.documentElement.scrollTop;')
  42.     if new_height == last_height:
  43.         break
  44.     last_height = new_height
  45.    
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement