joelnazarene

scienCEOPEN

Mar 31st, 2019
87
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.11 KB | None | 0 0
  1. import time
  2. import bs4
  3. import re
  4. from selenium import webdriver
  5. from bs4 import BeautifulSoup as soup
  6. driver = webdriver.Firefox()
  7. url="https://www.scienceopen.com/search#('v'~3_'id'~''_'isExactMatch'~true_'context'~null_'kind'~77_'order'~0_'orderLowestFirst'~false_'query'~'"
  8.  
  9. b1=input("enter text \n")
  10. b2=b1.replace(" ","%20")
  11. b3="'_'filters'~!*)"
  12. #print(b2)
  13. print("\n\n")
  14. url=url+b2+b3
  15. driver.get(url)
  16. time.sleep(2)
  17. refs=driver.page_source
  18. pagesoup=soup(refs,"html.parser")
  19. #containers=pagesoup.findAll("div",{"class":'main-section'})
  20. containers=pagesoup.findAll("a",{"href":re.compile('/document.*')})
  21. #print(containers[0].a['href'])
  22.  
  23. i=0
  24. u="https://www.scienceopen.com"
  25. for container in containers :
  26.   try :
  27.     if(len(container.text)>20):
  28.         i=i+1
  29.        
  30.        # print(i)
  31.         if(i%2!=0):
  32.        
  33.            driver.get(u + container['href'])
  34.            refs=driver.page_source
  35.            s=soup(refs,"html.parser")
  36.            c=s.find("div",{"itemprop":"description"})
  37.            print(c.text)
  38.         print("\n")
  39.   except :
  40.         continue
  41. driver.close()
Add Comment
Please, Sign In to add comment