import pandas as pd 
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys 
from selenium.webdriver.support.ui import WebDriverWait
import time

url='https://help.sap.com/glossary/'
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
options.add_argument('--headless')
driver = webdriver.Chrome()
driver.get(url)
html=driver.page_source

url='https://help.sap.com/glossary/'
driver=webdriver.Chrome()
driver.get(url)
time.sleep(10)
html=driver.page_source
column_names = ["Menu_item", "Category", "Description"]
pd.DataFrame(columns=column_names).to_csv("SAP_DATA.csv", index=False, sep="\t", encoding="UTF-8")
doc = BeautifulSoup(html,'html')


# def click_more():
#     myelement = driver.find_element_by_xpath('//*[@id="load-more-button"]')
#     if myelement.is_displayed():
#         myelement.click()
#         time.sleep(3)
#     else:
#         print("All Xpath button clicked")

def main_data():
    myelement = driver.find_element_by_xpath('//*[@id="load-more-button"]')
    if myelement.is_displayed():
        myelement.click()
        time.sleep(3)
    else:
        print("All Xpath button clicked")
    html=driver.page_source
    doc = BeautifulSoup(html,'html')
    for x in doc:
        links = driver.find_elements_by_tag_name('a[data-v-7be238b0]')
        visited=set()
        links_clicked=[]
        for x in range(0,len(links)):
            if links[x].is_displayed():
                links[x].click()
                for e in links:
                    if e not in visited:
                        visited.add(e)
                        e.click()
                    else:
                        print("link is visited",links)
                # html=driver.page_source
                # link_data=BeautifulSoup(html,'html')
                # Sap_data=dict()
                # for element in link_data.select('[id^="details"]'):
                #     main_element=element.select_one('[id^="content"]')
                #     if(main_element):
                #         Sap_data["Menu_item"]=main_element.select_one('h2').text
                #         Sap_data["Category"]=main_element.select_one('samp').text
                #         Sap_data["Description"]=main_element.select_one('p').text
                


            # temp_data = []
            # result_data = dict()
            # for key, val in Sap_data.items():
            #     if val not in temp_data:
            #         temp_data.append(val)
            #         result_data[key] = val

            # result_list = []
            # for i in temp_data:
            #     if i not in result_list:
            #         result_list.append(i)        
                
            print(links_clicked)
            # pd.DataFrame([Sap_data], columns=column_names).to_csv("SAP_DATA.csv", index=False, header=False, sep="\t", encoding="UTF-8", mode="a")           


while True:
    main_data()
    print(html)

def scraping(html):
    html=driver.page_source 
    column_names = ["Menu-Item"]
    pd.DataFrame(columns=column_names).to_csv("data.csv", index=False, sep="\t", encoding="UTF-8")
    doc = BeautifulSoup(html,'html')
    for element in doc.select('[class^="sidebar-content"]'):
        ul_data= element.select_one('[id^="terms-ul"]')
        if(ul_data):
            temp_list=ul_data.select('li')
            list_data=[]
            for data in temp_list:
                list_data.append(data.text.split("\n"))

            result_list = []
            for i in list_data:
                if i not in result_list:
                    result_list.append(i)
            pd.DataFrame(result_list, columns=column_names).to_csv("data.csv", index=False, header=False, sep="\t", encoding="UTF-8", mode="a")

def click_more():
    myelement = driver.find_element_by_xpath('//*[@id="load-more-button"]')
    if myelement.is_displayed():
        myelement.click()
        
    else:
        print("All Xpath button clicked")

try:
    driver=webdriver.Chrome()
    driver.get('https://help.sap.com/glossary/')
    while True:
        time.sleep(5)
        click_more()
        html=driver.page_source
        scraping(html)
        main_data()
      
finally:
    driver.quit()
    # myelement = driver.find_element_by_xpath('//*[@id="load-more-button"]')
    # if myelement.is_displayed() == None:
    #     main_data()

driver=webdriver.Chrome()
driver.get('https://help.sap.com/glossary/')
click_more()