I have written a Python script using Selenium and ChromeDriver to scrape data. The script navigates through several pages and clicks on various buttons to retrieve the data. However, I am encountering the following error:
WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"No node with given id found"}
The error seems to occur at a specific point in the iteration, rather than being random. I have tried to troubleshoot the issue, but I am not sure what is causing it or how to fix it.
I am using Python 3.10.5 and the Selenium library with ChromeDriver version 113.0.5672.63 on a Windows 10 machine. Any help with resolving this issue would be greatly appreciated.
I'm still a beginner and this is my first time trying selenium. I have tried adding time.sleep(1) to make sure the web is loaded, check the visibility of the element, and the element is clickable but the problem still occurs.
This is the current script that I have written
url = '.../'
path = Service(r'...\chromedriver_win32')
options = Options()
options.add_experimental_option("debuggerAddress", "localhost:9222")
driver = webdriver.Chrome(service=path, options=options)
driver.get(url)
wait = WebDriverWait(driver, 10)
def scrape_left_table(prob, kab, kec):
data = []
rows = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
for row in rows:
wilayah = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button').text
persentasi = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > span').text
class_1= row.find_element(By.CSS_SELECTOR, 'td:nth-child(2)').text
class_2= row.find_element(By.CSS_SELECTOR, 'td:nth-child(3)').text
data.append([prob, kab, kec, wilayah, persentasi, class_1, class_2])
return data
def scrape_right_table(prob, kab, kec):
data = []
rows = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(2) > table > tbody > tr')
for row in rows:
wilayah = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button').text
persentasi = row.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > span').text
class_1= row.find_element(By.CSS_SELECTOR, 'td:nth-child(2)').text
class_2= row.find_element(By.CSS_SELECTOR, 'td:nth-child(3)').text
data.append([prob, kab, kec, wilayah, persentasi, class_1, class_2])
return data
data = []
provinsi = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
button = provinsi[1].find_element(By.TAG_NAME, 'button')
pro = button.text
wait.until(EC.element_to_be_clickable(button)).click()
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')))
for i in [1,2]:
time.sleep(1)
kabupaten = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')
for kab in kabupaten:
time.sleep(1)
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))
kab_button = kab.find_element(By.TAG_NAME, 'button')
kab_name = kab_button.text
driver.execute_script("arguments[0].scrollIntoView();", kab_button)
driver.execute_script("arguments[0].click();", kab_button)
for i in [1,2]:
time.sleep(1)
kecamatan = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')
for kec in kecamatan:
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))
kec_button = kec.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button')
kec_name = kec_button.text
driver.execute_script("arguments[0].scrollIntoView();", kec_button)
driver.execute_script("arguments[0].click();", kec_button)
kelurahan = driver.find_elements(By.CSS_SELECTOR, 'div:nth-child(1) > table > tbody > tr')
time.sleep(1)
left_table = scrape_left_table(pro, kab_name, kec_name)
right_table = scrape_right_table(pro, kab_name, kec_name)
data += left_table + right_table
back = driver.find_element(By.CSS_SELECTOR, '#app > div.sticky-top.bg-white > div > div:nth-child(2) > div > div > div > div:nth-child(5) > div > div > div.vs__actions > button')
driver.execute_script("arguments[0].scrollIntoView();", back)
driver.execute_script("arguments[0].click();", back)
back = driver.find_element(By.CSS_SELECTOR, '#app > div.sticky-top.bg-white > div > div:nth-child(2) > div > div > div > div:nth-child(4) > div > div > div.vs__actions > button')
driver.execute_script("arguments[0].scrollIntoView();", back)
driver.execute_script("arguments[0].click();", back)
After a certain iteration i.e. for provinsi[0] errors occur after 689 iterations for provinsi[1] errors occur after 35 iterations.
WebDriverException Traceback (most recent call last)
c:\...\web_scraping.ipynb Cell 4 in ()
23 for kec in kecamatan:
24 wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, 'div:nth-child(' + str(i) + ') > table > tbody > tr')))
---> 26 kec_button = kec.find_element(By.CSS_SELECTOR, 'td.text-xs-left.wilayah-name > button')
27 kec_name = kec_button.text
28 driver.execute_script("arguments[0].scrollIntoView();", kec_button)
WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"No node with given id found"}