java selenium web-scraping htmlunit-driver

Selenium 4 HtmlUnitDriver Button not clicked

I'm using Selenium 4 and HtmlUnitDriver with java, I have a problem with executing a form button. no error showed but I'm sure that the button was not clicked according to the output console result.

I add some system output to know the result of some elements :

This is my class :

package htmldriver;

import com.gargoylesoftware.htmlunit.BrowserVersion;

import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeUnit;

import org.openqa.selenium.*;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

public class htmlUnitYest {

    public static void main(String[] args) throws InterruptedException {

        // turn off htmlunit warnings
        java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(java.util.logging.Level.OFF);
        java.util.logging.Logger.getLogger("org.apache.http").setLevel(java.util.logging.Level.OFF);

        WebDriver driver = new HtmlUnitDriver(BrowserVersion.CHROME, true);

        driver.get("http://www.mahakim.ma/Ar/Annonces/RechercherAnnonces/?Page=AnnoncesJudiciaires#");

        // This code will print the page title
        System.out.println("Page title is: " + driver.getTitle());

        WebElement startDate = ((HtmlUnitDriver) driver).findElementById("txtDateD");
        WebElement endDate = ((HtmlUnitDriver) driver).findElementById("txtDateF");
        WebElement searchButton = ((HtmlUnitDriver) driver).findElementById("btnRechercheAnnByDateTypeJur");

        WebElement tabList = ((HtmlUnitDriver) driver).findElementByXPath("//ul[@role=\"tablist\" and @class=\"nav nav-pills TypeJuridiction\"]");
        List<WebElement> choices = ((HtmlUnitDriver) driver).findElementsByCssSelector("#TypeJuridiction .TypeJuridiction li");

        WebElement resultTab = ((HtmlUnitDriver) driver).findElementById("Res");
        WebElement resultDetails = ((HtmlUnitDriver) driver).findElementById("Res_Detail");

        startDate.sendKeys("14/03/2022");
        endDate.sendKeys("21/03/2022");

        int position = 1;
        for (WebElement choiceType : choices) {

            position += 1;

            if ( position < 3 ) {
                continue;
            }

            System.out.println("choiceType: " + choices);
            System.out.println("searchButton: " + searchButton);

            WebDriverWait wait = new WebDriverWait(driver,Duration.ofSeconds(20));

            wait.until(ExpectedConditions.elementToBeClickable(choiceType));

            ((HtmlUnitDriver) driver).findElementById("btnRechercheAnnByDateTypeJur").click();
            
            driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(20));

            WebElement ResDiv = ((HtmlUnitDriver) driver).findElementById("Res");

            List<WebElement> ColsInResDiv = ((HtmlUnitDriver) driver).findElementsByClassName("col-lg-12");

            System.out.println("resultTab: " + ColsInResDiv);

            driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(10));

            // when search button clicked this element will be exist and style of second div.col-lg-12 will become : display inline
            List<WebElement> da = ((HtmlUnitDriver) driver).findElementsByCssSelector("#Res .DetailAnnonce");

            System.out.println("da: " + da);

            List<WebElement> results = ((HtmlUnitDriver) driver).findElementsByXPath(".//a[@class=\"DetailAnnonce\"]");


            for (WebElement item : results) {

                System.out.println("item: " + item);

                WebDriverWait waitArticle = new WebDriverWait(driver,Duration.ofSeconds(20));

                waitArticle.until(ExpectedConditions.elementToBeClickable((item)));
                
                List<WebElement> details = resultDetails.findElements(By.xpath(".//a[@class=\"link-pointer\"]"));

                for (WebElement article : details) {

                    System.out.println("article: " + article.toString());

                    Thread.sleep(1);

                    waitArticle.until(ExpectedConditions.elementToBeClickable((article)));

                    WebElement downloadFile = waitArticle.until(ExpectedConditions.elementToBeClickable(((HtmlUnitDriver) driver).findElementByXPath("//a[@class=\"DownFiles\"]")));

                    downloadFile.click();

                    WebElement closeModel = waitArticle.until(ExpectedConditions.elementToBeClickable(((HtmlUnitDriver) driver).findElementByXPath("//div[@id=\"ModalPJ\"]//button[@data-dismiss=\"modal\"]")));

                    closeModel.click();


                }

            }

            position++;
        }

        // This code will print the page title
        System.out.println("Page title is: " + driver.getTitle());

        driver.quit();
    }
}

This is the output of system.out. in the console:

"C:\Program Files\Java\jdk1.8.0_171\bin\java.exe ..." 
Page title is: محاكم
choiceType: [<li role="presentation" class="active">, <li role="presentation">, <li role="presentation">]
searchButton: <button id="btnRechercheAnnByDateTypeJur" class="btn btn-primary">
resultTab: [<div class="col-lg-12">, <div class="col-lg-12" style="display: none;">, <div class="col-lg-12 well well-sm" id="Res_Detail">, <div class="footer_sepHoriz col-lg-12">]
da: []
choiceType: [<li role="presentation" class="active">, <li role="presentation">, <li role="presentation">]
searchButton: <button id="btnRechercheAnnByDateTypeJur" class="btn btn-primary">
resultTab: [<div class="col-lg-12">, <div class="col-lg-12" style="display: none;">, <div class="col-lg-12 well well-sm" id="Res_Detail">, <div class="footer_sepHoriz col-lg-12">]
da: []
Page title is: محاكم

Process finished with exit code 0

And this is a screenshot of the HTML page when I click on the search button manually:

PS: From the browser, when I click on the search button, it is executed via jquery like this $("#btnRechercheAnnByDateTypeJur").click(function (e) {... }, then an ajax script is called to get data from the server if that could help.

EDIT:

my pom.xml file:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>webscrapping</groupId>
    <artifactId>cap</artifactId>
    <version>1.0-SNAPSHOT</version>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>

        <dependency>
            <groupId>org.seleniumhq.selenium</groupId>
            <artifactId>selenium-java</artifactId>
            <version>2.53.1</version>
        </dependency>

    </dependencies>


</project>

Solution

I've checked your code and when you click it doesn't seem to load javascript properly. Javascript with htmlunit are not very friendly ;).

You can do it easy with playwright 1.20.0 (how to set playwright https://playwright.dev/java/docs/intro). Just add in your pom:

    <dependency>
        <groupId>com.microsoft.playwright</groupId>
        <artifactId>playwright</artifactId>
        <version>1.20.0</version>
    </dependency>

Its works, start downloading files, but after some downloads get stacked and can not open the modal giving an error timeout for waiting this element .DownFiles.
Tomorrow i will check if i can find where is the problem.

public static void main(String[] args) throws InterruptedException {

    try (Playwright playwright = Playwright.create()) {
        final BrowserType chromium = playwright.chromium();
        //you can do it with headless mode or with open browser
        final Browser browser = chromium.launch(new BrowserType
                .LaunchOptions()
                .setHeadless(false));
        final Page page = browser.newPage();
        page.navigate("http://www.mahakim.ma/Ar/Annonces/RechercherAnnonces/?Page=AnnoncesJudiciaires#");
        ElementHandle searchButton = page.querySelector("#btnRechercheAnnByDateTypeJur");
        ElementHandle tabList = page.querySelector("//ul[@role=\"tablist\" and @class=\"nav nav-pills TypeJuridiction\"]");
        List<ElementHandle> choices = page.querySelectorAll("#TypeJuridiction .TypeJuridiction li");
        //start date
        page.fill("#txtDateD", "14/03/2022");
        //end date
        page.fill("#txtDateF", "21/03/2022");
        page.querySelector("div[id='TypeJuridiction'] li:nth-child(3)").click();
        page.querySelector("#btnRechercheAnnByDateTypeJur").click();
        Thread.sleep(3000);
        page.querySelector(".DetailAnnonce").click();
        Thread.sleep(3000);
        List<ElementHandle> results = page.querySelectorAll(".DetailAnnonce-Plus");
        for (ElementHandle item : results) {
            //click for open modal
            try {
                item.querySelector(".link-pointer").click();
                page.waitForSelector(".DownFiles");
            } catch (Exception e) {
                System.out.println("Timeout can not open modal " + item.innerHTML());
                Thread.sleep(1000);
                continue;
            }
            // Wait for the download to start
            Download download = page.waitForDownload(() -> {
                // Perform the action that initiates download
                page.click(".DownFiles");
            });
            // Wait for the download process to complete
            Path path = download.path();
            System.out.println("Downloading the file of: " + path + System.lineSeparator()
                    + "item: " + item.innerHTML());
            Thread.sleep(2000);
            //close modal
            page.querySelector(".btn.btn-default").click();
        }
        System.out.println("Total downloaded files: " + results.size());
    }
}

Added 23/03/2022

Solved the problem with popup, change it to firefox(headless) and it works well.

final BrowserType chromium = playwright.firefox();
final Browser browser = chromium.launch();