I'm trying to paint a Windows 11 button on my JFrame. That worked well, but I can't properly add actionlisteners to it. Here is my code:

public class MyButton extends JButton implements MouseListener, ActionListener {
    public MyButton(String text) {
        //super(label);
        enableInputMethods(true);
        addMouseListener(this);

        setFocusPainted(false);
        setBorderPainted(false);

        buttonText = text;
    }

    @Override
    protected void paintComponent(Graphics g) {
        //super.paintComponent(g);

        Graphics2D g2 = (Graphics2D) g;
        FontMetrics metrics = g2.getFontMetrics(Constants.arialFont);
        g2.setStroke(new BasicStroke(2.0f));
        g2.setFont(Constants.arialFont);

        g2.setPaint(Constants.buttonNormal);
        rect = new RoundRectangle2D.Double(0, 0, metrics.stringWidth(buttonText) + 20, 27, 10, 10);
        g2.fill(rect);

        g2.setPaint(Constants.fontColor);
        g2.drawString(buttonText, Functions.getMiddleFromX(g2, rect, buttonText), Functions.getMiddleFromY(g2, rect, buttonText));
    }

I have tried to add mouse listeners to it as follows:

    @Override
    public void mouseClicked(MouseEvent e) {
        // Check if the click is on the button.
        Point p = e.getPoint();
        if(rect.contains(p)) System.out.println("Triangle contains point");
        else { return; }
    }

But when I run this class with this code:

        MyButton myButton = new MyButton("Epic button!");
        myButton.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {
                System.out.println("Action Performed!");
            }
        });
        frame.add(myButton);

I can click anywhere on the canvas (or JFrame) and still see "Action Performed!" printed on the console.

How would I filter the incoming events and only do stuff if the click is on my RoundRectangle2D.Double()?

Proof of concept

This is intended as a proof of concept based on the limited understand of the requirements and is not intended to be a complete, production ready solution - just saying

This is somewhat of an assumption on my part, but, it seems you what to have some kind of "shape based" button which can only be triggered by the mouse if it's over the "shape" in question.

This is somewhat more involved, as the only realistic way to modify the mouse handling is via the buttons ButtonUI delegate.

The following example basically creates a custom ButtonUI which presents a round button, but will only be triggered in the user presses within inside the shape of the circle itself. Of course, the user can still trigger the button via the keyboard, but that's another issue entirely (but is actually solved through the same mechanisms)

It has default icon support!

Please note, the red rectangle shows the physical bounds of the button itself and are only for demonstration purpose

import java.awt.Color;
import java.awt.Dimension;
import java.awt.EventQueue;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.GridBagLayout;
import java.awt.LinearGradientPaint;
import java.awt.Point;
import java.awt.Rectangle;
import java.awt.RenderingHints;
import java.awt.Shape;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.geom.Ellipse2D;
import javax.swing.AbstractButton;
import javax.swing.ButtonModel;
import javax.swing.JButton;
import javax.swing.JComponent;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.plaf.basic.BasicButtonUI;

public class Main {
    public static void main(String[] args) {
        new Main();
    }

    public Main() {
        EventQueue.invokeLater(new Runnable() {
            @Override
            public void run() {
                JFrame frame = new JFrame();
                frame.add(new TestPane());
                frame.pack();
                frame.setLocationRelativeTo(null);
                frame.setVisible(true);
            }
        });
    }

    public class TestPane extends JPanel {

        public TestPane() {
            setLayout(new GridBagLayout());
            setBackground(Color.RED);

            JButton circleButton = new JButton("Hello");
            circleButton.setUI(new CircleButtonUI());
            circleButton.addActionListener(new ActionListener() {
                @Override
                public void actionPerformed(ActionEvent e) {
                    System.out.println("I've been triggered");
                }
            });

            add(circleButton);
        }
    }

    public class CircleButtonUI extends BasicButtonUI {

        private Shape circleShape;

        @Override
        public void installUI(JComponent c) {
            super.installUI(c);
            c.setOpaque(false);
        }

        @Override
        public boolean contains(JComponent c, int x, int y) {
            if (circleShape == null) {
                return c.contains(x, y);
            }
            return circleShape.contains(x, y);
        }

        @Override
        public void paint(Graphics g, JComponent c) {
            int width = c.getWidth() - 2;
            int height = c.getHeight() - 2;
            int size = Math.min(width, height);
            int x = ((width - size) / 2) + 1;
            int y = ((height - size) / 2) + 1;
            circleShape = new Ellipse2D.Double(x, y, size, size);
            AbstractButton b = (AbstractButton) c;
            paintContent(g, b);
            super.paint(g, c);
        }

        protected void paintContent(Graphics g, AbstractButton b) {
            if (circleShape == null) {
                return;
            }
            Graphics2D g2d = (Graphics2D) g.create();
            // paint the interior of the button
            g2d.setRenderingHint(RenderingHints.KEY_ALPHA_INTERPOLATION, RenderingHints.VALUE_ALPHA_INTERPOLATION_QUALITY);
            g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
            g2d.setRenderingHint(RenderingHints.KEY_COLOR_RENDERING, RenderingHints.VALUE_COLOR_RENDER_QUALITY);
            g2d.setRenderingHint(RenderingHints.KEY_DITHERING, RenderingHints.VALUE_DITHER_ENABLE);
            g2d.setRenderingHint(RenderingHints.KEY_FRACTIONALMETRICS, RenderingHints.VALUE_FRACTIONALMETRICS_ON);
            g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
            g2d.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY);
            g2d.setRenderingHint(RenderingHints.KEY_STROKE_CONTROL, RenderingHints.VALUE_STROKE_PURE);

            ButtonModel model = b.getModel();
            Color highlight = b.getBackground();
            if (model.isArmed() && model.isPressed()) {
                highlight = highlight.darker();
            }
            Color darklight = highlight.darker();

            LinearGradientPaint lgp = new LinearGradientPaint(
                    circleShape.getBounds().getLocation(),
                    new Point((int) circleShape.getBounds().getMaxX(), (int) circleShape.getBounds().getMaxY()),
                    new float[]{0, 1f},
                    new Color[]{highlight, darklight});

            g2d.setPaint(lgp);
            g2d.fill(circleShape);

            // draw the perimeter of the button
            g2d.setColor(b.getBackground().darker().darker().darker());
            g2d.draw(circleShape);
            g2d.dispose();
        }

        @Override
        protected void paintFocus(Graphics g, AbstractButton b, Rectangle viewRect, Rectangle textRect, Rectangle iconRect) {
            // Paint focus highlight, if you want to
        }

        public Dimension getMinimumSize(JComponent c) {
            Dimension size = super.getMinimumSize(c);
            int maxSize = Math.max(size.width, size.height);
            return new Dimension(maxSize, maxSize);
        }

        public Dimension getPreferredSize(JComponent c) {
            Dimension size = super.getPreferredSize(c);
            int maxSize = Math.max(size.width, size.height);
            return new Dimension(maxSize, maxSize);
        }

        public Dimension getMaximumSize(JComponent c) {
            Dimension size = super.getPreferredSize(c);
            int maxSize = Math.max(size.width, size.height);
            return new Dimension(maxSize, maxSize);
        }
    }
}

More then one way to approach the problem...

As with most things, there's always more then one way to approach the problem.

This example just extends a JButton. The critical area is in overriding the contains method.

Personally, I like the ButtonUI approach, as it's relatively easy to introduce into an existing code base (arguably) and you don't end up with some weird result because the installed look and feel is doing something "different", just saying.

public class CircleButton extends JButton {

    private Shape circleShape;

    public CircleButton() {
        configureDefaults();
    }

    public CircleButton(Icon icon) {
        super(icon);
        configureDefaults();
    }

    public CircleButton(String text) {
        super(text);
        configureDefaults();
    }

    public CircleButton(Action a) {
        super(a);
        configureDefaults();
    }

    public CircleButton(String text, Icon icon) {
        super(text, icon);
        configureDefaults();
    }
    
    protected void configureDefaults() {
        setBorderPainted(false);
        setFocusPainted(false);
        setOpaque(false);
    }

    @Override
    public void invalidate() {
        super.invalidate();
        circleShape = null;
    }

    @Override
    public boolean contains(int x, int y) {
        if (circleShape == null) {
            return super.contains(x, y);
        }
        return circleShape.contains(x, y);
    }

    public Dimension getMinimumSize() {
        Dimension size = super.getMinimumSize();
        int maxSize = Math.max(size.width, size.height);
        return new Dimension(maxSize, maxSize);
    }

    public Dimension getPreferredSize() {
        Dimension size = super.getPreferredSize();
        int maxSize = Math.max(size.width, size.height);
        return new Dimension(maxSize, maxSize);
    }

    public Dimension getMaximumSize() {
        Dimension size = super.getPreferredSize();
        int maxSize = Math.max(size.width, size.height);
        return new Dimension(maxSize, maxSize);
    }

    @Override
    protected void paintComponent(Graphics g) {
        if (circleShape == null) {
            int width = getWidth() - 2;
            int height = getHeight() - 2;
            int size = Math.min(width, height);
            int x = ((width - size) / 2) + 1;
            int y = ((height - size) / 2) + 1;
            circleShape = new Ellipse2D.Double(x, y, size, size);
        }
        Graphics2D g2d = (Graphics2D) g.create();
        // paint the interior of the button
        g2d.setRenderingHint(RenderingHints.KEY_ALPHA_INTERPOLATION, RenderingHints.VALUE_ALPHA_INTERPOLATION_QUALITY);
        g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
        g2d.setRenderingHint(RenderingHints.KEY_COLOR_RENDERING, RenderingHints.VALUE_COLOR_RENDER_QUALITY);
        g2d.setRenderingHint(RenderingHints.KEY_DITHERING, RenderingHints.VALUE_DITHER_ENABLE);
        g2d.setRenderingHint(RenderingHints.KEY_FRACTIONALMETRICS, RenderingHints.VALUE_FRACTIONALMETRICS_ON);
        g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
        g2d.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY);
        g2d.setRenderingHint(RenderingHints.KEY_STROKE_CONTROL, RenderingHints.VALUE_STROKE_PURE);

        ButtonModel model = getModel();
        Color highlight = getBackground();
        if (model.isArmed() && model.isPressed()) {
            highlight = highlight.darker();
        }
        Color darklight = highlight.darker();

        LinearGradientPaint lgp = new LinearGradientPaint(
                circleShape.getBounds().getLocation(),
                new Point((int) circleShape.getBounds().getMaxX(), (int) circleShape.getBounds().getMaxY()),
                new float[]{0, 1f},
                new Color[]{highlight, darklight});

        g2d.setPaint(lgp);
        g2d.fill(circleShape);

        // draw the perimeter of the button
        g2d.setColor(getBackground().darker().darker().darker());
        g2d.draw(circleShape);
        g2d.dispose();
        super.paintComponent(g);
    }
}