How to avoid components pushing up/down on one another

I'm new to using Java Swing and am looking to produce a GUI that resembles something like this:

I am pretty close but having an issue where a JLabel pushes up on the grid design like so:

How can I avoid this happening? I place the grid into a GridBagLayout panel and the JLabel into another GridBag panel(plan on putting a JToolbar here too), then add them to the main frame.

GridBagLayout tileLayout = new GridBagLayout();
GridBagConstraints tileLayoutConstraints = new GridBagConstraints();
JPanel tilePanel = new JPanel(tileLayout);
JPanel selectionPanel = new JPanel(tileLayout);
TileButton[][] tileAccessMatrix = new TileButton[4][4];

public BoardUILayer() {
    setSize(1920, 1080);
    buildBoardTiles();
    buildResourcePrompt();
    add(selectionPanel, BorderLayout.SOUTH);
    add(tilePanel, BorderLayout.CENTER);
}
private void buildBoardTiles() {
    for (int r = 0; r < tileAccessMatrix.length; r++) {
        for (int c = 0; c < tileAccessMatrix[r].length; c++) {
            TileButton temp = new TileButton(r, c);
            tileAccessMatrix[r][c] = temp;
            tileLayoutConstraints.ipadx = 115;
            tileLayoutConstraints.ipady = 115;
            tileLayoutConstraints.gridx = r;
            tileLayoutConstraints.gridy = c;
            temp.addActionListener(temp);
            tilePanel.add(temp, tileLayoutConstraints);
        }
    }
}

public void buildResourcePrompt() {
    final JLabel resourceTextLabel = new JLabel("What resource would you like?");
    tileLayoutConstraints.ipadx = 0;
    tileLayoutConstraints.ipady = 400;
    tileLayoutConstraints.gridx = 50;
    tileLayoutConstraints.gridy = 50;
    selectionPanel.add(resourceTextLabel, tileLayoutConstraints);
}

I created a GUI based on your drawing. This was one of the most complex GUI's I have ever created in Swing. The challenge was keeping the JButton sizes constant.

Maybe a MigLayout would work better for you.

Oracle has a helpful tutorial, Creating a GUI With Swing. Skip the Netbeans section.

I created 11 inner JPanels to construct this GUI.

JFrame
    main JPanel
        inner JPanel
            turnButton JPanel
                turn JPanel
                buttonGrid JPanel
        resource JPanel
            resourceLabel JPanel
            resourceButton JPanel
    manual JPanel
        outerEastButton JPanel
            eastButton JPanel

Here's the complete runnable code.

import java.awt.BorderLayout;
import java.awt.Dimension;
import java.awt.FlowLayout;
import java.awt.Font;
import java.awt.GridLayout;

import javax.swing.BorderFactory;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JTextArea;
import javax.swing.SwingUtilities;

public class RandomGameView implements Runnable {

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new RandomGameView());
    }

    @Override
    public void run() {
        JFrame frame = new JFrame("Random Game");
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        
        frame.add(createMainPanel(), BorderLayout.CENTER);
        frame.add(createManualPanel(), BorderLayout.EAST);
        
        frame.pack();
        frame.setLocationByPlatform(true);
        frame.setVisible(true);
    }
    
    private JPanel createMainPanel() {
        JPanel panel = new JPanel(new BorderLayout());
        
        JPanel upperPanel = createTurnButtonPanel();
        JPanel lowerPanel = createResourcePanel();
        JPanel innerPanel = createUpperPanel(upperPanel, lowerPanel);
        panel.add(innerPanel, BorderLayout.NORTH);
        panel.add(lowerPanel, BorderLayout.SOUTH);
        
        return panel;
    }

    private JPanel createUpperPanel(JPanel upperPanel, JPanel lowerPanel) {
        Dimension upperSize = upperPanel.getPreferredSize();
        Dimension lowerSize = lowerPanel.getPreferredSize();
        
        JPanel panel = new JPanel(new FlowLayout(FlowLayout.CENTER, 0, 0));
        int difference = lowerSize.width - upperSize.width;
        int left = difference / 2;
        int right = difference - left;
        panel.setBorder(BorderFactory.createEmptyBorder(0, left, 0, right));
        panel.add(upperPanel);
        
        return panel;
    }
    
    private JPanel createTurnButtonPanel() {
        JPanel panel = new JPanel(new BorderLayout());
        
        panel.add(createTurnPanel(), BorderLayout.NORTH);
        panel.add(createButtonGrid(), BorderLayout.SOUTH);
        
        return panel;
    }
    
    private JPanel createTurnPanel() {
        JPanel panel = new JPanel(new FlowLayout());
        panel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
        
        Font font = panel.getFont().deriveFont(Font.BOLD, 36f);
        
        JLabel playerLabel = new JLabel("Player's Turn");
        playerLabel.setFont(font);
        panel.add(playerLabel);
        
        return panel;
    }
    
    private JPanel createButtonGrid() {
        JPanel panel = new JPanel(new GridLayout(0, 4, 2, 2));
        panel.setBorder(BorderFactory.createEmptyBorder(2, 2, 2, 2));
        
        JButton[] buttonArray = new JButton[16];
        Dimension buttonSize = new Dimension(100, 100);
        for (int index = 0; index < buttonArray.length; index++) {
            buttonArray[index] = new JButton("Empty");
            buttonArray[index].setPreferredSize(buttonSize);
            panel.add(buttonArray[index]);
        }
        
        return panel;
    }
    
    private JPanel createResourcePanel() {
        JPanel panel = new JPanel(new BorderLayout());
        
        panel.add(createResourceLabelPanel(), BorderLayout.NORTH);
        panel.add(createResourceButtonPanel(), BorderLayout.CENTER);
        
        return panel;
    }
    
    private JPanel createResourceLabelPanel() {
        JPanel panel = new JPanel(new FlowLayout());
        panel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
        
        Font font = panel.getFont().deriveFont(Font.BOLD, 36f);
        
        JLabel label = new JLabel("Select a Resource");
        label.setFont(font);
        panel.add(label);
        
        return panel;
    }
    
    private JPanel createResourceButtonPanel() {
        JPanel panel = new JPanel(new FlowLayout(FlowLayout.CENTER, 30, 5));
        panel.setBorder(BorderFactory.createEmptyBorder(5, 5, 5, 5));
        
        JButton[] resourceButton = new JButton[5];
        Dimension buttonSize = new Dimension(100, 100);
        for (int index = 0; index < resourceButton.length; index++) {
            resourceButton[index] = new JButton("Empty");
            resourceButton[index].setPreferredSize(buttonSize);
            panel.add(resourceButton[index]);
        }
        
        return panel;
    }
    
    private JPanel createManualPanel() {
        JPanel panel = new JPanel(new BorderLayout());
        panel.setBorder(BorderFactory.createEmptyBorder(10, 20, 300, 20));
        
        Font font = panel.getFont().deriveFont(Font.BOLD, 36f);
        
        JLabel label = new JLabel("Manual");
        label.setHorizontalAlignment(JLabel.CENTER);
        label.setFont(font);
        panel.add(label, BorderLayout.NORTH);
        
        JTextArea textArea = new JTextArea(5, 20);
        panel.add(textArea, BorderLayout.CENTER);
        
        JPanel eastButtonPanel = createEastButtonPanel();
        JPanel innerPanel = createOuterEastButtonPanel(textArea, eastButtonPanel);
        
        panel.add(innerPanel, BorderLayout.SOUTH);
        
        return panel;
    }

    private JPanel createOuterEastButtonPanel(JTextArea textArea, 
            JPanel eastButtonPanel) {
        Dimension eastButtonPanelSize = eastButtonPanel.getPreferredSize();
        Dimension textAreaSize = textArea.getPreferredSize();
        
        int difference = textAreaSize.width = eastButtonPanelSize.width;
        int left = difference / 2;
        int right = difference - left;
        
        JPanel panel = new JPanel(new FlowLayout(FlowLayout.CENTER, 0, 0));
        panel.setBorder(BorderFactory.createEmptyBorder(0, left, 0, right));
        panel.add(eastButtonPanel);
        
        return panel;
    }
    
    private JPanel createEastButtonPanel() {
        JPanel panel = new JPanel(new GridLayout(0, 2, 2, 2));
        panel.setBorder(BorderFactory.createEmptyBorder(2, 2, 2, 2));
        
        JButton[] resourceButton = new JButton[4];
        Dimension buttonSize = new Dimension(50, 50);
        for (int index = 0; index < resourceButton.length; index++) {
            resourceButton[index] = new JButton();
            resourceButton[index].setPreferredSize(buttonSize);
            panel.add(resourceButton[index]);
        }
        
        return panel;
    }

}