Changing the xoff coordinates of the mesh after changing the angle to pan in correct direction

I am struggling with setting up coordinates so that when you change the angle, the mesh moves like a mouse. So if I move the mouse up, the mesh also goes up and not at the angle I changed. I will be grateful for any hints. Thanks. This is my code:

let angle = 0.0;
let dim = 10
let sz;
let xoff;
let yoff;
function setup() {
  createCanvas(400, 400);
  sz = width/ dim;
}

function mouseDragged() {
  xoff += (mouseX - pmouseX)/10;
  yoff += (mouseY - pmouseY)/10;
}

function draw() {
  background(255);
  
  for (let i = 0; i < dim+2; i++) {
    for (let j = 0; j < dim+2; j++) {

      let x = ((pmouseX + j * sz) % (width+sz)) - sz;
      if (x < -sz) x += width+sz;

      let y = ((pmouseY + i * sz) % (height+sz)) - sz;
      if (y < -sz) y += height+sz;
push();
imageMode(CENTER);
rectMode(CENTER);
translate( width / 2  , height / 2  );
rotate(angle);
translate( -width / 2 , -height / 2  );
      rect(x, y, sz, sz);
      text(i * 10 + j, x + sz/2, y + sz/2);
      pop(); 
    }
  }
}

If I understand your problem correctly I think you can just transform the mouse position into the frame of reference that you are drawing in by rotating the mouse position vector in the reverse direction from the rotation you apply before drawing:

// let angle = 0.0;
let dim = 10;
let sz;
let xoff;
let yoff;

let angleSlider;

function setup() {
  createCanvas(400, 400);
  sz = width / dim;
  
  angleSlider = createSlider(-180, 180, 0);
  angleSlider.position(10, 10);
  
  angleMode(DEGREES);
}

function mouseDragged() {
  xoff += (mouseX - pmouseX) / 10;
  yoff += (mouseY - pmouseY) / 10;
}

function draw() {
  background(255);

  let mousePos = createVector(mouseX, mouseY).rotate(-angleSlider.value());
  for (let i = 0; i < dim + 2; i++) {
    for (let j = 0; j < dim + 2; j++) {
      let x = ((mousePos.x + j * sz) % (width + sz)) - sz;
      if (x < -sz) x += width + sz;

      let y = ((mousePos.y + i * sz) % (height + sz)) - sz;
      if (y < -sz) y += height + sz;
      push();
      imageMode(CENTER);
      rectMode(CENTER);
      translate(width / 2, height / 2);
      rotate(angleSlider.value());
      translate(-width / 2, -height / 2);
      rect(x, y, sz, sz);
      text(i * 10 + j, x + sz / 2, y + sz / 2);
      pop();
    }
  }
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/p5.js"></script>