I'm working on 3d app in libgdx engine.
I just figured out that decalBatch isn't drawing into stencil buffer. I wanned to make stencil masks for 3d world, and it's not working at all.
This is the code which works for sprite batch, but it's not working with decal batch. Pls help!
Gdx.gl.glClearColor(1, 1, 1, 1);
Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_STENCIL_BUFFER_BIT);
// batch.setProjectionMatrix(camera.combined);
// setup drawing to stencil buffer
Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);
Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
Gdx.gl20.glStencilOp(GL20.GL_REPLACE, GL20.GL_REPLACE, GL20.GL_REPLACE);
Gdx.gl20.glColorMask(false, false, false, false);
// draw base pattern
shapeRenderer.begin(ShapeType.Filled);
shapeRenderer.identity();
shapeRenderer.setColor(1f, 0f, 0f, 0.5f);
shapeRenderer.rect(Gdx.graphics.getWidth(), Gdx.graphics.getHeight(), 100, 100);
shapeRenderer.end();
spriteBatch.begin();
// fix stencil buffer, enable color buffer
Gdx.gl20.glColorMask(true, true, true, true);
Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);
// draw where pattern has NOT been drawn
Gdx.gl20.glStencilFunc(GL20.GL_NOTEQUAL, 0x1, 0xff);
// decalBatch.add(decal);
// decalBatch.flush();
spriteBatch.draw(Assets.instance.background, 0, 0, Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
// draw where pattern HAS been drawn.
// Gdx.gl20.glStencilFunc(GL20.GL_EQUAL, 0x1, 0xff);
// spriteBatch.draw(Assets.instance.actor1, -Gdx.graphics.getWidth() /
// 2, -Gdx.graphics.getHeight() / 2, Gdx.graphics.getWidth(),
// Gdx.graphics.getHeight());
spriteBatch.end();
EDIT:
I figured out that there should be clearing of Depth buffer, and enabling and disabling of DepthMask, but I cant manage it to work.
Oh I just figured it out. Depth buffer must be enabled couse Decals are in 3d world.
this is the solution if somebody is interested:
Gdx.gl.glClearColor(1, 1, 1, 1);
Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_STENCIL_BUFFER_BIT | GL20.GL_DEPTH_BUFFER_BIT);
// setup drawing to stencil buffer
Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);
Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
Gdx.gl20.glStencilOp(GL20.GL_REPLACE, GL20.GL_REPLACE, GL20.GL_REPLACE);
Gdx.gl20.glColorMask(false, false, false, false);
Gdx.gl20.glDepthMask(false);
// draw base pattern
shapeRenderer.begin(ShapeType.Filled);
shapeRenderer.identity();
shapeRenderer.setColor(1f, 0f, 0f, 0.5f);
shapeRenderer.rect(Gdx.graphics.getWidth() / 2, Gdx.graphics.getHeight() / 2, 100, 100);
shapeRenderer.end();
// fix stencil buffer, enable color buffer
Gdx.gl20.glColorMask(true, true, true, true);
Gdx.gl20.glDepthMask(true);
Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);
// draw where pattern has NOT been drawn
Gdx.gl20.glStencilFunc(GL20.GL_NOTEQUAL, 0x1, 0xff);
decalBatch.add(decal);
decalBatch.flush();
Gdx.gl20.glDisable(GL20.GL_STENCIL_TEST);