I have a problem to run all integration tests running with Couchbase TestContainer in my Spring Boot Example.
Here is the relevant part of application.yml file shown below
couchbase:
connection-string: couchbase://${COUCHBASE_HOST:127.0.0.1}
username: ${COUCHBASE_USERNAME:Administrator}
password: ${COUCHBASE_PASSWORD:123456}
bucket: ${COUCHBASE_BUCKET:todo_list}
scopes:
user-scope: user
task-scope: task
invalid-token-scope : invalid-token
log-scope: log
collections:
user-collection: user
task-collection: task
invalid-token-collection: invalid-token
log-collection : log
Here is the class shown below
@Slf4j
@Testcontainers
public abstract class AbstractTestContainerConfiguration {
private static final String BUCKET_NAME = "todo_list";
static CouchbaseContainer COUCHBASE_CONTAINER = new CouchbaseContainer("couchbase/server:7.0.3")
.withCredentials("Administrator", "123456");
@BeforeAll
public static void setup() {
COUCHBASE_CONTAINER.start();
// Connect to Couchbase cluster
Cluster cluster = Cluster.connect(
COUCHBASE_CONTAINER.getConnectionString(),
COUCHBASE_CONTAINER.getUsername(),
COUCHBASE_CONTAINER.getPassword()
);
// Create the bucket if it doesn't exist
createBucketIfNotExists(cluster);
// Access the bucket and ensure it's ready
Bucket bucket = cluster.bucket(BUCKET_NAME);
bucket.waitUntilReady(Duration.ofSeconds(10));
CollectionManager collectionManager = bucket.collections();
// Create scopes and collections (according to your configuration)
createScopeAndCollection(collectionManager, "invalid-token", "invalid-token");
createScopeAndCollection(collectionManager, "user", "user");
createScopeAndCollection(collectionManager, "task", "task");
createScopeAndCollection(collectionManager, "log", "log");
// Ensure all necessary collections exist
ensureCollectionExists(bucket, "invalid-token", "invalid-token");
ensureCollectionExists(bucket, "user", "user");
ensureCollectionExists(bucket, "task", "task");
ensureCollectionExists(bucket, "log", "log");
cluster.disconnect();
}
private static void createBucketIfNotExists(Cluster cluster) {
try {
// Check if the bucket already exists
cluster.buckets().getBucket(AbstractTestContainerConfiguration.BUCKET_NAME);
} catch (Exception e) {
log.info("Bucket '{}' not found. Creating a new bucket.", AbstractTestContainerConfiguration.BUCKET_NAME);
cluster.buckets().createBucket(
BucketSettings.create(AbstractTestContainerConfiguration.BUCKET_NAME)
.flushEnabled(true)
.ramQuotaMB(100)
.numReplicas(1)
);
}
}
private static void createScopeAndCollection(CollectionManager manager, String scopeName, String collectionName) {
try {
// Create the scope if it does not exist
try {
manager.createScope(scopeName);
log.info("Scope created: {}", scopeName);
} catch (ScopeExistsException ignored) {
log.warn("Scope already exists: {}", scopeName);
}
// Create the collection under the given scope if it does not exist
try {
manager.createCollection(scopeName, collectionName);
log.info("Collection created: {}.{}", scopeName, collectionName);
} catch (CollectionExistsException ignored) {
log.warn("Collection already exists in scope {}: {}", scopeName, collectionName);
}
} catch (Exception e) {
log.error("Error creating scope or collection: {}.{}", scopeName, collectionName, e);
}
}
private static void ensureCollectionExists(Bucket bucket, String scopeName, String collectionName) {
try {
Collection collection = bucket.scope(scopeName).collection(collectionName);
log.info("Collection exists: {}.{}", scopeName, collectionName);
} catch (CollectionNotFoundException e) {
log.info("Collection not found: {}.{} - Creating it.", scopeName, collectionName);
bucket.collections().createCollection(scopeName, collectionName);
}
}
@DynamicPropertySource
private static void overrideProps(DynamicPropertyRegistry dynamicPropertyRegistry) {
dynamicPropertyRegistry.add("spring.couchbase.connection-string", COUCHBASE_CONTAINER::getConnectionString);
dynamicPropertyRegistry.add("spring.couchbase.username", COUCHBASE_CONTAINER::getUsername);
dynamicPropertyRegistry.add("spring.couchbase.password", COUCHBASE_CONTAINER::getPassword);
dynamicPropertyRegistry.add("spring.couchbase.bucket", () -> BUCKET_NAME);
}
}
Here is the error shown below
com.couchbase.client.core.error.IndexFailureException: The server reported an issue with the underlying index {"completed":true,"coreId":"0x42adbe3e00000002","errors":[{"code":12021,"message":"Scope not found in CB datastore default:todo_list.invalid_token","retry":false}],"httpStatus":500,"idempotent":true,"lastDispatchedFrom":"127.0.0.1:26562","lastDispatchedTo":"localhost:26462","requestId":47,"requestType":"QueryRequest","retried":0,"service":{"bucket":"todo_list","operationId":"null","scope":"invalid_token","statement":"SELECT `_class`, META(`invalid_token`).`id` AS __id, `TOKEN_ID`, `createdAt`, `createdBy`, `updatedAt`, `updatedBy` FROM `invalid_token` WHERE `_class` = \"com.example.todowithcouchbase.auth.model.entity.InvalidTokenEntity\" AND `TOKEN_ID` = $1","type":"query"},"timeoutMs":75000,"timings":{"dispatchMicros":34547,"totalDispatchMicros":34547,"totalMicros":99591}}
How can I fix the issue?
Here is my solution shown below
1 ) I revised scopes and collections as more meaningful names in application.yml file
couchbase:
connection-string: couchbase://${COUCHBASE_HOST:127.0.0.1}
username: ${COUCHBASE_USERNAME:Administrator}
password: ${COUCHBASE_PASSWORD:123456}
bucket: ${COUCHBASE_BUCKET:todo_list}
scopes:
user-scope: user-scope
task-scope: task-scope
invalid-token-scope : invalid-token-scope
log-scope: log-scope
collections:
user-collection: user-collection
task-collection: task-collection
invalid-token-collection: invalid-token-collection
log-collection : log-collection
2 ) I defined scope, collection and their indexes in AbstractTestContainerConfiguration
@Slf4j
@Testcontainers
public abstract class AbstractTestContainerConfiguration {
private static final String BUCKET_NAME = "todo_list";
static CouchbaseContainer COUCHBASE_CONTAINER = new CouchbaseContainer("couchbase/server:7.0.3")
.withCredentials("Administrator", "123456");
@BeforeAll
public static void setup() {
COUCHBASE_CONTAINER.start();
// Connect to Couchbase cluster
Cluster cluster = Cluster.connect(
COUCHBASE_CONTAINER.getConnectionString(),
COUCHBASE_CONTAINER.getUsername(),
COUCHBASE_CONTAINER.getPassword()
);
// Create the bucket if it doesn't exist
createBucketIfNotExists(cluster);
// Access the bucket and ensure it's ready
Bucket bucket = cluster.bucket(BUCKET_NAME);
bucket.waitUntilReady(Duration.ofSeconds(10));
// Ensure that the query service is available
waitForQueryService(cluster);
CollectionManager collectionManager = bucket.collections();
// Create scopes and collections (according to your configuration)
createScopeAndCollection(collectionManager, "invalid-token-scope", "invalid-token-collection");
createScopeAndCollection(collectionManager, "user-scope", "user-collection");
createScopeAndCollection(collectionManager, "task-scope", "task-collection");
createScopeAndCollection(collectionManager, "log-scope", "log-collection");
// Ensure all necessary collections exist
ensureCollectionExists(bucket, "invalid-token-scope", "invalid-token-collection");
ensureCollectionExists(bucket, "user-scope", "user-collection");
ensureCollectionExists(bucket, "task-scope", "task-collection");
ensureCollectionExists(bucket, "log-scope", "log-collection");
// Ensure primary indexes are created on all collections (across scopes)
createPrimaryIndexIfNotExists(bucket, cluster, "invalid-token-scope", "invalid-token-collection");
createPrimaryIndexIfNotExists(bucket, cluster, "user-scope", "user-collection");
createPrimaryIndexIfNotExists(bucket, cluster, "task-scope", "task-collection");
createPrimaryIndexIfNotExists(bucket, cluster, "log-scope", "log-collection");
cluster.disconnect();
}
private static void createBucketIfNotExists(Cluster cluster) {
try {
// Check if the bucket already exists
cluster.buckets().getBucket(BUCKET_NAME);
} catch (Exception e) {
log.info("Bucket '{}' not found. Creating a new bucket.", BUCKET_NAME);
cluster.buckets().createBucket(
BucketSettings.create(BUCKET_NAME)
.flushEnabled(true)
.ramQuotaMB(100)
.numReplicas(1)
);
}
}
private static void createScopeAndCollection(CollectionManager manager, String scopeName, String collectionName) {
try {
// Create the scope if it does not exist
try {
manager.createScope(scopeName);
log.info("Scope created: {}", scopeName);
} catch (ScopeExistsException ignored) {
log.warn("Scope already exists: {}", scopeName);
}
// Create the collection under the given scope if it does not exist
try {
manager.createCollection(scopeName, collectionName);
log.info("Collection created: {}.{}", scopeName, collectionName);
} catch (CollectionExistsException ignored) {
log.warn("Collection already exists in scope {}: {}", scopeName, collectionName);
}
} catch (Exception e) {
log.error("Error creating scope or collection: {}.{}", scopeName, collectionName, e);
}
}
private static void ensureCollectionExists(Bucket bucket, String scopeName, String collectionName) {
try {
Collection collection = bucket.scope(scopeName).collection(collectionName);
log.info("Collection exists: {}.{}", scopeName, collectionName);
} catch (CollectionNotFoundException e) {
log.info("Collection not found: {}.{} - Creating it.", scopeName, collectionName);
bucket.collections().createCollection(scopeName, collectionName);
}
}
// Create primary index on the collection to cover all items within the collection
private static void createPrimaryIndexIfNotExists(Bucket bucket, Cluster cluster, String scopeName, String collectionName) {
try {
String createIndexQuery = String.format(
"CREATE PRIMARY INDEX ON `%s`.`%s`.`%s` USING GSI",
bucket.name(),
scopeName,
collectionName
);
// Run the query to create the primary index
try {
cluster.query(createIndexQuery);
log.info("Primary index created on collection {}.{} in scope {}.", collectionName, scopeName, bucket.name());
} catch (CouchbaseQueryExecutionException e) {
if (e.getCause().getMessage().contains("No index available")) {
log.warn("Primary index creation failed on collection {}.{} in scope {}. Retrying...", collectionName, scopeName, bucket.name());
cluster.query(createIndexQuery); // Retry creating index
log.info("Primary index created on collection {}.{} in scope {} after retry.", collectionName, scopeName, bucket.name());
} else {
log.error("Error executing query for primary index on collection {}.{} in scope {}: {}", collectionName, scopeName, bucket.name(), e.getMessage());
}
}
} catch (Exception e) {
log.error("Error creating primary index for collection {}.{} in scope {}: {}", collectionName, scopeName, bucket.name(), e);
}
}
// Wait for the query service to be ready before creating indexes
private static void waitForQueryService(Cluster cluster) {
try {
cluster.query("SELECT 1");
log.info("Query service is available.");
} catch (Exception e) {
log.error("Query service is not available. Retrying...", e);
// Retry mechanism could be implemented if needed
try {
Thread.sleep(5000); // Wait for 5 seconds before retrying
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
}
waitForQueryService(cluster); // Retry
}
}
@DynamicPropertySource
private static void overrideProps(DynamicPropertyRegistry dynamicPropertyRegistry) {
dynamicPropertyRegistry.add("spring.couchbase.connection-string", COUCHBASE_CONTAINER::getConnectionString);
dynamicPropertyRegistry.add("spring.couchbase.username", COUCHBASE_CONTAINER::getUsername);
dynamicPropertyRegistry.add("spring.couchbase.password", COUCHBASE_CONTAINER::getPassword);
dynamicPropertyRegistry.add("spring.couchbase.bucket", () -> BUCKET_NAME);
}
}