I am working on a Pepper application on android that uses a QiChatbot to communicate with users about minerals. I want to make the chatbot also available for on-screen use for people with hearing and/or speech impairment. So i am building a UI like facebook messenger. While I am able to get the reply from pepper and show it on the screen, I cannot find a way to force a String that I am getting from an EditText by the user to the chatbot.
I have tried using:
Future<ReplyReaction> replyToFuture = chatbot.async().replyTo(userPhrase, localeEN);
and
Future<Void> acknowledgeHeardFuture = chatbot.async().acknowledgeHeard(userPhrase,localeEN);
with no success
Below you can see the whole chat and chatbot setup:
private QiChatbot buildQiChatbot() {
Topic lexicon = TopicBuilder
.with(qiContext)
.withResource(R.raw.lexicon)
.build();
Topic topic_minerals = TopicBuilder
.with(qiContext)
.withResource(R.raw.topic_minerals)
.build();
Topic test = TopicBuilder
.with(qiContext)
.withResource(R.raw.test_topic)
.build();
List<Topic> topicList = new LinkedList<Topic>();
topicList.add(test);
//topicList.add(lexicon);
//topicList.add(topic_minerals);
chatbot = QiChatbotBuilder
.with(qiContext)
.withTopics(topicList)
.withLocale(localeEN)
.build();
chatbot.addOnBookmarkReachedListener(bookmark -> {
Log.i(TAG, "Bookmark " + bookmark.getName() + " reached.");
});
chatbot.addOnEndedListener(endReason -> {
Log.i(TAG, "Chatbot ended for reason: " + endReason);
chatFuture.requestCancellation();
});
return chatbot;
}
private Chat buildChat(QiChatbot chatbot) {
chat = ChatBuilder
.with(qiContext)
.withChatbot(chatbot)
.build();
chat.addOnStartedListener(() -> {
Log.i(TAG, "[CHAT] Chat started.");
});
chat.addOnListeningChangedListener(listening -> {
if (listening) {
Log.i(TAG, "[CHAT] Listening START.");
} else {
Log.i(TAG, "[CHAT] Listening END.");
}
});
chat.addOnHeardListener(heardPhrase -> {
Log.i(TAG, "[CHAT] Heard phrase: " + heardPhrase.getText());
});
chat.addOnNormalReplyFoundForListener(input -> {
Log.i(TAG, "[CHAT] Reply found for user message: " + input.getText());
});
chat.addOnNoPhraseRecognizedListener(() -> {
Log.i(TAG, "[CHAT] No phrase recognized.");
});
chat.addOnSayingChangedListener(sayingPhrase -> {
if (!sayingPhrase.getText().isEmpty()) {
Log.i(TAG, "[CHAT] Pepper Reply: " + sayingPhrase.getText());
messageItemList.add(new MessageItem(LayoutRobot, R.drawable.icons8_user_100, sayingPhrase.getText()));
messageAdapter.notifyItemInserted(messageItemList.size());
}
});
chat.addOnFallbackReplyFoundForListener(input -> {
Log.i(TAG, "[CHAT] Fallback Reply found for user message: " + input.getText());
});
chat.addOnNoReplyFoundForListener(input -> {
Log.i(TAG, "[CHAT] NO Reply found for user message: " + input.getText());
});
return chat;
}
I tried to use replyTo the way @Victor Paléologue mentioned:
Future<ReplyReaction> replyFuture = chatbot.async().replyTo(userPhrase, localeEN);
replyFuture.thenConsume(future -> {
if (future.hasError()) {
Log.e(TAG, "Reply Future [ERROR]: " + future.getErrorMessage());
} else {
Log.i(TAG, "Reply Future [SUCCESS]: ");
ReplyReaction replyReaction = future.get();
ChatbotReaction chatbotReaction = replyReaction.getChatbotReaction();
chatbotReaction.runWith(speechEngine);
}
});
This code produces no errors. If the userPhrase is not present in the Topic that the chatbot runs, the error gets captured at the first if clause.
If the chatbot knows the phrase the following code seems to run but I am not getting any feedback. For some reason the chatbot doesn't actually get the phrase because if it was I would see the logs from the following method.
chatbot.addOnBookmarkReachedListener(bookmark -> {
Log.i(TAG, "Bookmark " + bookmark.getName() + " reached.");
});
Upon further inspection I chained all the operations one after the other to see how the futures are completed:
Future<ReplyReaction> replyFuture = chatbot.async().replyTo(userPhrase, localeEN);
replyFuture.thenConsume(future -> {
if (future.hasError()) {
Log.e(TAG, "Reply Reaction Future [ERROR]: " + future.getErrorMessage());
} else if(future.isCancelled()) {
Log.i(TAG, "Reply Reaction Future [CANCELED]: ");
} else if(future.isDone()) {
Log.i(TAG, "Reply Reaction Future [DONE]: ");
func1(future);
} else if(future.isSuccess()) {
Log.i(TAG, "Reply Reaction Future [SUCCESS]: ");
func1(future);
}
});
private void func1(Future<ReplyReaction> replyReactionFuture) throws ExecutionException {
ReplyReaction replyReaction = replyReactionFuture.get();
Future<ChatbotReaction> chatbotReactionFuture = replyReaction.async().getChatbotReaction();
chatbotReactionFuture.thenConsume(future -> {
if (future.hasError()){
Log.e(TAG, "Chatbot Reaction Future [ERROR]: " + future.getErrorMessage());
} else if(future.isCancelled()) {
Log.i(TAG, "Chatbot Reaction Future [CANCELED]: ");
} else if(future.isDone()) {
Log.i(TAG, "Chatbot Reaction Future [DONE]: ");
func2(future);
} else if(future.isSuccess()) {
Log.i(TAG, "Chatbot Reaction Future [SUCCESS]: ");
func2(future);
}
});
}
private void func2(Future<ChatbotReaction> chatbotReactionFuture) throws ExecutionException {
ChatbotReaction chatbotReaction = chatbotReactionFuture.get();
Future<Void> runWithFuture = chatbotReaction.async().runWith(speechEngine);
runWithFuture.thenConsume(future -> {
if (future.hasError()){
Log.e(TAG, "runWith Future [ERROR]: " + future.getErrorMessage());
} else if(future.isDone()) {
Log.i(TAG, "runWith Future [DONE]: ");
} else if(future.isCancelled()) {
Log.i(TAG, "runWith Future [CANCELED]: ");
} else if(future.isSuccess()) {
Log.i(TAG, "runWith Future [SUCCESS]: ");
}
});
}
The Log output is the following:
2023-11-17 21:34:56.035 10000-10031/? I/Main Activity: Reply Reaction Future [DONE]:
2023-11-17 21:34:56.044 10000-10030/? I/Main Activity: Chatbot Reaction Future [DONE]:
2023-11-17 21:34:56.068 10000-10031/? I/Main Activity: runWith Future [DONE]:
Which means that all the futures are completed by nothing inside the chatbot gets triggered.
I canceled the chat before running replyTo and all the subsequent code and it works. Then I start the chat again after the runWithFuture is done.
This is the complete code that runs in the onCLick method.
chatFuture.requestCancellation();
Future<ReplyReaction> replyToFuture = chatbot.async().replyTo(userPhrase, localeEN);
replyToFuture.thenConsume(replyReactionFuture -> {
if (replyReactionFuture.hasError()) {
Log.e(TAG, "Reply Reaction Future [ERROR]: " + replyReactionFuture.getErrorMessage());
} else {
ReplyReaction replyReaction = replyReactionFuture.get();
Future<ChatbotReaction> getChatbotReactionFuture = replyReaction.async().getChatbotReaction();
getChatbotReactionFuture.thenConsume(chatbotReactionFuture -> {
if (chatbotReactionFuture.hasError()){
Log.e(TAG, "Chatbot Reaction Future [ERROR]: " + chatbotReactionFuture.getErrorMessage());
} else {
ChatbotReaction chatbotReaction = chatbotReactionFuture.get();
Future<Void> runWithFuture = chatbotReaction.async().runWith(speechEngine);
runWithFuture.thenConsume(future -> {
if (future.hasError()){
Log.e(TAG, "runWith Future [ERROR]: " + future.getErrorMessage());
} else {
chat = buildChat(chatbot);
chatFuture = chat.async().run();
chatFuture.thenConsume(future1 -> {
if (future1.hasError()) {
Log.e(TAG, "Chat finished with error: " + future1.getErrorMessage());
} else {
Log.e(TAG, "Chat finished: " + future1.get().toString());
}
});
}
});
}
});
}
});
Also in order to get the chatbot reply when the chat is not running I had to add a listener after making the SpeechEngine
speechEngine = conversation.makeSpeechEngine(robotContext);
speechEngine.addOnSayingChangedListener(phrase -> {
String sayingText = phrase.getText();
if (!sayingText.isEmpty()) {
Log.i(TAG, "[SPEECH ENGINE] Pepper Reply: " + sayingText);
messageItemList.add(new MessageItem(LayoutRobot, R.drawable.ic_pepper_w, sayingText));
runOnUiThread(() -> {
messageAdapter.notifyItemInserted(messageAdapter
.getItemCount() - 1);
recyclerView.scrollToPosition(messageItemList.size() - 1);
});
}
});
chatbot.replyTo returns a reply object. It has a chatbotReaction that you can runWith(SpeechEngine engine). The speech engine is given by the Conversation service.
How to use it is tricky and not well documented. In theory this is meant to work:
RobotContext robotContext = qiContext.getRobotContext();
Conversation conversation = qiContext.getConversation();
SpeechEngine speechEngine = conversation.makeSpeechEngine(robotContext);
ReplyReaction replyReaction = chatbot.replyTo(userPhrase, localeEN);
ChatbotReaction reaction = replyReaction.getChatbotReaction();
reaction.runWith(speechEngine);
Note that you may need to stop the chat before running the reaction with the speechEngine.