/* * Copyright 2017, The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.example.androidthings.assistant; import android.app.Activity; import android.content.Context; import android.content.SharedPreferences; import android.content.SharedPreferences.Editor; import android.media.AudioDeviceInfo; import android.media.AudioManager; import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.preference.PreferenceManager; import android.util.Base64; import android.util.Log; import android.view.View; import android.view.View.OnClickListener; import android.webkit.WebView; import android.widget.ArrayAdapter; import android.widget.CheckBox; import android.widget.CompoundButton; import android.widget.CompoundButton.OnCheckedChangeListener; import android.widget.ListView; import com.example.androidthings.assistant.EmbeddedAssistant.ConversationCallback; import com.example.androidthings.assistant.EmbeddedAssistant.RequestCallback; import com.google.android.things.contrib.driver.button.Button; import com.google.android.things.contrib.driver.voicehat.Max98357A; import com.google.android.things.contrib.driver.voicehat.VoiceHat; import com.google.android.things.pio.Gpio; import com.google.android.things.pio.PeripheralManager; import com.google.assistant.embedded.v1alpha2.SpeechRecognitionResult; import com.google.auth.oauth2.UserCredentials; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; import org.json.JSONException; import org.json.JSONObject; public class AssistantActivity extends Activity implements Button.OnButtonEventListener { private static final String TAG = AssistantActivity.class.getSimpleName(); // Peripheral and drivers constants. private static final int BUTTON_DEBOUNCE_DELAY_MS = 20; // Default on using the Voice Hat on Raspberry Pi 3. private static final boolean USE_VOICEHAT_I2S_DAC = Build.DEVICE.equals(BoardDefaults.DEVICE_RPI3); // Audio constants. private static final String PREF_CURRENT_VOLUME = "current_volume"; private static final int SAMPLE_RATE = 16000; private static final int DEFAULT_VOLUME = 100; // Assistant SDK constants. private static final String DEVICE_MODEL_ID = "PLACEHOLDER"; private static final String DEVICE_INSTANCE_ID = "PLACEHOLDER"; private static final String LANGUAGE_CODE = "en-US"; // Hardware peripherals. private Button mButton; private android.widget.Button mButtonWidget; private Gpio mLed; private Max98357A mDac; private Handler mMainHandler; // List & adapter to store and display the history of Assistant Requests. private EmbeddedAssistant mEmbeddedAssistant; private ArrayList mAssistantRequests = new ArrayList<>(); private ArrayAdapter mAssistantRequestsAdapter; private CheckBox mHtmlOutputCheckbox; private WebView mWebView; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); Log.i(TAG, "starting assistant demo"); setContentView(R.layout.activity_main); final ListView assistantRequestsListView = findViewById(R.id.assistantRequestsListView); mAssistantRequestsAdapter = new ArrayAdapter<>(this, android.R.layout.simple_list_item_1, mAssistantRequests); assistantRequestsListView.setAdapter(mAssistantRequestsAdapter); mHtmlOutputCheckbox = findViewById(R.id.htmlOutput); mHtmlOutputCheckbox.setOnCheckedChangeListener(new OnCheckedChangeListener() { @Override public void onCheckedChanged(CompoundButton compoundButton, boolean useHtml) { mWebView.setVisibility(useHtml ? View.VISIBLE : View.GONE); assistantRequestsListView.setVisibility(useHtml ? View.GONE : View.VISIBLE); mEmbeddedAssistant.setResponseFormat(useHtml ? EmbeddedAssistant.HTML : EmbeddedAssistant.TEXT); } }); mWebView = findViewById(R.id.webview); mWebView.getSettings().setJavaScriptEnabled(true); mMainHandler = new Handler(getMainLooper()); mButtonWidget = findViewById(R.id.assistantQueryButton); mButtonWidget.setOnClickListener(new OnClickListener() { @Override public void onClick(View view) { mEmbeddedAssistant.startConversation(); } }); // Audio routing configuration: use default routing. AudioDeviceInfo audioInputDevice = null; AudioDeviceInfo audioOutputDevice = null; if (USE_VOICEHAT_I2S_DAC) { audioInputDevice = findAudioDevice(AudioManager.GET_DEVICES_INPUTS, AudioDeviceInfo.TYPE_BUS); if (audioInputDevice == null) { Log.e(TAG, "failed to find I2S audio input device, using default"); } audioOutputDevice = findAudioDevice(AudioManager.GET_DEVICES_OUTPUTS, AudioDeviceInfo.TYPE_BUS); if (audioOutputDevice == null) { Log.e(TAG, "failed to found I2S audio output device, using default"); } } try { if (USE_VOICEHAT_I2S_DAC) { Log.i(TAG, "initializing DAC trigger"); mDac = VoiceHat.openDac(); mDac.setSdMode(Max98357A.SD_MODE_SHUTDOWN); mButton = VoiceHat.openButton(); mLed = VoiceHat.openLed(); } else { PeripheralManager pioManager = PeripheralManager.getInstance(); mButton = new Button(BoardDefaults.getGPIOForButton(), Button.LogicState.PRESSED_WHEN_LOW); mLed = pioManager.openGpio(BoardDefaults.getGPIOForLED()); } mButton.setDebounceDelay(BUTTON_DEBOUNCE_DELAY_MS); mButton.setOnButtonEventListener(this); mLed.setDirection(Gpio.DIRECTION_OUT_INITIALLY_LOW); mLed.setActiveType(Gpio.ACTIVE_HIGH); } catch (IOException e) { Log.e(TAG, "error configuring peripherals:", e); return; } // Set volume from preferences SharedPreferences preferences = PreferenceManager.getDefaultSharedPreferences(this); int initVolume = preferences.getInt(PREF_CURRENT_VOLUME, DEFAULT_VOLUME); Log.i(TAG, "setting audio track volume to: " + initVolume); UserCredentials userCredentials = null; try { userCredentials = EmbeddedAssistant.generateCredentials(this, R.raw.credentials); } catch (IOException | JSONException e) { Log.e(TAG, "error getting user credentials", e); } mEmbeddedAssistant = new EmbeddedAssistant.Builder() .setCredentials(userCredentials) .setDeviceInstanceId(DEVICE_INSTANCE_ID) .setDeviceModelId(DEVICE_MODEL_ID) .setLanguageCode(LANGUAGE_CODE) .setAudioInputDevice(audioInputDevice) .setAudioOutputDevice(audioOutputDevice) .setAudioSampleRate(SAMPLE_RATE) .setAudioVolume(initVolume) .setRequestCallback(new RequestCallback() { @Override public void onRequestStart() { Log.i(TAG, "starting assistant request, enable microphones"); mButtonWidget.setText(R.string.button_listening); mButtonWidget.setEnabled(false); } @Override public void onSpeechRecognition(List results) { for (final SpeechRecognitionResult result : results) { Log.i(TAG, "assistant request text: " + result.getTranscript() + " stability: " + Float.toString(result.getStability())); mAssistantRequestsAdapter.add(result.getTranscript()); } } }) .setConversationCallback(new ConversationCallback() { @Override public void onResponseStarted() { super.onResponseStarted(); // When bus type is switched, the AudioManager needs to reset the stream volume if (mDac != null) { try { mDac.setSdMode(Max98357A.SD_MODE_LEFT); } catch (IOException e) { Log.e(TAG, "error enabling DAC", e); } } } @Override public void onResponseFinished() { super.onResponseFinished(); if (mDac != null) { try { mDac.setSdMode(Max98357A.SD_MODE_SHUTDOWN); } catch (IOException e) { Log.e(TAG, "error disabling DAC", e); } } if (mLed != null) { try { mLed.setValue(false); } catch (IOException e) { Log.e(TAG, "cannot turn off LED", e); } } } @Override public void onError(Throwable throwable) { Log.e(TAG, "assist error: " + throwable.getMessage(), throwable); } @Override public void onVolumeChanged(int percentage) { Log.i(TAG, "assistant volume changed: " + percentage); // Update our shared preferences Editor editor = PreferenceManager .getDefaultSharedPreferences(AssistantActivity.this) .edit(); editor.putInt(PREF_CURRENT_VOLUME, percentage); editor.apply(); } @Override public void onConversationFinished() { Log.i(TAG, "assistant conversation finished"); mButtonWidget.setText(R.string.button_new_request); mButtonWidget.setEnabled(true); } @Override public void onAssistantResponse(final String response) { if(!response.isEmpty()) { mMainHandler.post(new Runnable() { @Override public void run() { mAssistantRequestsAdapter.add("Google Assistant: " + response); } }); } } @Override public void onAssistantDisplayOut(final String html) { mMainHandler.post(new Runnable() { @Override public void run() { // Need to convert to base64 try { final byte[] data = html.getBytes("UTF-8"); final String base64String = Base64.encodeToString(data, Base64.DEFAULT); mWebView.loadData(base64String, "text/html; charset=utf-8", "base64"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } } }); } public void onDeviceAction(String intentName, JSONObject parameters) { if (parameters != null) { Log.d(TAG, "Get device action " + intentName + " with parameters: " + parameters.toString()); } else { Log.d(TAG, "Get device action " + intentName + " with no paramete" + "rs"); } if (intentName.equals("action.devices.commands.OnOff")) { try { boolean turnOn = parameters.getBoolean("on"); mLed.setValue(turnOn); } catch (JSONException e) { Log.e(TAG, "Cannot get value of command", e); } catch (IOException e) { Log.e(TAG, "Cannot set value of LED", e); } } } }) .build(); mEmbeddedAssistant.connect(); } private AudioDeviceInfo findAudioDevice(int deviceFlag, int deviceType) { AudioManager manager = (AudioManager) this.getSystemService(Context.AUDIO_SERVICE); AudioDeviceInfo[] adis = manager.getDevices(deviceFlag); for (AudioDeviceInfo adi : adis) { if (adi.getType() == deviceType) { return adi; } } return null; } @Override public void onButtonEvent(Button button, boolean pressed) { try { if (mLed != null) { mLed.setValue(pressed); } } catch (IOException e) { Log.d(TAG, "error toggling LED:", e); } if (pressed) { mEmbeddedAssistant.startConversation(); } } @Override protected void onDestroy() { super.onDestroy(); Log.i(TAG, "destroying assistant demo"); if (mLed != null) { try { mLed.close(); } catch (IOException e) { Log.w(TAG, "error closing LED", e); } mLed = null; } if (mButton != null) { try { mButton.close(); } catch (IOException e) { Log.w(TAG, "error closing button", e); } mButton = null; } if (mDac != null) { try { mDac.close(); } catch (IOException e) { Log.w(TAG, "error closing voice hat trigger", e); } mDac = null; } mEmbeddedAssistant.destroy(); } }