/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.service.voice;
import static java.util.Objects.requireNonNull;
import android.annotation.DurationMillisLong;
import android.annotation.FlaggedApi;
import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.SdkConstant;
import android.annotation.SuppressLint;
import android.annotation.SystemApi;
import android.annotation.TestApi;
import android.app.Service;
import android.content.ContentCaptureOptions;
import android.content.Context;
import android.content.Intent;
import android.hardware.soundtrigger.SoundTrigger;
import android.media.AudioFormat;
import android.media.AudioSystem;
import android.os.IBinder;
import android.os.IRemoteCallback;
import android.os.ParcelFileDescriptor;
import android.os.PersistableBundle;
import android.os.RemoteException;
import android.os.SharedMemory;
import android.speech.IRecognitionServiceManager;
import android.util.Log;
import android.view.contentcapture.ContentCaptureManager;
import android.view.contentcapture.IContentCaptureManager;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.Locale;
import java.util.function.IntConsumer;
/**
* Implemented by an application that wants to offer detection for hotword. The service can be used
* for both DSP and non-DSP detectors.
*
* The system will bind an application's {@link VoiceInteractionService} first. When {@link
* VoiceInteractionService#createHotwordDetector(PersistableBundle, SharedMemory,
* HotwordDetector.Callback)} or {@link VoiceInteractionService#createAlwaysOnHotwordDetector(
* String, Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} is called,
* the system will bind application's {@link HotwordDetectionService}. Either on a hardware
* trigger or on request from the {@link VoiceInteractionService}, the system calls into the
* {@link HotwordDetectionService} to request detection. The {@link HotwordDetectionService} then
* uses {@link Callback#onDetected(HotwordDetectedResult)} to inform the system that a relevant
* keyphrase was detected, or if applicable uses {@link Callback#onRejected(HotwordRejectedResult)}
* to inform the system that a keyphrase was not detected. The system then relays this result to
* the {@link VoiceInteractionService} through {@link HotwordDetector.Callback}.
*
* Note: Methods in this class may be called concurrently
*
* @hide
*/
@SystemApi
public abstract class HotwordDetectionService extends Service
implements SandboxedDetectionInitializer {
private static final String TAG = "HotwordDetectionService";
private static final boolean DBG = false;
private static final long UPDATE_TIMEOUT_MILLIS = 20000;
/**
* The PersistableBundle options key used in {@link #onDetect(ParcelFileDescriptor, AudioFormat,
* PersistableBundle, Callback)} to indicate whether the system will close the audio stream
* after {@code Callback} is invoked.
*/
@FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)
public static final String KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK =
"android.service.voice.HotwordDetectionService."
+ "KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK";
/**
* Feature flag for Attention Service.
*
* @hide
*/
@TestApi
public static final boolean ENABLE_PROXIMITY_RESULT = true;
/**
* Indicates that the updated status is successful.
*
* @deprecated Replaced with
* {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_SUCCESS}
*/
@Deprecated
public static final int INITIALIZATION_STATUS_SUCCESS =
SandboxedDetectionInitializer.INITIALIZATION_STATUS_SUCCESS;
/**
* Indicates that the callback wasn’t invoked within the timeout.
* This is used by system.
*
* @deprecated Replaced with
* {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_UNKNOWN}
*/
@Deprecated
public static final int INITIALIZATION_STATUS_UNKNOWN =
SandboxedDetectionInitializer.INITIALIZATION_STATUS_UNKNOWN;
/**
* Source for the given audio stream.
*
* @hide
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@IntDef({
AUDIO_SOURCE_MICROPHONE,
AUDIO_SOURCE_EXTERNAL
})
@interface AudioSource {}
/** @hide */
public static final int AUDIO_SOURCE_MICROPHONE = 1;
/** @hide */
public static final int AUDIO_SOURCE_EXTERNAL = 2;
/**
* The {@link Intent} that must be declared as handled by the service.
* To be supported, the service must also require the
* {@link android.Manifest.permission#BIND_HOTWORD_DETECTION_SERVICE} permission so
* that other applications can not abuse it.
*/
@SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION)
public static final String SERVICE_INTERFACE =
"android.service.voice.HotwordDetectionService";
@Nullable
private ContentCaptureManager mContentCaptureManager;
@Nullable
private IRecognitionServiceManager mIRecognitionServiceManager;
private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {
@Override
public void detectFromDspSource(
SoundTrigger.KeyphraseRecognitionEvent event,
AudioFormat audioFormat,
long timeoutMillis,
IDspHotwordDetectionCallback callback)
throws RemoteException {
if (DBG) {
Log.d(TAG, "#detectFromDspSource");
}
HotwordDetectionService.this.onDetect(
new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(),
timeoutMillis,
new Callback(callback));
}
@Override
public void updateState(PersistableBundle options, SharedMemory sharedMemory,
IRemoteCallback callback) throws RemoteException {
Log.v(TAG, "#updateState" + (callback != null ? " with callback" : ""));
HotwordDetectionService.this.onUpdateStateInternal(
options,
sharedMemory,
callback);
}
@Override
public void detectFromMicrophoneSource(
ParcelFileDescriptor audioStream,
@AudioSource int audioSource,
AudioFormat audioFormat,
PersistableBundle options,
IDspHotwordDetectionCallback callback)
throws RemoteException {
if (DBG) {
Log.d(TAG, "#detectFromMicrophoneSource");
}
switch (audioSource) {
case AUDIO_SOURCE_MICROPHONE:
HotwordDetectionService.this.onDetect(
new Callback(callback));
break;
case AUDIO_SOURCE_EXTERNAL:
HotwordDetectionService.this.onDetect(
audioStream,
audioFormat,
options,
new Callback(callback));
break;
default:
Log.i(TAG, "Unsupported audio source " + audioSource);
}
}
@Override
public void detectWithVisualSignals(
IDetectorSessionVisualQueryDetectionCallback callback) {
throw new UnsupportedOperationException("Not supported by HotwordDetectionService");
}
@Override
public void updateAudioFlinger(IBinder audioFlinger) {
AudioSystem.setAudioFlingerBinder(audioFlinger);
}
@Override
public void updateContentCaptureManager(IContentCaptureManager manager,
ContentCaptureOptions options) {
mContentCaptureManager = new ContentCaptureManager(
HotwordDetectionService.this, manager, options);
}
@Override
public void updateRecognitionServiceManager(IRecognitionServiceManager manager) {
mIRecognitionServiceManager = manager;
}
@Override
public void ping(IRemoteCallback callback) throws RemoteException {
callback.sendResult(null);
}
@Override
public void stopDetection() {
HotwordDetectionService.this.onStopDetection();
}
@Override
public void registerRemoteStorageService(IDetectorSessionStorageService
detectorSessionStorageService) {
throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
}
};
@Override
@Nullable
public final IBinder onBind(@NonNull Intent intent) {
if (SERVICE_INTERFACE.equals(intent.getAction())) {
return mInterface.asBinder();
}
Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": "
+ intent);
return null;
}
@Override
@SuppressLint("OnNameExpected")
public @Nullable Object getSystemService(@ServiceName @NonNull String name) {
if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) {
return mContentCaptureManager;
} else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name)
&& mIRecognitionServiceManager != null) {
return mIRecognitionServiceManager.asBinder();
} else {
return super.getSystemService(name);
}
}
/**
* Returns the maximum number of initialization status for some application specific failed
* reasons.
*
* Note: The value 0 is reserved for success.
*
* @hide
* @deprecated Replaced with
* {@link SandboxedDetectionInitializer#getMaxCustomInitializationStatus()}
*/
@SystemApi
@Deprecated
public static int getMaxCustomInitializationStatus() {
return MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR;
}
/**
* Called when the device hardware (such as a DSP) detected the hotword, to request second stage
* validation before handing over the audio to the {@link AlwaysOnHotwordDetector}.
*
* After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the
* appropriate {@link AlwaysOnHotwordDetector.Callback callback}.
*
*
When responding to a detection event, the
* {@link HotwordDetectedResult#getHotwordPhraseId()} must match a keyphrase ID listed
* in the eventPayload's
* {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()} list. This is
* forcing the intention of the {@link HotwordDetectionService} to validate an event from the
* voice engine and not augment its result.
*
* @param eventPayload Payload data for the hardware detection event. This may contain the
* trigger audio, if requested when calling
* {@link AlwaysOnHotwordDetector#startRecognition(int)}.
* Each {@link AlwaysOnHotwordDetector} will be associated with at minimum a unique
* keyphrase ID indicated by
* {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()}[0].
* Any extra
* {@link android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra}'s
* in the eventPayload represent additional phrases detected by the voice engine.
* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
* the application fails to abide by the timeout, system will close the
* microphone and cancel the operation.
* @param callback The callback to use for responding to the detection request.
*
* @hide
*/
@SystemApi
public void onDetect(
@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload,
@DurationMillisLong long timeoutMillis,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale,
* PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or
* {@link AlwaysOnHotwordDetector#updateState(PersistableBundle, SharedMemory)} requests an
* update of the hotword detection parameters.
*
* {@inheritDoc}
* @hide
*/
@Override
@SystemApi
public void onUpdateState(
@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory,
@DurationMillisLong long callbackTimeoutMillis,
@Nullable IntConsumer statusCallback) {}
/**
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly
* from the device microphone.
*
* On successful detection of a hotword, call
* {@link Callback#onDetected(HotwordDetectedResult)}.
*
* @param callback The callback to use for responding to the detection request.
* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
*/
public void onDetect(@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
/**
* Called when the {@link VoiceInteractionService} requests that this service
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
* PersistableBundle)} run} hotword recognition on audio coming from an external connected
* microphone.
*
*
Upon invoking the {@code callback}, the system will send the detection result to
* the {@link HotwordDetector}'s callback. If {@code
* options.getBoolean(KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK, true)} returns true,
* the system will also close the {@code audioStream} after {@code callback} is invoked.
*
* @param audioStream Stream containing audio bytes returned from a microphone
* @param audioFormat Format of the supplied audio
* @param options Options supporting detection, such as configuration specific to the source of
* the audio, provided through
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
* PersistableBundle)}.
* @param callback The callback to use for responding to the detection request.
*/
public void onDetect(
@NonNull ParcelFileDescriptor audioStream,
@NonNull AudioFormat audioFormat,
@Nullable PersistableBundle options,
@NonNull Callback callback) {
// TODO: Add a helpful error message.
throw new UnsupportedOperationException();
}
private void onUpdateStateInternal(@Nullable PersistableBundle options,
@Nullable SharedMemory sharedMemory, IRemoteCallback callback) {
IntConsumer intConsumer =
SandboxedDetectionInitializer.createInitializationStatusConsumer(callback);
onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer);
}
/**
* Called when the {@link VoiceInteractionService}
* {@link HotwordDetector#stopRecognition() requests} that hotword recognition be stopped.
*
* Any open {@link android.media.AudioRecord} should be closed here.
*/
public void onStopDetection() {
}
/**
* Callback for returning the detection result.
*
* @hide
*/
@SystemApi
public static final class Callback {
// TODO: consider making the constructor a test api for testing purpose
private final IDspHotwordDetectionCallback mRemoteCallback;
private Callback(IDspHotwordDetectionCallback remoteCallback) {
mRemoteCallback = remoteCallback;
}
/**
* Informs the {@link HotwordDetector} that the keyphrase was detected.
*
* @param result Info about the detection result. This is provided to the
* {@link HotwordDetector}.
*/
public void onDetected(@NonNull HotwordDetectedResult result) {
requireNonNull(result);
final PersistableBundle persistableBundle = result.getExtras();
if (!persistableBundle.isEmpty() && HotwordDetectedResult.getParcelableSize(
persistableBundle) > HotwordDetectedResult.getMaxBundleSize()) {
throw new IllegalArgumentException(
"The bundle size of result is larger than max bundle size ("
+ HotwordDetectedResult.getMaxBundleSize()
+ ") of HotwordDetectedResult");
}
try {
mRemoteCallback.onDetected(result);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
/**
* Informs the {@link HotwordDetector} that the keyphrase was not detected.
*
* This cannot not be used when recognition is done through
* {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}.
*
* @param result Info about the second stage detection result. This is provided to
* the {@link HotwordDetector}.
*/
public void onRejected(@NonNull HotwordRejectedResult result) {
requireNonNull(result);
try {
mRemoteCallback.onRejected(result);
} catch (RemoteException e) {
throw e.rethrowFromSystemServer();
}
}
}
}