/* * Copyright 2012, The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //#define LOG_NDEBUG 0 #include "hidl/HidlSupport.h" #define LOG_TAG "MediaCodec" #include #include #include #include #include #include #include #include #include "include/SoftwareRenderer.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace android { using Status = ::ndk::ScopedAStatus; using aidl::android::media::BnResourceManagerClient; using aidl::android::media::IResourceManagerClient; using aidl::android::media::IResourceManagerService; using aidl::android::media::ClientInfoParcel; using server_configurable_flags::GetServerConfigurableFlag; using FreezeEvent = VideoRenderQualityTracker::FreezeEvent; using JudderEvent = VideoRenderQualityTracker::JudderEvent; // key for media statistics static const char *kCodecKeyName = "codec"; // attrs for media statistics // NB: these are matched with public Java API constants defined // in frameworks/base/media/java/android/media/MediaCodec.java // These must be kept synchronized with the constants there. static const char *kCodecLogSessionId = "android.media.mediacodec.log-session-id"; static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */ static const char *kCodecId = "android.media.mediacodec.id"; static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */ static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */ static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */ static const char *kCodecModeAudio = "audio"; static const char *kCodecModeImage = "image"; static const char *kCodecModeUnknown = "unknown"; static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */ static const char *kCodecHardware = "android.media.mediacodec.hardware"; /* 0,1 */ static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */ static const char *kCodecTunneled = "android.media.mediacodec.tunneled"; /* 0,1 */ static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */ static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */ static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */ static const char *kCodecColorFormat = "android.media.mediacodec.color-format"; static const char *kCodecFrameRate = "android.media.mediacodec.frame-rate"; static const char *kCodecCaptureRate = "android.media.mediacodec.capture-rate"; static const char *kCodecOperatingRate = "android.media.mediacodec.operating-rate"; static const char *kCodecPriority = "android.media.mediacodec.priority"; // Min/Max QP before shaping static const char *kCodecOriginalVideoQPIMin = "android.media.mediacodec.original-video-qp-i-min"; static const char *kCodecOriginalVideoQPIMax = "android.media.mediacodec.original-video-qp-i-max"; static const char *kCodecOriginalVideoQPPMin = "android.media.mediacodec.original-video-qp-p-min"; static const char *kCodecOriginalVideoQPPMax = "android.media.mediacodec.original-video-qp-p-max"; static const char *kCodecOriginalVideoQPBMin = "android.media.mediacodec.original-video-qp-b-min"; static const char *kCodecOriginalVideoQPBMax = "android.media.mediacodec.original-video-qp-b-max"; // Min/Max QP after shaping static const char *kCodecRequestedVideoQPIMin = "android.media.mediacodec.video-qp-i-min"; static const char *kCodecRequestedVideoQPIMax = "android.media.mediacodec.video-qp-i-max"; static const char *kCodecRequestedVideoQPPMin = "android.media.mediacodec.video-qp-p-min"; static const char *kCodecRequestedVideoQPPMax = "android.media.mediacodec.video-qp-p-max"; static const char *kCodecRequestedVideoQPBMin = "android.media.mediacodec.video-qp-b-min"; static const char *kCodecRequestedVideoQPBMax = "android.media.mediacodec.video-qp-b-max"; // NB: These are not yet exposed as public Java API constants. static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */ static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */ static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */ static const char *kCodecBitrateMode = "android.media.mediacodec.bitrate_mode"; /* CQ/VBR/CBR */ static const char *kCodecBitrate = "android.media.mediacodec.bitrate"; /* 0..n */ static const char *kCodecOriginalBitrate = "android.media.mediacodec.original.bitrate"; /* 0..n */ static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */ static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */ static const char *kCodecError = "android.media.mediacodec.errcode"; static const char *kCodecLifetimeMs = "android.media.mediacodec.lifetimeMs"; /* 0..n ms*/ static const char *kCodecErrorState = "android.media.mediacodec.errstate"; static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */ static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */ static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */ static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n"; static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */ static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown"; static const char *kCodecQueueSecureInputBufferError = "android.media.mediacodec.queueSecureInputBufferError"; static const char *kCodecQueueInputBufferError = "android.media.mediacodec.queueInputBufferError"; static const char *kCodecComponentColorFormat = "android.media.mediacodec.component-color-format"; static const char *kCodecNumLowLatencyModeOn = "android.media.mediacodec.low-latency.on"; /* 0..n */ static const char *kCodecNumLowLatencyModeOff = "android.media.mediacodec.low-latency.off"; /* 0..n */ static const char *kCodecFirstFrameIndexLowLatencyModeOn = "android.media.mediacodec.low-latency.first-frame"; /* 0..n */ static const char *kCodecChannelCount = "android.media.mediacodec.channelCount"; static const char *kCodecSampleRate = "android.media.mediacodec.sampleRate"; static const char *kCodecVideoEncodedBytes = "android.media.mediacodec.vencode.bytes"; static const char *kCodecVideoEncodedFrames = "android.media.mediacodec.vencode.frames"; static const char *kCodecVideoInputBytes = "android.media.mediacodec.video.input.bytes"; static const char *kCodecVideoInputFrames = "android.media.mediacodec.video.input.frames"; static const char *kCodecVideoEncodedDurationUs = "android.media.mediacodec.vencode.durationUs"; // HDR metrics static const char *kCodecConfigColorStandard = "android.media.mediacodec.config-color-standard"; static const char *kCodecConfigColorRange = "android.media.mediacodec.config-color-range"; static const char *kCodecConfigColorTransfer = "android.media.mediacodec.config-color-transfer"; static const char *kCodecParsedColorStandard = "android.media.mediacodec.parsed-color-standard"; static const char *kCodecParsedColorRange = "android.media.mediacodec.parsed-color-range"; static const char *kCodecParsedColorTransfer = "android.media.mediacodec.parsed-color-transfer"; static const char *kCodecHdrStaticInfo = "android.media.mediacodec.hdr-static-info"; static const char *kCodecHdr10PlusInfo = "android.media.mediacodec.hdr10-plus-info"; static const char *kCodecHdrFormat = "android.media.mediacodec.hdr-format"; // array/sync/async/block modes static const char *kCodecArrayMode = "android.media.mediacodec.array-mode"; static const char *kCodecOperationMode = "android.media.mediacodec.operation-mode"; static const char *kCodecOutputSurface = "android.media.mediacodec.output-surface"; // max size configured by the app static const char *kCodecAppMaxInputSize = "android.media.mediacodec.app-max-input-size"; // max size actually used static const char *kCodecUsedMaxInputSize = "android.media.mediacodec.used-max-input-size"; // max size suggested by the codec static const char *kCodecCodecMaxInputSize = "android.media.mediacodec.codec-max-input-size"; static const char *kCodecFlushCount = "android.media.mediacodec.flush-count"; static const char *kCodecSetSurfaceCount = "android.media.mediacodec.set-surface-count"; static const char *kCodecResolutionChangeCount = "android.media.mediacodec.resolution-change-count"; // the kCodecRecent* fields appear only in getMetrics() results static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */ static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */ static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */ static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n"; static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */ /* -1: shaper disabled >=0: number of fields changed */ static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped"; // Render metrics static const char *kCodecPlaybackDurationSec = "android.media.mediacodec.playback-duration-sec"; static const char *kCodecFirstRenderTimeUs = "android.media.mediacodec.first-render-time-us"; static const char *kCodecLastRenderTimeUs = "android.media.mediacodec.last-render-time-us"; static const char *kCodecFramesReleased = "android.media.mediacodec.frames-released"; static const char *kCodecFramesRendered = "android.media.mediacodec.frames-rendered"; static const char *kCodecFramesDropped = "android.media.mediacodec.frames-dropped"; static const char *kCodecFramesSkipped = "android.media.mediacodec.frames-skipped"; static const char *kCodecFramerateContent = "android.media.mediacodec.framerate-content"; static const char *kCodecFramerateDesired = "android.media.mediacodec.framerate-desired"; static const char *kCodecFramerateActual = "android.media.mediacodec.framerate-actual"; // Freeze static const char *kCodecFreezeCount = "android.media.mediacodec.freeze-count"; static const char *kCodecFreezeScore = "android.media.mediacodec.freeze-score"; static const char *kCodecFreezeRate = "android.media.mediacodec.freeze-rate"; static const char *kCodecFreezeDurationMsAvg = "android.media.mediacodec.freeze-duration-ms-avg"; static const char *kCodecFreezeDurationMsMax = "android.media.mediacodec.freeze-duration-ms-max"; static const char *kCodecFreezeDurationMsHistogram = "android.media.mediacodec.freeze-duration-ms-histogram"; static const char *kCodecFreezeDurationMsHistogramBuckets = "android.media.mediacodec.freeze-duration-ms-histogram-buckets"; static const char *kCodecFreezeDistanceMsAvg = "android.media.mediacodec.freeze-distance-ms-avg"; static const char *kCodecFreezeDistanceMsHistogram = "android.media.mediacodec.freeze-distance-ms-histogram"; static const char *kCodecFreezeDistanceMsHistogramBuckets = "android.media.mediacodec.freeze-distance-ms-histogram-buckets"; // Judder static const char *kCodecJudderCount = "android.media.mediacodec.judder-count"; static const char *kCodecJudderScore = "android.media.mediacodec.judder-score"; static const char *kCodecJudderRate = "android.media.mediacodec.judder-rate"; static const char *kCodecJudderScoreAvg = "android.media.mediacodec.judder-score-avg"; static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder-score-max"; static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder-score-histogram"; static const char *kCodecJudderScoreHistogramBuckets = "android.media.mediacodec.judder-score-histogram-buckets"; // Freeze event static const char *kCodecFreezeEventCount = "android.media.mediacodec.freeze-event-count"; static const char *kFreezeEventKeyName = "videofreeze"; static const char *kFreezeEventInitialTimeUs = "android.media.mediacodec.freeze.initial-time-us"; static const char *kFreezeEventDurationMs = "android.media.mediacodec.freeze.duration-ms"; static const char *kFreezeEventCount = "android.media.mediacodec.freeze.count"; static const char *kFreezeEventAvgDurationMs = "android.media.mediacodec.freeze.avg-duration-ms"; static const char *kFreezeEventAvgDistanceMs = "android.media.mediacodec.freeze.avg-distance-ms"; static const char *kFreezeEventDetailsDurationMs = "android.media.mediacodec.freeze.details-duration-ms"; static const char *kFreezeEventDetailsDistanceMs = "android.media.mediacodec.freeze.details-distance-ms"; // Judder event static const char *kCodecJudderEventCount = "android.media.mediacodec.judder-event-count"; static const char *kJudderEventKeyName = "videojudder"; static const char *kJudderEventInitialTimeUs = "android.media.mediacodec.judder.initial-time-us"; static const char *kJudderEventDurationMs = "android.media.mediacodec.judder.duration-ms"; static const char *kJudderEventCount = "android.media.mediacodec.judder.count"; static const char *kJudderEventAvgScore = "android.media.mediacodec.judder.avg-score"; static const char *kJudderEventAvgDistanceMs = "android.media.mediacodec.judder.avg-distance-ms"; static const char *kJudderEventDetailsActualDurationUs = "android.media.mediacodec.judder.details-actual-duration-us"; static const char *kJudderEventDetailsContentDurationUs = "android.media.mediacodec.judder.details-content-duration-us"; static const char *kJudderEventDetailsDistanceMs = "android.media.mediacodec.judder.details-distance-ms"; // XXX suppress until we get our representation right static bool kEmitHistogram = false; typedef WrapperObject> BufferInfosWrapper; // Multi access unit helpers static status_t generateFlagsFromAccessUnitInfo( sp &msg, const sp &bufferInfos) { msg->setInt64("timeUs", bufferInfos->value[0].mTimestamp); msg->setInt32("flags", bufferInfos->value[0].mFlags); // will prevent any access-unit info copy. if (bufferInfos->value.size() > 1) { uint32_t bufferFlags = 0; uint32_t flagsInAllAU = BUFFER_FLAG_DECODE_ONLY | BUFFER_FLAG_CODEC_CONFIG; uint32_t andFlags = flagsInAllAU; int infoIdx = 0; bool foundEndOfStream = false; for ( ; infoIdx < bufferInfos->value.size() && !foundEndOfStream; ++infoIdx) { bufferFlags |= bufferInfos->value[infoIdx].mFlags; andFlags &= bufferInfos->value[infoIdx].mFlags; if (bufferFlags & BUFFER_FLAG_END_OF_STREAM) { foundEndOfStream = true; } } bufferFlags = bufferFlags & (andFlags | (~flagsInAllAU)); if (infoIdx != bufferInfos->value.size()) { ALOGE("Error: incorrect access-units"); return -EINVAL; } msg->setInt32("flags", bufferFlags); } return OK; } static int64_t getId(IResourceManagerClient const * client) { return (int64_t) client; } static int64_t getId(const std::shared_ptr &client) { return getId(client.get()); } static bool isResourceError(status_t err) { return (err == NO_MEMORY); } static bool areRenderMetricsEnabled() { std::string v = GetServerConfigurableFlag("media_native", "render_metrics_enabled", "false"); return v == "true"; } static const int kMaxRetry = 2; static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s static const int kNumBuffersAlign = 16; static const C2MemoryUsage kDefaultReadWriteUsage{ C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE}; //////////////////////////////////////////////////////////////////////////////// /* * Implementation of IResourceManagerClient interrface that facilitates * MediaCodec reclaim for the ResourceManagerService. */ struct ResourceManagerClient : public BnResourceManagerClient { explicit ResourceManagerClient(MediaCodec* codec, int32_t pid, int32_t uid) : mMediaCodec(codec), mPid(pid), mUid(uid) {} Status reclaimResource(bool* _aidl_return) override { sp codec = mMediaCodec.promote(); if (codec == NULL) { // Codec is already gone, so remove the resources as well ::ndk::SpAIBinder binder(AServiceManager_waitForService("media.resource_manager")); std::shared_ptr service = IResourceManagerService::fromBinder(binder); if (service == nullptr) { ALOGE("MediaCodec::ResourceManagerClient unable to find ResourceManagerService"); *_aidl_return = false; return Status::fromStatus(STATUS_INVALID_OPERATION); } ClientInfoParcel clientInfo{.pid = static_cast(mPid), .uid = static_cast(mUid), .id = getId(this)}; service->removeClient(clientInfo); *_aidl_return = true; return Status::ok(); } status_t err = codec->reclaim(); if (err == WOULD_BLOCK) { ALOGD("Wait for the client to release codec."); usleep(kMaxReclaimWaitTimeInUs); ALOGD("Try to reclaim again."); err = codec->reclaim(true /* force */); } if (err != OK) { ALOGW("ResourceManagerClient failed to release codec with err %d", err); } *_aidl_return = (err == OK); return Status::ok(); } Status getName(::std::string* _aidl_return) override { _aidl_return->clear(); sp codec = mMediaCodec.promote(); if (codec == NULL) { // codec is already gone. return Status::ok(); } AString name; if (codec->getName(&name) == OK) { *_aidl_return = name.c_str(); } return Status::ok(); } virtual ~ResourceManagerClient() {} private: wp mMediaCodec; int32_t mPid; int32_t mUid; DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); }; /* * Proxy for ResourceManagerService that communicates with the * ResourceManagerService for MediaCodec */ struct MediaCodec::ResourceManagerServiceProxy : public std::enable_shared_from_this { // BinderDiedContext defines the cookie that is passed as DeathRecipient. // Since this can maintain more context than a raw pointer, we can // validate the scope of ResourceManagerServiceProxy, // before deferencing it upon the binder death. struct BinderDiedContext { std::weak_ptr mRMServiceProxy; }; ResourceManagerServiceProxy(pid_t pid, uid_t uid, const std::shared_ptr &client); ~ResourceManagerServiceProxy(); status_t init(); void addResource(const MediaResourceParcel &resource); void removeResource(const MediaResourceParcel &resource); void removeClient(); void markClientForPendingRemoval(); bool reclaimResource(const std::vector &resources); void notifyClientCreated(); void notifyClientStarted(ClientConfigParcel& clientConfig); void notifyClientStopped(ClientConfigParcel& clientConfig); void notifyClientConfigChanged(ClientConfigParcel& clientConfig); inline void setCodecName(const char* name) { mCodecName = name; } inline void setImportance(int importance) { mImportance = importance; } private: // To get the binder interface to ResourceManagerService. void getService() { std::scoped_lock lock{mLock}; getService_l(); } std::shared_ptr getService_l(); // To add/register all the resources currently added/registered with // the ResourceManagerService. // This function will be called right after the death of the Resource // Manager to make sure that the newly started ResourceManagerService // knows about the current resource usage. void reRegisterAllResources_l(); void deinit() { std::scoped_lock lock{mLock}; // Unregistering from DeathRecipient notification. if (mService != nullptr) { AIBinder_unlinkToDeath(mService->asBinder().get(), mDeathRecipient.get(), mCookie); mService = nullptr; } } // For binder death handling static void BinderDiedCallback(void* cookie); static void BinderUnlinkedCallback(void* cookie); void binderDied() { std::scoped_lock lock{mLock}; ALOGE("ResourceManagerService died."); mService = nullptr; mBinderDied = true; // start an async operation that will reconnect with the RM and // re-registers all the resources. mGetServiceFuture = std::async(std::launch::async, [this] { getService(); }); } /** * Get the ClientInfo to communicate with the ResourceManager. * * ClientInfo includes: * - {pid, uid} of the process * - identifier for the client * - name of the client/codec * - importance associated with the client */ inline ClientInfoParcel getClientInfo() const { ClientInfoParcel clientInfo{.pid = static_cast(mPid), .uid = static_cast(mUid), .id = getId(mClient), .name = mCodecName, .importance = mImportance}; return clientInfo; } private: std::mutex mLock; bool mBinderDied = false; pid_t mPid; uid_t mUid; int mImportance = 0; std::string mCodecName; /** * Reconnecting with the ResourceManagerService, after its binder interface dies, * is done asynchronously. It will also make sure that, all the resources * asssociated with this Proxy (MediaCodec) is added with the new instance * of the ResourceManagerService to persist the state of resources. * We must store the reference of the furture to guarantee real asynchronous operation. */ std::future mGetServiceFuture; // To maintain the list of all the resources currently added/registered with // the ResourceManagerService. std::set mMediaResourceParcel; std::shared_ptr mClient; ::ndk::ScopedAIBinder_DeathRecipient mDeathRecipient; std::shared_ptr mService; BinderDiedContext* mCookie; }; MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy( pid_t pid, uid_t uid, const std::shared_ptr &client) : mPid(pid), mUid(uid), mClient(client), mDeathRecipient(::ndk::ScopedAIBinder_DeathRecipient( AIBinder_DeathRecipient_new(BinderDiedCallback))), mCookie(nullptr) { if (mUid == MediaCodec::kNoUid) { mUid = AIBinder_getCallingUid(); } if (mPid == MediaCodec::kNoPid) { mPid = AIBinder_getCallingPid(); } // Setting callback notification when DeathRecipient gets deleted. AIBinder_DeathRecipient_setOnUnlinked(mDeathRecipient.get(), BinderUnlinkedCallback); } MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { deinit(); } status_t MediaCodec::ResourceManagerServiceProxy::init() { std::scoped_lock lock{mLock}; int callerPid = AIBinder_getCallingPid(); int callerUid = AIBinder_getCallingUid(); if (mPid != callerPid || mUid != callerUid) { // Media processes don't need special permissions to act on behalf of other processes. if (callerUid != AID_MEDIA) { char const * permission = "android.permission.MEDIA_RESOURCE_OVERRIDE_PID"; if (!checkCallingPermission(String16(permission))) { ALOGW("%s is required to override the caller's PID for media resource management.", permission); return PERMISSION_DENIED; } } } mService = getService_l(); if (mService == nullptr) { return DEAD_OBJECT; } // Kill clients pending removal. mService->reclaimResourcesFromClientsPendingRemoval(mPid); return OK; } std::shared_ptr MediaCodec::ResourceManagerServiceProxy::getService_l() { if (mService != nullptr) { return mService; } // Get binder interface to resource manager. ::ndk::SpAIBinder binder(AServiceManager_waitForService("media.resource_manager")); mService = IResourceManagerService::fromBinder(binder); if (mService == nullptr) { ALOGE("Failed to get ResourceManagerService"); return mService; } // Create the context that is passed as cookie to the binder death notification. // The context gets deleted at BinderUnlinkedCallback. mCookie = new BinderDiedContext{.mRMServiceProxy = weak_from_this()}; // Register for the callbacks by linking to death notification. AIBinder_linkToDeath(mService->asBinder().get(), mDeathRecipient.get(), mCookie); // If the RM was restarted, re-register all the resources. if (mBinderDied) { reRegisterAllResources_l(); mBinderDied = false; } return mService; } void MediaCodec::ResourceManagerServiceProxy::reRegisterAllResources_l() { if (mMediaResourceParcel.empty()) { ALOGV("No resources to add"); return; } if (mService == nullptr) { ALOGW("Service isn't available"); return; } std::vector resources; std::copy(mMediaResourceParcel.begin(), mMediaResourceParcel.end(), std::back_inserter(resources)); mService->addResource(getClientInfo(), mClient, resources); } void MediaCodec::ResourceManagerServiceProxy::BinderDiedCallback(void* cookie) { BinderDiedContext* context = reinterpret_cast(cookie); // Validate the context and check if the ResourceManagerServiceProxy object is still in scope. if (context != nullptr) { std::shared_ptr thiz = context->mRMServiceProxy.lock(); if (thiz != nullptr) { thiz->binderDied(); } else { ALOGI("ResourceManagerServiceProxy is out of scope already"); } } } void MediaCodec::ResourceManagerServiceProxy::BinderUnlinkedCallback(void* cookie) { BinderDiedContext* context = reinterpret_cast(cookie); // Since we don't need the context anymore, we are deleting it now. delete context; } void MediaCodec::ResourceManagerServiceProxy::addResource( const MediaResourceParcel &resource) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } std::vector resources; resources.push_back(resource); service->addResource(getClientInfo(), mClient, resources); mMediaResourceParcel.emplace(resource); } void MediaCodec::ResourceManagerServiceProxy::removeResource( const MediaResourceParcel &resource) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } std::vector resources; resources.push_back(resource); service->removeResource(getClientInfo(), resources); mMediaResourceParcel.erase(resource); } void MediaCodec::ResourceManagerServiceProxy::removeClient() { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } service->removeClient(getClientInfo()); mMediaResourceParcel.clear(); } void MediaCodec::ResourceManagerServiceProxy::markClientForPendingRemoval() { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } service->markClientForPendingRemoval(getClientInfo()); mMediaResourceParcel.clear(); } bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( const std::vector &resources) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return false; } bool success; Status status = service->reclaimResource(getClientInfo(), resources, &success); return status.isOk() && success; } void MediaCodec::ResourceManagerServiceProxy::notifyClientCreated() { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } service->notifyClientCreated(getClientInfo()); } void MediaCodec::ResourceManagerServiceProxy::notifyClientStarted( ClientConfigParcel& clientConfig) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } clientConfig.clientInfo = getClientInfo(); service->notifyClientStarted(clientConfig); } void MediaCodec::ResourceManagerServiceProxy::notifyClientStopped( ClientConfigParcel& clientConfig) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } clientConfig.clientInfo = getClientInfo(); service->notifyClientStopped(clientConfig); } void MediaCodec::ResourceManagerServiceProxy::notifyClientConfigChanged( ClientConfigParcel& clientConfig) { std::scoped_lock lock{mLock}; std::shared_ptr service = getService_l(); if (service == nullptr) { ALOGW("Service isn't available"); return; } clientConfig.clientInfo = getClientInfo(); service->notifyClientConfigChanged(clientConfig); } //////////////////////////////////////////////////////////////////////////////// MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {} //////////////////////////////////////////////////////////////////////////////// class MediaCodec::ReleaseSurface { public: explicit ReleaseSurface(uint64_t usage) { BufferQueue::createBufferQueue(&mProducer, &mConsumer); mSurface = new Surface(mProducer, false /* controlledByApp */); struct ConsumerListener : public BnConsumerListener { ConsumerListener(const sp &consumer) { mConsumer = consumer; } void onFrameAvailable(const BufferItem&) override { BufferItem buffer; // consume buffer sp consumer = mConsumer.promote(); if (consumer != nullptr && consumer->acquireBuffer(&buffer, 0) == NO_ERROR) { consumer->releaseBuffer(buffer.mSlot, buffer.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, buffer.mFence); } } wp mConsumer; void onBuffersReleased() override {} void onSidebandStreamChanged() override {} }; sp listener{new ConsumerListener(mConsumer)}; mConsumer->consumerConnect(listener, false); mConsumer->setConsumerName(String8{"MediaCodec.release"}); mConsumer->setConsumerUsageBits(usage); } const sp &getSurface() { return mSurface; } private: sp mProducer; sp mConsumer; sp mSurface; }; //////////////////////////////////////////////////////////////////////////////// namespace { enum { kWhatFillThisBuffer = 'fill', kWhatDrainThisBuffer = 'drai', kWhatEOS = 'eos ', kWhatStartCompleted = 'Scom', kWhatStopCompleted = 'scom', kWhatReleaseCompleted = 'rcom', kWhatFlushCompleted = 'fcom', kWhatError = 'erro', kWhatCryptoError = 'ercp', kWhatComponentAllocated = 'cAll', kWhatComponentConfigured = 'cCon', kWhatInputSurfaceCreated = 'isfc', kWhatInputSurfaceAccepted = 'isfa', kWhatSignaledInputEOS = 'seos', kWhatOutputFramesRendered = 'outR', kWhatOutputBuffersChanged = 'outC', kWhatFirstTunnelFrameReady = 'ftfR', kWhatPollForRenderedBuffers = 'plrb', kWhatMetricsUpdated = 'mtru', }; class CryptoAsyncCallback : public CryptoAsync::CryptoAsyncCallback { public: explicit CryptoAsyncCallback(const sp & notify):mNotify(notify) { } ~CryptoAsyncCallback() {} void onDecryptComplete(const sp &result) override { (void)result; } void onDecryptError(const std::list> &errorMsgs) override { // This error may be decrypt/queue error. status_t errorCode ; for (auto &emsg : errorMsgs) { sp notify(mNotify->dup()); if(emsg->findInt32("err", &errorCode)) { if (isCryptoError(errorCode)) { notify->setInt32("what", kWhatCryptoError); } else { notify->setInt32("what", kWhatError); } notify->extend(emsg); notify->post(); } else { ALOGW("Buffers with no errorCode are not expected"); } } } private: const sp mNotify; }; class OnBufferReleasedListener : public ::android::BnProducerListener{ private: uint32_t mGeneration; std::weak_ptr mBufferChannel; void notifyBufferReleased() { auto p = mBufferChannel.lock(); if (p) { p->onBufferReleasedFromOutputSurface(mGeneration); } } void notifyBufferAttached() { auto p = mBufferChannel.lock(); if (p) { p->onBufferAttachedToOutputSurface(mGeneration); } } public: explicit OnBufferReleasedListener( uint32_t generation, const std::shared_ptr &bufferChannel) : mGeneration(generation), mBufferChannel(bufferChannel) {} virtual ~OnBufferReleasedListener() = default; void onBufferReleased() override { notifyBufferReleased(); } void onBufferDetached([[maybe_unused]] int slot) override { notifyBufferReleased(); } bool needsReleaseNotify() override { return true; } #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_CONSUMER_ATTACH_CALLBACK) void onBufferAttached() override { notifyBufferAttached(); } bool needsAttachNotify() override { return true; } #endif }; class BufferCallback : public CodecBase::BufferCallback { public: explicit BufferCallback(const sp ¬ify); virtual ~BufferCallback() = default; virtual void onInputBufferAvailable( size_t index, const sp &buffer) override; virtual void onOutputBufferAvailable( size_t index, const sp &buffer) override; private: const sp mNotify; }; BufferCallback::BufferCallback(const sp ¬ify) : mNotify(notify) {} void BufferCallback::onInputBufferAvailable( size_t index, const sp &buffer) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatFillThisBuffer); notify->setSize("index", index); notify->setObject("buffer", buffer); notify->post(); } void BufferCallback::onOutputBufferAvailable( size_t index, const sp &buffer) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatDrainThisBuffer); notify->setSize("index", index); notify->setObject("buffer", buffer); notify->post(); } class CodecCallback : public CodecBase::CodecCallback { public: explicit CodecCallback(const sp ¬ify); virtual ~CodecCallback() = default; virtual void onEos(status_t err) override; virtual void onStartCompleted() override; virtual void onStopCompleted() override; virtual void onReleaseCompleted() override; virtual void onFlushCompleted() override; virtual void onError(status_t err, enum ActionCode actionCode) override; virtual void onComponentAllocated(const char *componentName) override; virtual void onComponentConfigured( const sp &inputFormat, const sp &outputFormat) override; virtual void onInputSurfaceCreated( const sp &inputFormat, const sp &outputFormat, const sp &inputSurface) override; virtual void onInputSurfaceCreationFailed(status_t err) override; virtual void onInputSurfaceAccepted( const sp &inputFormat, const sp &outputFormat) override; virtual void onInputSurfaceDeclined(status_t err) override; virtual void onSignaledInputEOS(status_t err) override; virtual void onOutputFramesRendered(const std::list &done) override; virtual void onOutputBuffersChanged() override; virtual void onFirstTunnelFrameReady() override; virtual void onMetricsUpdated(const sp &updatedMetrics) override; private: const sp mNotify; }; CodecCallback::CodecCallback(const sp ¬ify) : mNotify(notify) {} void CodecCallback::onEos(status_t err) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatEOS); notify->setInt32("err", err); notify->post(); } void CodecCallback::onStartCompleted() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatStartCompleted); notify->post(); } void CodecCallback::onStopCompleted() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatStopCompleted); notify->post(); } void CodecCallback::onReleaseCompleted() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatReleaseCompleted); notify->post(); } void CodecCallback::onFlushCompleted() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatFlushCompleted); notify->post(); } void CodecCallback::onError(status_t err, enum ActionCode actionCode) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatError); notify->setInt32("err", err); notify->setInt32("actionCode", actionCode); notify->post(); } void CodecCallback::onComponentAllocated(const char *componentName) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatComponentAllocated); notify->setString("componentName", componentName); notify->post(); } void CodecCallback::onComponentConfigured( const sp &inputFormat, const sp &outputFormat) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatComponentConfigured); notify->setMessage("input-format", inputFormat); notify->setMessage("output-format", outputFormat); notify->post(); } void CodecCallback::onInputSurfaceCreated( const sp &inputFormat, const sp &outputFormat, const sp &inputSurface) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatInputSurfaceCreated); notify->setMessage("input-format", inputFormat); notify->setMessage("output-format", outputFormat); notify->setObject("input-surface", inputSurface); notify->post(); } void CodecCallback::onInputSurfaceCreationFailed(status_t err) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatInputSurfaceCreated); notify->setInt32("err", err); notify->post(); } void CodecCallback::onInputSurfaceAccepted( const sp &inputFormat, const sp &outputFormat) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatInputSurfaceAccepted); notify->setMessage("input-format", inputFormat); notify->setMessage("output-format", outputFormat); notify->post(); } void CodecCallback::onInputSurfaceDeclined(status_t err) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatInputSurfaceAccepted); notify->setInt32("err", err); notify->post(); } void CodecCallback::onSignaledInputEOS(status_t err) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatSignaledInputEOS); if (err != OK) { notify->setInt32("err", err); } notify->post(); } void CodecCallback::onOutputFramesRendered(const std::list &done) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatOutputFramesRendered); if (MediaCodec::CreateFramesRenderedMessage(done, notify)) { notify->post(); } } void CodecCallback::onOutputBuffersChanged() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatOutputBuffersChanged); notify->post(); } void CodecCallback::onFirstTunnelFrameReady() { sp notify(mNotify->dup()); notify->setInt32("what", kWhatFirstTunnelFrameReady); notify->post(); } void CodecCallback::onMetricsUpdated(const sp &updatedMetrics) { sp notify(mNotify->dup()); notify->setInt32("what", kWhatMetricsUpdated); notify->setMessage("updated-metrics", updatedMetrics); notify->post(); } static MediaResourceSubType toMediaResourceSubType(bool isHardware, MediaCodec::Domain domain) { switch (domain) { case MediaCodec::DOMAIN_VIDEO: return isHardware? MediaResourceSubType::kHwVideoCodec : MediaResourceSubType::kSwVideoCodec; case MediaCodec::DOMAIN_AUDIO: return isHardware? MediaResourceSubType::kHwAudioCodec : MediaResourceSubType::kSwAudioCodec; case MediaCodec::DOMAIN_IMAGE: return isHardware? MediaResourceSubType::kHwImageCodec : MediaResourceSubType::kSwImageCodec; default: return MediaResourceSubType::kUnspecifiedSubType; } } static const char * toCodecMode(MediaCodec::Domain domain) { switch (domain) { case MediaCodec::DOMAIN_VIDEO: return kCodecModeVideo; case MediaCodec::DOMAIN_AUDIO: return kCodecModeAudio; case MediaCodec::DOMAIN_IMAGE: return kCodecModeImage; default: return kCodecModeUnknown; } } } // namespace //////////////////////////////////////////////////////////////////////////////// // static sp MediaCodec::CreateByType( const sp &looper, const AString &mime, bool encoder, status_t *err, pid_t pid, uid_t uid) { sp format; return CreateByType(looper, mime, encoder, err, pid, uid, format); } sp MediaCodec::CreateByType( const sp &looper, const AString &mime, bool encoder, status_t *err, pid_t pid, uid_t uid, sp format) { Vector matchingCodecs; MediaCodecList::findMatchingCodecs( mime.c_str(), encoder, 0, format, &matchingCodecs); if (err != NULL) { *err = NAME_NOT_FOUND; } for (size_t i = 0; i < matchingCodecs.size(); ++i) { sp codec = new MediaCodec(looper, pid, uid); AString componentName = matchingCodecs[i]; status_t ret = codec->init(componentName); if (err != NULL) { *err = ret; } if (ret == OK) { return codec; } ALOGD("Allocating component '%s' failed (%d), try next one.", componentName.c_str(), ret); } return NULL; } // static sp MediaCodec::CreateByComponentName( const sp &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) { sp codec = new MediaCodec(looper, pid, uid); const status_t ret = codec->init(name); if (err != NULL) { *err = ret; } return ret == OK ? codec : NULL; // NULL deallocates codec. } // static sp MediaCodec::CreatePersistentInputSurface() { sp pluginSurface = CCodec::CreateInputSurface(); if (pluginSurface != nullptr) { return pluginSurface; } OMXClient client; if (client.connect() != OK) { ALOGE("Failed to connect to OMX to create persistent input surface."); return NULL; } sp omx = client.interface(); sp bufferProducer; sp bufferSource; status_t err = omx->createInputSurface(&bufferProducer, &bufferSource); if (err != OK) { ALOGE("Failed to create persistent input surface."); return NULL; } return new PersistentSurface(bufferProducer, bufferSource); } //static status_t MediaCodec::getGloballyAvailableResources(std::vector& resources) { resources.clear(); // Make sure codec availability feature is on. if (!android::media::codec::codec_availability()) { return ERROR_UNSUPPORTED; } // TODO: For now this is just an empty function. // The actual implementation should use component store to query the // available resources from hal, and fill in resources with the same. return ERROR_UNSUPPORTED; } // GenerateCodecId generates a 64bit Random ID for each codec that is created. // The Codec ID is generated as: // - A process-unique random high 32bits // - An atomic sequence low 32bits // static uint64_t GenerateCodecId() { static std::atomic_uint64_t sId = [] { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution distrib(0, UINT32_MAX); uint32_t randomID = distrib(gen); uint64_t id = randomID; return id << 32; }(); return sId++; } MediaCodec::MediaCodec( const sp &looper, pid_t pid, uid_t uid, std::function(const AString &, const char *)> getCodecBase, std::function *)> getCodecInfo) : mState(UNINITIALIZED), mReleasedByResourceManager(false), mLooper(looper), mCodec(NULL), mReplyID(0), mFlags(0), mStickyError(OK), mSoftRenderer(NULL), mDomain(DOMAIN_UNKNOWN), mWidth(0), mHeight(0), mRotationDegrees(0), mDequeueInputTimeoutGeneration(0), mDequeueInputReplyID(0), mDequeueOutputTimeoutGeneration(0), mDequeueOutputReplyID(0), mTunneledInputWidth(0), mTunneledInputHeight(0), mTunneled(false), mTunnelPeekState(TunnelPeekState::kLegacyMode), mTunnelPeekEnabled(false), mHaveInputSurface(false), mHavePendingInputBuffers(false), mCpuBoostRequested(false), mIsSurfaceToDisplay(false), mAreRenderMetricsEnabled(areRenderMetricsEnabled()), mVideoRenderQualityTracker( VideoRenderQualityTracker::Configuration::getFromServerConfigurableFlags( GetServerConfigurableFlag)), mLatencyUnknown(0), mBytesEncoded(0), mEarliestEncodedPtsUs(INT64_MAX), mLatestEncodedPtsUs(INT64_MIN), mFramesEncoded(0), mNumLowLatencyEnables(0), mNumLowLatencyDisables(0), mIsLowLatencyModeOn(false), mIndexOfFirstFrameWhenLowLatencyOn(-1), mInputBufferCounter(0), mGetCodecBase(getCodecBase), mGetCodecInfo(getCodecInfo) { mCodecId = GenerateCodecId(); mResourceManagerProxy = std::make_shared(pid, uid, ::ndk::SharedRefBase::make(this, pid, uid)); if (!mGetCodecBase) { mGetCodecBase = [](const AString &name, const char *owner) { return GetCodecBase(name, owner); }; } if (!mGetCodecInfo) { mGetCodecInfo = [&log = mErrorLog](const AString &name, sp *info) -> status_t { *info = nullptr; const sp mcl = MediaCodecList::getInstance(); if (!mcl) { log.log(LOG_TAG, "Fatal error: failed to initialize MediaCodecList"); return NO_INIT; // if called from Java should raise IOException } AString tmp = name; if (tmp.endsWith(".secure")) { tmp.erase(tmp.size() - 7, 7); } for (const AString &codecName : { name, tmp }) { ssize_t codecIdx = mcl->findCodecByName(codecName.c_str()); if (codecIdx < 0) { continue; } *info = mcl->getCodecInfo(codecIdx); return OK; } log.log(LOG_TAG, base::StringPrintf("Codec with name '%s' is not found on the device.", name.c_str())); return NAME_NOT_FOUND; }; } // we want an empty metrics record for any early getMetrics() call // this should be the *only* initMediametrics() call that's not on the Looper thread initMediametrics(); } MediaCodec::~MediaCodec() { CHECK_EQ(mState, UNINITIALIZED); mResourceManagerProxy->removeClient(); flushMediametrics(); // this deletes mMetricsHandle // don't keep the last metrics handle around if (mLastMetricsHandle != 0) { mediametrics_delete(mLastMetricsHandle); mLastMetricsHandle = 0; } // clean any saved metrics info we stored as part of configure() if (mConfigureMsg != nullptr) { mediametrics_handle_t metricsHandle; if (mConfigureMsg->findInt64("metrics", &metricsHandle)) { mediametrics_delete(metricsHandle); } } } // except for in constructor, called from the looper thread (and therefore not mutexed) void MediaCodec::initMediametrics() { if (mMetricsHandle == 0) { mMetricsHandle = mediametrics_create(kCodecKeyName); } mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor); { Mutex::Autolock al(mRecentLock); for (int i = 0; i 0) { mediametrics_setInt64(mMetricsHandle, kCodecFirstRenderTimeUs, m.firstRenderTimeUs); mediametrics_setInt64(mMetricsHandle, kCodecLastRenderTimeUs, m.lastRenderTimeUs); mediametrics_setInt64(mMetricsHandle, kCodecFramesReleased, m.frameReleasedCount); mediametrics_setInt64(mMetricsHandle, kCodecFramesRendered, m.frameRenderedCount); mediametrics_setInt64(mMetricsHandle, kCodecFramesSkipped, m.frameSkippedCount); mediametrics_setInt64(mMetricsHandle, kCodecFramesDropped, m.frameDroppedCount); mediametrics_setDouble(mMetricsHandle, kCodecFramerateContent, m.contentFrameRate); mediametrics_setDouble(mMetricsHandle, kCodecFramerateDesired, m.desiredFrameRate); mediametrics_setDouble(mMetricsHandle, kCodecFramerateActual, m.actualFrameRate); } if (m.freezeDurationMsHistogram.getCount() >= 1) { const MediaHistogram &h = m.freezeDurationMsHistogram; mediametrics_setInt64(mMetricsHandle, kCodecFreezeScore, m.freezeScore); mediametrics_setDouble(mMetricsHandle, kCodecFreezeRate, m.freezeRate); mediametrics_setInt64(mMetricsHandle, kCodecFreezeCount, h.getCount()); mediametrics_setInt32(mMetricsHandle, kCodecFreezeDurationMsAvg, h.getAvg()); mediametrics_setInt32(mMetricsHandle, kCodecFreezeDurationMsMax, h.getMax()); mediametrics_setString(mMetricsHandle, kCodecFreezeDurationMsHistogram, h.emit()); mediametrics_setString(mMetricsHandle, kCodecFreezeDurationMsHistogramBuckets, h.emitBuckets()); } if (m.freezeDistanceMsHistogram.getCount() >= 1) { const MediaHistogram &h = m.freezeDistanceMsHistogram; mediametrics_setInt32(mMetricsHandle, kCodecFreezeDistanceMsAvg, h.getAvg()); mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceMsHistogram, h.emit()); mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceMsHistogramBuckets, h.emitBuckets()); } if (m.judderScoreHistogram.getCount() >= 1) { const MediaHistogram &h = m.judderScoreHistogram; mediametrics_setInt64(mMetricsHandle, kCodecJudderScore, m.judderScore); mediametrics_setDouble(mMetricsHandle, kCodecJudderRate, m.judderRate); mediametrics_setInt64(mMetricsHandle, kCodecJudderCount, h.getCount()); mediametrics_setInt32(mMetricsHandle, kCodecJudderScoreAvg, h.getAvg()); mediametrics_setInt32(mMetricsHandle, kCodecJudderScoreMax, h.getMax()); mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogram, h.emit()); mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogramBuckets, h.emitBuckets()); } if (m.freezeEventCount != 0) { mediametrics_setInt32(mMetricsHandle, kCodecFreezeEventCount, m.freezeEventCount); } if (m.judderEventCount != 0) { mediametrics_setInt32(mMetricsHandle, kCodecJudderEventCount, m.judderEventCount); } } if (mLatencyHist.getCount() != 0 ) { mediametrics_setInt64(mMetricsHandle, kCodecLatencyMax, mLatencyHist.getMax()); mediametrics_setInt64(mMetricsHandle, kCodecLatencyMin, mLatencyHist.getMin()); mediametrics_setInt64(mMetricsHandle, kCodecLatencyAvg, mLatencyHist.getAvg()); mediametrics_setInt64(mMetricsHandle, kCodecLatencyCount, mLatencyHist.getCount()); if (kEmitHistogram) { // and the histogram itself std::string hist = mLatencyHist.emit(); mediametrics_setCString(mMetricsHandle, kCodecLatencyHist, hist.c_str()); } } if (mLatencyUnknown > 0) { mediametrics_setInt64(mMetricsHandle, kCodecLatencyUnknown, mLatencyUnknown); } int64_t playbackDurationSec = mPlaybackDurationAccumulator.getDurationInSeconds(); if (playbackDurationSec > 0) { mediametrics_setInt64(mMetricsHandle, kCodecPlaybackDurationSec, playbackDurationSec); } if (mLifetimeStartNs > 0) { nsecs_t lifetime = systemTime(SYSTEM_TIME_MONOTONIC) - mLifetimeStartNs; lifetime = lifetime / (1000 * 1000); // emitted in ms, truncated not rounded mediametrics_setInt64(mMetricsHandle, kCodecLifetimeMs, lifetime); } if (mBytesEncoded) { Mutex::Autolock al(mOutputStatsLock); mediametrics_setInt64(mMetricsHandle, kCodecVideoEncodedBytes, mBytesEncoded); int64_t duration = 0; if (mLatestEncodedPtsUs > mEarliestEncodedPtsUs) { duration = mLatestEncodedPtsUs - mEarliestEncodedPtsUs; } mediametrics_setInt64(mMetricsHandle, kCodecVideoEncodedDurationUs, duration); mediametrics_setInt64(mMetricsHandle, kCodecVideoEncodedFrames, mFramesEncoded); mediametrics_setInt64(mMetricsHandle, kCodecVideoInputFrames, mFramesInput); mediametrics_setInt64(mMetricsHandle, kCodecVideoInputBytes, mBytesInput); } { Mutex::Autolock al(mLatencyLock); mediametrics_setInt64(mMetricsHandle, kCodecNumLowLatencyModeOn, mNumLowLatencyEnables); mediametrics_setInt64(mMetricsHandle, kCodecNumLowLatencyModeOff, mNumLowLatencyDisables); mediametrics_setInt64(mMetricsHandle, kCodecFirstFrameIndexLowLatencyModeOn, mIndexOfFirstFrameWhenLowLatencyOn); } #if 0 // enable for short term, only while debugging updateEphemeralMediametrics(mMetricsHandle); #endif } void MediaCodec::updateHdrMetrics(bool isConfig) { if ((mDomain != DOMAIN_VIDEO && mDomain != DOMAIN_IMAGE) || mMetricsHandle == 0) { return; } int32_t colorStandard = -1; if (mOutputFormat->findInt32(KEY_COLOR_STANDARD, &colorStandard)) { mediametrics_setInt32(mMetricsHandle, isConfig ? kCodecConfigColorStandard : kCodecParsedColorStandard, colorStandard); } int32_t colorRange = -1; if (mOutputFormat->findInt32(KEY_COLOR_RANGE, &colorRange)) { mediametrics_setInt32(mMetricsHandle, isConfig ? kCodecConfigColorRange : kCodecParsedColorRange, colorRange); } int32_t colorTransfer = -1; if (mOutputFormat->findInt32(KEY_COLOR_TRANSFER, &colorTransfer)) { mediametrics_setInt32(mMetricsHandle, isConfig ? kCodecConfigColorTransfer : kCodecParsedColorTransfer, colorTransfer); } HDRStaticInfo info; if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info) && ColorUtils::isHDRStaticInfoValid(&info)) { mHdrInfoFlags |= kFlagHasHdrStaticInfo; } mediametrics_setInt32(mMetricsHandle, kCodecHdrStaticInfo, (mHdrInfoFlags & kFlagHasHdrStaticInfo) ? 1 : 0); sp hdr10PlusInfo; if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo) && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) { mHdrInfoFlags |= kFlagHasHdr10PlusInfo; } mediametrics_setInt32(mMetricsHandle, kCodecHdr10PlusInfo, (mHdrInfoFlags & kFlagHasHdr10PlusInfo) ? 1 : 0); // hdr format sp codedFormat = (mFlags & kFlagIsEncoder) ? mOutputFormat : mInputFormat; AString mime; int32_t profile = -1; if (codedFormat->findString("mime", &mime) && codedFormat->findInt32(KEY_PROFILE, &profile) && colorTransfer != -1) { hdr_format hdrFormat = getHdrFormat(mime, profile, colorTransfer); mediametrics_setInt32(mMetricsHandle, kCodecHdrFormat, static_cast(hdrFormat)); } } hdr_format MediaCodec::getHdrFormat(const AString &mime, const int32_t profile, const int32_t colorTransfer) { return (mFlags & kFlagIsEncoder) ? getHdrFormatForEncoder(mime, profile, colorTransfer) : getHdrFormatForDecoder(mime, profile, colorTransfer); } hdr_format MediaCodec::getHdrFormatForEncoder(const AString &mime, const int32_t profile, const int32_t colorTransfer) { switch (colorTransfer) { case COLOR_TRANSFER_ST2084: if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_VP9)) { switch (profile) { case VP9Profile2HDR: return HDR_FORMAT_HDR10; case VP9Profile2HDR10Plus: return HDR_FORMAT_HDR10PLUS; default: return HDR_FORMAT_NONE; } } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_AV1)) { switch (profile) { case AV1ProfileMain10HDR10: return HDR_FORMAT_HDR10; case AV1ProfileMain10HDR10Plus: return HDR_FORMAT_HDR10PLUS; default: return HDR_FORMAT_NONE; } } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_HEVC)) { switch (profile) { case HEVCProfileMain10HDR10: return HDR_FORMAT_HDR10; case HEVCProfileMain10HDR10Plus: return HDR_FORMAT_HDR10PLUS; default: return HDR_FORMAT_NONE; } } else { return HDR_FORMAT_NONE; } case COLOR_TRANSFER_HLG: if (!mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) { return HDR_FORMAT_HLG; } else { // TODO: DOLBY format return HDR_FORMAT_NONE; } default: return HDR_FORMAT_NONE; } } hdr_format MediaCodec::getHdrFormatForDecoder(const AString &mime, const int32_t profile, const int32_t colorTransfer) { switch (colorTransfer) { case COLOR_TRANSFER_ST2084: if (!(mHdrInfoFlags & kFlagHasHdrStaticInfo) || !profileSupport10Bits(mime, profile)) { return HDR_FORMAT_NONE; } return mHdrInfoFlags & kFlagHasHdr10PlusInfo ? HDR_FORMAT_HDR10PLUS : HDR_FORMAT_HDR10; case COLOR_TRANSFER_HLG: if (!mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_DOLBY_VISION)) { return HDR_FORMAT_HLG; } // TODO: DOLBY format } return HDR_FORMAT_NONE; } bool MediaCodec::profileSupport10Bits(const AString &mime, const int32_t profile) { if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_AV1)) { return true; } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_VP9)) { switch (profile) { case VP9Profile2: case VP9Profile3: case VP9Profile2HDR: case VP9Profile3HDR: case VP9Profile2HDR10Plus: case VP9Profile3HDR10Plus: return true; } } else if (mime.equalsIgnoreCase(MEDIA_MIMETYPE_VIDEO_HEVC)) { switch (profile) { case HEVCProfileMain10: case HEVCProfileMain10HDR10: case HEVCProfileMain10HDR10Plus: return true; } } return false; } // called to update info being passed back via getMetrics(), which is a // unique copy for that call, no concurrent access worries. void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) { ALOGD("MediaCodec::updateEphemeralMediametrics()"); if (item == 0) { return; } // build an empty histogram MediaHistogram recentHist; recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor); // stuff it with the samples in the ring buffer { Mutex::Autolock al(mRecentLock); for (int i = 0; i < kRecentLatencyFrames; i++) { if (mRecentSamples[i] != kRecentSampleInvalid) { recentHist.insert(mRecentSamples[i]); } } } // spit the data (if any) into the supplied analytics record if (recentHist.getCount() != 0 ) { mediametrics_setInt64(item, kCodecRecentLatencyMax, recentHist.getMax()); mediametrics_setInt64(item, kCodecRecentLatencyMin, recentHist.getMin()); mediametrics_setInt64(item, kCodecRecentLatencyAvg, recentHist.getAvg()); mediametrics_setInt64(item, kCodecRecentLatencyCount, recentHist.getCount()); if (kEmitHistogram) { // and the histogram itself std::string hist = recentHist.emit(); mediametrics_setCString(item, kCodecRecentLatencyHist, hist.c_str()); } } } static std::string emitVector(std::vector vector) { std::ostringstream sstr; for (size_t i = 0; i < vector.size(); ++i) { if (i != 0) { sstr << ','; } sstr << vector[i]; } return sstr.str(); } static void reportToMediaMetricsIfValid(const FreezeEvent &e) { if (e.valid) { mediametrics_handle_t handle = mediametrics_create(kFreezeEventKeyName); mediametrics_setInt64(handle, kFreezeEventInitialTimeUs, e.initialTimeUs); mediametrics_setInt32(handle, kFreezeEventDurationMs, e.durationMs); mediametrics_setInt64(handle, kFreezeEventCount, e.count); mediametrics_setInt32(handle, kFreezeEventAvgDurationMs, e.sumDurationMs / e.count); mediametrics_setInt32(handle, kFreezeEventAvgDistanceMs, e.sumDistanceMs / e.count); mediametrics_setString(handle, kFreezeEventDetailsDurationMs, emitVector(e.details.durationMs)); mediametrics_setString(handle, kFreezeEventDetailsDistanceMs, emitVector(e.details.distanceMs)); mediametrics_selfRecord(handle); mediametrics_delete(handle); } } static void reportToMediaMetricsIfValid(const JudderEvent &e) { if (e.valid) { mediametrics_handle_t handle = mediametrics_create(kJudderEventKeyName); mediametrics_setInt64(handle, kJudderEventInitialTimeUs, e.initialTimeUs); mediametrics_setInt32(handle, kJudderEventDurationMs, e.durationMs); mediametrics_setInt64(handle, kJudderEventCount, e.count); mediametrics_setInt32(handle, kJudderEventAvgScore, e.sumScore / e.count); mediametrics_setInt32(handle, kJudderEventAvgDistanceMs, e.sumDistanceMs / e.count); mediametrics_setString(handle, kJudderEventDetailsActualDurationUs, emitVector(e.details.actualRenderDurationUs)); mediametrics_setString(handle, kJudderEventDetailsContentDurationUs, emitVector(e.details.contentRenderDurationUs)); mediametrics_setString(handle, kJudderEventDetailsDistanceMs, emitVector(e.details.distanceMs)); mediametrics_selfRecord(handle); mediametrics_delete(handle); } } // except for in destructor, called from the looper thread void MediaCodec::flushMediametrics() { ALOGV("flushMediametrics"); // update does its own mutex locking updateMediametrics(); resetMetricsFields(); // ensure mutex while we do our own work Mutex::Autolock _lock(mMetricsLock); if (mMetricsHandle != 0) { if (mMetricsToUpload && mediametrics_count(mMetricsHandle) > 0) { mediametrics_selfRecord(mMetricsHandle); } // keep previous metrics handle for subsequent getMetrics() calls. // NOTE: There could be multiple error events, each flushing the metrics. // We keep the last non-empty metrics handle, so getMetrics() in the // next call will get the latest metrics prior to the errors. if (mLastMetricsHandle != 0) { mediametrics_delete(mLastMetricsHandle); } mLastMetricsHandle = mMetricsHandle; mMetricsHandle = 0; } // we no longer have anything pending upload mMetricsToUpload = false; // Freeze and judder events are reported separately reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetFreezeEvent()); reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetJudderEvent()); } void MediaCodec::updateLowLatency(const sp &msg) { int32_t lowLatency = 0; if (msg->findInt32("low-latency", &lowLatency)) { Mutex::Autolock al(mLatencyLock); if (lowLatency > 0) { ++mNumLowLatencyEnables; // This is just an estimate since low latency mode change happens ONLY at key frame mIsLowLatencyModeOn = true; } else if (lowLatency == 0) { ++mNumLowLatencyDisables; // This is just an estimate since low latency mode change happens ONLY at key frame mIsLowLatencyModeOn = false; } } } void MediaCodec::updateCodecImportance(const sp& msg) { // Update the codec importance. int32_t importance = 0; if (msg->findInt32(KEY_IMPORTANCE, &importance)) { // Ignoring the negative importance. if (importance >= 0) { // Notify RM about the change in the importance. mResourceManagerProxy->setImportance(importance); ClientConfigParcel clientConfig; initClientConfigParcel(clientConfig); mResourceManagerProxy->notifyClientConfigChanged(clientConfig); } } } constexpr const char *MediaCodec::asString(TunnelPeekState state, const char *default_string){ switch(state) { case TunnelPeekState::kLegacyMode: return "LegacyMode"; case TunnelPeekState::kEnabledNoBuffer: return "EnabledNoBuffer"; case TunnelPeekState::kDisabledNoBuffer: return "DisabledNoBuffer"; case TunnelPeekState::kBufferDecoded: return "BufferDecoded"; case TunnelPeekState::kBufferRendered: return "BufferRendered"; case TunnelPeekState::kDisabledQueued: return "DisabledQueued"; case TunnelPeekState::kEnabledQueued: return "EnabledQueued"; default: return default_string; } } void MediaCodec::updateTunnelPeek(const sp &msg) { int32_t tunnelPeek = 0; if (!msg->findInt32("tunnel-peek", &tunnelPeek)){ return; } TunnelPeekState previousState = mTunnelPeekState; if(tunnelPeek == 0){ mTunnelPeekEnabled = false; switch (mTunnelPeekState) { case TunnelPeekState::kLegacyMode: msg->setInt32("android._tunnel-peek-set-legacy", 0); [[fallthrough]]; case TunnelPeekState::kEnabledNoBuffer: mTunnelPeekState = TunnelPeekState::kDisabledNoBuffer; break; case TunnelPeekState::kEnabledQueued: mTunnelPeekState = TunnelPeekState::kDisabledQueued; break; default: ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState)); return; } } else { mTunnelPeekEnabled = true; switch (mTunnelPeekState) { case TunnelPeekState::kLegacyMode: msg->setInt32("android._tunnel-peek-set-legacy", 0); [[fallthrough]]; case TunnelPeekState::kDisabledNoBuffer: mTunnelPeekState = TunnelPeekState::kEnabledNoBuffer; break; case TunnelPeekState::kDisabledQueued: mTunnelPeekState = TunnelPeekState::kEnabledQueued; break; case TunnelPeekState::kBufferDecoded: msg->setInt32("android._trigger-tunnel-peek", 1); mTunnelPeekState = TunnelPeekState::kBufferRendered; break; default: ALOGV("Ignoring tunnel-peek=%d for %s", tunnelPeek, asString(mTunnelPeekState)); return; } } ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); } void MediaCodec::processRenderedFrames(const sp &msg) { int what = 0; msg->findInt32("what", &what); if (msg->what() != kWhatCodecNotify && what != kWhatOutputFramesRendered) { static bool logged = false; if (!logged) { logged = true; ALOGE("processRenderedFrames: expected kWhatOutputFramesRendered (%d)", msg->what()); } return; } // Rendered frames only matter if they're being sent to the display if (mIsSurfaceToDisplay) { int64_t renderTimeNs; for (size_t index = 0; msg->findInt64(AStringPrintf("%zu-system-nano", index).c_str(), &renderTimeNs); index++) { // Capture metrics for playback duration mPlaybackDurationAccumulator.onFrameRendered(renderTimeNs); // Capture metrics for quality int64_t mediaTimeUs = 0; if (!msg->findInt64(AStringPrintf("%zu-media-time-us", index).c_str(), &mediaTimeUs)) { ALOGE("processRenderedFrames: no media time found"); continue; } // Tunneled frames use INT64_MAX to indicate end-of-stream, so don't report it as a // rendered frame. if (!mTunneled || mediaTimeUs != INT64_MAX) { FreezeEvent freezeEvent; JudderEvent judderEvent; mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs, &freezeEvent, &judderEvent); reportToMediaMetricsIfValid(freezeEvent); reportToMediaMetricsIfValid(judderEvent); } } } } // when we send a buffer to the codec; void MediaCodec::statsBufferSent(int64_t presentationUs, const sp &buffer) { // only enqueue if we have a legitimate time if (presentationUs <= 0) { ALOGV("presentation time: %" PRId64, presentationUs); return; } if (mBatteryChecker != nullptr) { mBatteryChecker->onCodecActivity([this] () { mResourceManagerProxy->addResource(MediaResource::VideoBatteryResource(mIsHardware)); }); } // NOTE: these were erroneously restricted to video encoders, but we want them for all // codecs. if (android::media::codec::provider_->subsession_metrics() || (mDomain == DOMAIN_VIDEO && (mFlags & kFlagIsEncoder))) { mBytesInput += buffer->size(); mFramesInput++; } // mutex access to mBuffersInFlight and other stats Mutex::Autolock al(mLatencyLock); // XXX: we *could* make sure that the time is later than the end of queue // as part of a consistency check... if (!mTunneled) { const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); BufferFlightTiming_t startdata = { presentationUs, nowNs }; mBuffersInFlight.push_back(startdata); } if (mIsLowLatencyModeOn && mIndexOfFirstFrameWhenLowLatencyOn < 0) { mIndexOfFirstFrameWhenLowLatencyOn = mInputBufferCounter; } ++mInputBufferCounter; } // when we get a buffer back from the codec, always called from the looper thread void MediaCodec::statsBufferReceived(int64_t presentationUs, const sp &buffer) { CHECK_NE(mState, UNINITIALIZED); // NOTE: these were erroneously restricted to video encoders, but we want them for all // codecs. if (android::media::codec::provider_->subsession_metrics() || (mDomain == DOMAIN_VIDEO && (mFlags & kFlagIsEncoder))) { int32_t flags = 0; (void) buffer->meta()->findInt32("flags", &flags); // some of these frames, we don't want to count // standalone EOS.... has an invalid timestamp if ((flags & (BUFFER_FLAG_CODECCONFIG|BUFFER_FLAG_EOS)) == 0) { mBytesEncoded += buffer->size(); mFramesEncoded++; Mutex::Autolock al(mOutputStatsLock); int64_t timeUs = 0; if (buffer->meta()->findInt64("timeUs", &timeUs)) { if (timeUs > mLatestEncodedPtsUs) { mLatestEncodedPtsUs = timeUs; } // can't chain as an else-if or this never triggers if (timeUs < mEarliestEncodedPtsUs) { mEarliestEncodedPtsUs = timeUs; } } } } // mutex access to mBuffersInFlight and other stats Mutex::Autolock al(mLatencyLock); // how long this buffer took for the round trip through the codec // NB: pipelining can/will make these times larger. e.g., if each packet // is always 2 msec and we have 3 in flight at any given time, we're going to // see "6 msec" as an answer. // ignore stuff with no presentation time if (presentationUs <= 0) { ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs); mLatencyUnknown++; return; } if (mBatteryChecker != nullptr) { mBatteryChecker->onCodecActivity([this] () { mResourceManagerProxy->addResource(MediaResource::VideoBatteryResource(mIsHardware)); }); } BufferFlightTiming_t startdata; bool valid = false; while (mBuffersInFlight.size() > 0) { startdata = *mBuffersInFlight.begin(); ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64, startdata.presentationUs, startdata.startedNs); if (startdata.presentationUs == presentationUs) { // a match ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64, startdata.presentationUs, presentationUs); mBuffersInFlight.pop_front(); valid = true; break; } else if (startdata.presentationUs < presentationUs) { // we must have missed the match for this, drop it and keep looking ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64, startdata.presentationUs, presentationUs); mBuffersInFlight.pop_front(); continue; } else { // head is after, so we don't have a frame for ourselves ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64 " we have nothing to pair with", startdata.presentationUs, presentationUs); mLatencyUnknown++; return; } } if (!valid) { ALOGV("-- empty queue, so ignore that."); mLatencyUnknown++; return; } // now start our calculations const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000; mLatencyHist.insert(latencyUs); // push into the recent samples { Mutex::Autolock al(mRecentLock); if (mRecentHead >= kRecentLatencyFrames) { mRecentHead = 0; } mRecentSamples[mRecentHead++] = latencyUs; } } bool MediaCodec::discardDecodeOnlyOutputBuffer(size_t index) { Mutex::Autolock al(mBufferLock); BufferInfo *info = &mPortBuffers[kPortIndexOutput][index]; sp buffer = info->mData; int32_t flags; CHECK(buffer->meta()->findInt32("flags", &flags)); if (flags & BUFFER_FLAG_DECODE_ONLY) { ALOGV("discardDecodeOnlyOutputBuffer: mPortBuffers[out][%zu] NOT owned by client", index); info->mOwnedByClient = false; info->mData.clear(); mBufferChannel->discardBuffer(buffer); return true; } return false; } // static status_t MediaCodec::PostAndAwaitResponse( const sp &msg, sp *response) { status_t err = msg->postAndAwaitResponse(response); if (err != OK) { return err; } if (!(*response)->findInt32("err", &err)) { err = OK; } return err; } void MediaCodec::PostReplyWithError(const sp &msg, int32_t err) { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); PostReplyWithError(replyID, err); } void MediaCodec::PostReplyWithError(const sp &replyID, int32_t err) { int32_t finalErr = err; if (mReleasedByResourceManager) { // override the err code if MediaCodec has been released by ResourceManager. finalErr = DEAD_OBJECT; } sp response = new AMessage; response->setInt32("err", finalErr); response->postReply(replyID); } static CodecBase *CreateCCodec() { return new CCodec; } //static sp MediaCodec::GetCodecBase(const AString &name, const char *owner) { if (owner) { if (strcmp(owner, "default") == 0) { return new ACodec; } else if (strncmp(owner, "codec2", 6) == 0) { return CreateCCodec(); } } if (name.startsWithIgnoreCase("c2.")) { return CreateCCodec(); } else if (name.startsWithIgnoreCase("omx.")) { // at this time only ACodec specifies a mime type. return new ACodec; } else { return NULL; } } struct CodecListCache { CodecListCache() : mCodecInfoMap{[] { const sp mcl = MediaCodecList::getInstance(); size_t count = mcl->countCodecs(); std::map> codecInfoMap; for (size_t i = 0; i < count; ++i) { sp info = mcl->getCodecInfo(i); codecInfoMap.emplace(info->getCodecName(), info); } return codecInfoMap; }()} { } const std::map> mCodecInfoMap; }; static const CodecListCache &GetCodecListCache() { static CodecListCache sCache{}; return sCache; } status_t MediaCodec::init(const AString &name) { status_t err = mResourceManagerProxy->init(); if (err != OK) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Fatal error: failed to initialize ResourceManager (err=%d)", err)); mCodec = NULL; // remove the codec return err; } // save init parameters for reset mInitName = name; // Current video decoders do not return from OMX_FillThisBuffer // quickly, violating the OpenMAX specs, until that is remedied // we need to invest in an extra looper to free the main event // queue. mCodecInfo.clear(); bool secureCodec = false; const char *owner = ""; if (!name.startsWith("android.filter.")) { err = mGetCodecInfo(name, &mCodecInfo); if (err != OK) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Getting codec info with name '%s' failed (err=%d)", name.c_str(), err)); mCodec = NULL; // remove the codec. return err; } if (mCodecInfo == nullptr) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Getting codec info with name '%s' failed", name.c_str())); return NAME_NOT_FOUND; } secureCodec = name.endsWith(".secure"); Vector mediaTypes; mCodecInfo->getSupportedMediaTypes(&mediaTypes); for (size_t i = 0; i < mediaTypes.size(); ++i) { if (mediaTypes[i].startsWith("video/")) { mDomain = DOMAIN_VIDEO; break; } else if (mediaTypes[i].startsWith("audio/")) { mDomain = DOMAIN_AUDIO; break; } else if (mediaTypes[i].startsWith("image/")) { mDomain = DOMAIN_IMAGE; break; } } owner = mCodecInfo->getOwnerName(); } mCodec = mGetCodecBase(name, owner); if (mCodec == NULL) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Getting codec base with name '%s' (from '%s' HAL) failed", name.c_str(), owner)); return NAME_NOT_FOUND; } if (mDomain == DOMAIN_VIDEO) { // video codec needs dedicated looper if (mCodecLooper == NULL) { status_t err = OK; mCodecLooper = new ALooper; mCodecLooper->setName("CodecLooper"); err = mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); if (OK != err) { mErrorLog.log(LOG_TAG, "Fatal error: codec looper failed to start"); return err; } } mCodecLooper->registerHandler(mCodec); } else { mLooper->registerHandler(mCodec); } mLooper->registerHandler(this); mCodec->setCallback( std::unique_ptr( new CodecCallback(new AMessage(kWhatCodecNotify, this)))); mBufferChannel = mCodec->getBufferChannel(); mBufferChannel->setCallback( std::unique_ptr( new BufferCallback(new AMessage(kWhatCodecNotify, this)))); sp msg = new AMessage(kWhatInit, this); if (mCodecInfo) { msg->setObject("codecInfo", mCodecInfo); // name may be different from mCodecInfo->getCodecName() if we stripped // ".secure" } msg->setString("name", name); // initial naming setup covers the period before the first call to ::configure(). // after that, we manage this through ::configure() and the setup message. if (mMetricsHandle != 0) { mediametrics_setCString(mMetricsHandle, kCodecCodec, name.c_str()); mediametrics_setCString(mMetricsHandle, kCodecMode, toCodecMode(mDomain)); } if (mDomain == DOMAIN_VIDEO) { mBatteryChecker = new BatteryChecker(new AMessage(kWhatCheckBatteryStats, this)); } // If the ComponentName is not set yet, use the name passed by the user. if (mComponentName.empty()) { mIsHardware = !MediaCodecList::isSoftwareCodec(name); mResourceManagerProxy->setCodecName(name.c_str()); } std::vector resources; resources.push_back(MediaResource::CodecResource(secureCodec, toMediaResourceSubType(mIsHardware, mDomain))); for (int i = 0; i <= kMaxRetry; ++i) { if (i > 0) { // Don't try to reclaim resource for the first time. if (!mResourceManagerProxy->reclaimResource(resources)) { break; } } sp response; err = PostAndAwaitResponse(msg, &response); if (!isResourceError(err)) { break; } } if (OK == err) { // Notify the ResourceManager that, this codec has been created // (initialized) successfully. mResourceManagerProxy->notifyClientCreated(); } return err; } status_t MediaCodec::setCallback(const sp &callback) { sp msg = new AMessage(kWhatSetCallback, this); msg->setMessage("callback", callback); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::setOnFrameRenderedNotification(const sp ¬ify) { sp msg = new AMessage(kWhatSetNotification, this); msg->setMessage("on-frame-rendered", notify); return msg->post(); } status_t MediaCodec::setOnFirstTunnelFrameReadyNotification(const sp ¬ify) { sp msg = new AMessage(kWhatSetNotification, this); msg->setMessage("first-tunnel-frame-ready", notify); return msg->post(); } /* * MediaFormat Shaping forward declarations * including the property name we use for control. */ static int enableMediaFormatShapingDefault = 1; static const char enableMediaFormatShapingProperty[] = "debug.stagefright.enableshaping"; static void mapFormat(AString componentName, const sp &format, const char *kind, bool reverse); mediametrics_handle_t MediaCodec::createMediaMetrics(const sp& format, uint32_t flags, status_t* err) { *err = OK; mediametrics_handle_t nextMetricsHandle = mediametrics_create(kCodecKeyName); bool isEncoder = (flags & CONFIGURE_FLAG_ENCODE); // TODO: validity check log-session-id: it should be a 32-hex-digit. format->findString("log-session-id", &mLogSessionId); if (nextMetricsHandle != 0) { mediametrics_setInt64(nextMetricsHandle, kCodecId, mCodecId); int32_t profile = 0; if (format->findInt32("profile", &profile)) { mediametrics_setInt32(nextMetricsHandle, kCodecProfile, profile); } int32_t level = 0; if (format->findInt32("level", &level)) { mediametrics_setInt32(nextMetricsHandle, kCodecLevel, level); } mediametrics_setInt32(nextMetricsHandle, kCodecEncoder, isEncoder); if (!mLogSessionId.empty()) { mediametrics_setCString(nextMetricsHandle, kCodecLogSessionId, mLogSessionId.c_str()); } // moved here from ::init() mediametrics_setCString(nextMetricsHandle, kCodecCodec, mInitName.c_str()); mediametrics_setCString(nextMetricsHandle, kCodecMode, toCodecMode(mDomain)); } if (mDomain == DOMAIN_VIDEO || mDomain == DOMAIN_IMAGE) { format->findInt32("width", &mWidth); format->findInt32("height", &mHeight); if (!format->findInt32("rotation-degrees", &mRotationDegrees)) { mRotationDegrees = 0; } if (nextMetricsHandle != 0) { mediametrics_setInt32(nextMetricsHandle, kCodecWidth, mWidth); mediametrics_setInt32(nextMetricsHandle, kCodecHeight, mHeight); mediametrics_setInt32(nextMetricsHandle, kCodecRotation, mRotationDegrees); int32_t maxWidth = 0; if (format->findInt32("max-width", &maxWidth)) { mediametrics_setInt32(nextMetricsHandle, kCodecMaxWidth, maxWidth); } int32_t maxHeight = 0; if (format->findInt32("max-height", &maxHeight)) { mediametrics_setInt32(nextMetricsHandle, kCodecMaxHeight, maxHeight); } int32_t colorFormat = -1; if (format->findInt32("color-format", &colorFormat)) { mediametrics_setInt32(nextMetricsHandle, kCodecColorFormat, colorFormat); } int32_t appMaxInputSize = -1; if (format->findInt32(KEY_MAX_INPUT_SIZE, &appMaxInputSize)) { mApiUsageMetrics.inputBufferSize.appMax = appMaxInputSize; } if (mDomain == DOMAIN_VIDEO) { float frameRate = -1.0; if (format->findFloat("frame-rate", &frameRate)) { mediametrics_setDouble(nextMetricsHandle, kCodecFrameRate, frameRate); } float captureRate = -1.0; if (format->findFloat("capture-rate", &captureRate)) { mediametrics_setDouble(nextMetricsHandle, kCodecCaptureRate, captureRate); } float operatingRate = -1.0; if (format->findFloat("operating-rate", &operatingRate)) { mediametrics_setDouble(nextMetricsHandle, kCodecOperatingRate, operatingRate); } int32_t priority = -1; if (format->findInt32("priority", &priority)) { mediametrics_setInt32(nextMetricsHandle, kCodecPriority, priority); } } } // Prevent possible integer overflow in downstream code. if (mWidth < 0 || mHeight < 0 || (uint64_t)mWidth * mHeight > (uint64_t)INT32_MAX / 4) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Invalid size(s), width=%d, height=%d", mWidth, mHeight)); mediametrics_delete(nextMetricsHandle); // Set the error code and return null handle. *err = BAD_VALUE; return 0; } } else { if (nextMetricsHandle != 0) { int32_t channelCount; if (format->findInt32(KEY_CHANNEL_COUNT, &channelCount)) { mediametrics_setInt32(nextMetricsHandle, kCodecChannelCount, channelCount); } int32_t sampleRate; if (format->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { mediametrics_setInt32(nextMetricsHandle, kCodecSampleRate, sampleRate); } } } if (isEncoder) { int8_t enableShaping = property_get_bool(enableMediaFormatShapingProperty, enableMediaFormatShapingDefault); if (!enableShaping) { ALOGI("format shaping disabled, property '%s'", enableMediaFormatShapingProperty); if (nextMetricsHandle != 0) { mediametrics_setInt32(nextMetricsHandle, kCodecShapingEnhanced, -1); } } else { (void) shapeMediaFormat(format, flags, nextMetricsHandle); // XXX: do we want to do this regardless of shaping enablement? mapFormat(mComponentName, format, nullptr, false); } } // push min/max QP to MediaMetrics after shaping if (mDomain == DOMAIN_VIDEO && nextMetricsHandle != 0) { int32_t qpIMin = -1; if (format->findInt32("video-qp-i-min", &qpIMin)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPIMin, qpIMin); } int32_t qpIMax = -1; if (format->findInt32("video-qp-i-max", &qpIMax)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPIMax, qpIMax); } int32_t qpPMin = -1; if (format->findInt32("video-qp-p-min", &qpPMin)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPPMin, qpPMin); } int32_t qpPMax = -1; if (format->findInt32("video-qp-p-max", &qpPMax)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPPMax, qpPMax); } int32_t qpBMin = -1; if (format->findInt32("video-qp-b-min", &qpBMin)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPBMin, qpBMin); } int32_t qpBMax = -1; if (format->findInt32("video-qp-b-max", &qpBMax)) { mediametrics_setInt32(nextMetricsHandle, kCodecRequestedVideoQPBMax, qpBMax); } } updateLowLatency(format); return nextMetricsHandle; } status_t MediaCodec::configure( const sp &format, const sp &nativeWindow, const sp &crypto, uint32_t flags) { return configure(format, nativeWindow, crypto, NULL, flags); } status_t MediaCodec::configure( const sp &format, const sp &surface, const sp &crypto, const sp &descrambler, uint32_t flags) { // Update the codec importance. updateCodecImportance(format); // Create and set up metrics for this codec. status_t err = OK; mediametrics_handle_t nextMetricsHandle = createMediaMetrics(format, flags, &err); if (err != OK) { return err; } sp msg = new AMessage(kWhatConfigure, this); msg->setMessage("format", format); msg->setInt32("flags", flags); msg->setObject("surface", surface); if (crypto != NULL || descrambler != NULL) { if (crypto != NULL) { msg->setPointer("crypto", crypto.get()); } else { msg->setPointer("descrambler", descrambler.get()); } if (nextMetricsHandle != 0) { mediametrics_setInt32(nextMetricsHandle, kCodecCrypto, 1); } } else if (mFlags & kFlagIsSecure) { // We'll catch this later when we process the buffers. ALOGW("Crypto or descrambler should be given for secure codec"); } if (mConfigureMsg != nullptr) { // if re-configuring, we have one of these from before. // Recover the space before we discard the old mConfigureMsg mediametrics_handle_t metricsHandle; if (mConfigureMsg->findInt64("metrics", &metricsHandle)) { mediametrics_delete(metricsHandle); } } msg->setInt64("metrics", nextMetricsHandle); // save msg for reset mConfigureMsg = msg; sp callback = mCallback; std::vector resources; resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, toMediaResourceSubType(mIsHardware, mDomain))); if (mDomain == DOMAIN_VIDEO || mDomain == DOMAIN_IMAGE) { // Don't know the buffer size at this point, but it's fine to use 1 because // the reclaimResource call doesn't consider the requester's buffer size for now. resources.push_back(MediaResource::GraphicMemoryResource(1)); } for (int i = 0; i <= kMaxRetry; ++i) { sp response; err = PostAndAwaitResponse(msg, &response); if (err != OK && err != INVALID_OPERATION) { if (isResourceError(err) && !mResourceManagerProxy->reclaimResource(resources)) { break; } // MediaCodec now set state to UNINITIALIZED upon any fatal error. // To maintain backward-compatibility, do a reset() to put codec // back into INITIALIZED state. // But don't reset if the err is INVALID_OPERATION, which means // the configure failure is due to wrong state. ALOGE("configure failed with err 0x%08x, resetting...", err); status_t err2 = reset(); if (err2 != OK) { ALOGE("retrying configure: failed to reset codec (%08x)", err2); break; } if (callback != nullptr) { err2 = setCallback(callback); if (err2 != OK) { ALOGE("retrying configure: failed to set callback (%08x)", err2); break; } } } if (!isResourceError(err)) { break; } } return err; } status_t MediaCodec::getRequiredResources(std::vector& resources) { resources.clear(); // Make sure codec availability feature is on. if (!android::media::codec::codec_availability()) { return ERROR_UNSUPPORTED; } // Make sure that the codec was configured already. if (mState != CONFIGURED && mState != STARTING && mState != STARTED && mState != FLUSHING && mState != FLUSHED) { ALOGE("Codec wasn't configured yet!"); return INVALID_OPERATION; } if (!mRequiredResourceInfo.empty()) { resources = mRequiredResourceInfo; return OK; } // TODO: For now this is just an empty function. // The actual implementation should use component interface // (for example, through mCodec->getRequiredDeviceResources) to query the // the required resources for this configuration, and fill in resources with the same. return ERROR_UNSUPPORTED; } // Media Format Shaping support // static android::mediaformatshaper::FormatShaperOps_t *sShaperOps = NULL; static bool sIsHandheld = true; static bool connectFormatShaper() { static std::once_flag sCheckOnce; ALOGV("connectFormatShaper..."); std::call_once(sCheckOnce, [&](){ void *libHandle = NULL; nsecs_t loading_started = systemTime(SYSTEM_TIME_MONOTONIC); // prefer any copy in the mainline module // android_namespace_t *mediaNs = android_get_exported_namespace("com_android_media"); AString libraryName = "libmediaformatshaper.so"; if (mediaNs != NULL) { static const android_dlextinfo dlextinfo = { .flags = ANDROID_DLEXT_USE_NAMESPACE, .library_namespace = mediaNs, }; AString libraryMainline = "/apex/com.android.media/"; #if __LP64__ libraryMainline.append("lib64/"); #else libraryMainline.append("lib/"); #endif libraryMainline.append(libraryName); libHandle = android_dlopen_ext(libraryMainline.c_str(), RTLD_NOW|RTLD_NODELETE, &dlextinfo); if (libHandle != NULL) { sShaperOps = (android::mediaformatshaper::FormatShaperOps_t*) dlsym(libHandle, "shaper_ops"); } else { ALOGW("connectFormatShaper: unable to load mainline formatshaper %s", libraryMainline.c_str()); } } else { ALOGV("connectFormatShaper: couldn't find media namespace."); } // fall back to the system partition, if present. // if (sShaperOps == NULL) { libHandle = dlopen(libraryName.c_str(), RTLD_NOW|RTLD_NODELETE); if (libHandle != NULL) { sShaperOps = (android::mediaformatshaper::FormatShaperOps_t*) dlsym(libHandle, "shaper_ops"); } else { ALOGW("connectFormatShaper: unable to load formatshaper %s", libraryName.c_str()); } } if (sShaperOps != nullptr && sShaperOps->version != android::mediaformatshaper::SHAPER_VERSION_V1) { ALOGW("connectFormatShaper: unhandled version ShaperOps: %d, DISABLED", sShaperOps->version); sShaperOps = nullptr; } if (sShaperOps != nullptr) { ALOGV("connectFormatShaper: connected to library %s", libraryName.c_str()); } nsecs_t loading_finished = systemTime(SYSTEM_TIME_MONOTONIC); ALOGV("connectFormatShaper: loaded libraries: %" PRId64 " us", (loading_finished - loading_started)/1000); // we also want to know whether this is a handheld device // start with assumption that the device is handheld. sIsHandheld = true; sp serviceMgr = defaultServiceManager(); sp packageMgr; if (serviceMgr.get() != nullptr) { sp binder = serviceMgr->waitForService(String16("package_native")); packageMgr = interface_cast(binder); } // if we didn't get serviceMgr, we'll leave packageMgr as default null if (packageMgr != nullptr) { // MUST have these static const String16 featuresNeeded[] = { String16("android.hardware.touchscreen") }; // these must be present to be a handheld for (::android::String16 required : featuresNeeded) { bool hasFeature = false; binder::Status status = packageMgr->hasSystemFeature(required, 0, &hasFeature); if (!status.isOk()) { ALOGE("%s: hasSystemFeature failed: %s", __func__, status.exceptionMessage().c_str()); continue; } ALOGV("feature %s says %d", String8(required).c_str(), hasFeature); if (!hasFeature) { ALOGV("... which means we are not handheld"); sIsHandheld = false; break; } } // MUST NOT have these static const String16 featuresDisallowed[] = { String16("android.hardware.type.automotive"), String16("android.hardware.type.television"), String16("android.hardware.type.watch") }; // any of these present -- we aren't a handheld for (::android::String16 forbidden : featuresDisallowed) { bool hasFeature = false; binder::Status status = packageMgr->hasSystemFeature(forbidden, 0, &hasFeature); if (!status.isOk()) { ALOGE("%s: hasSystemFeature failed: %s", __func__, status.exceptionMessage().c_str()); continue; } ALOGV("feature %s says %d", String8(forbidden).c_str(), hasFeature); if (hasFeature) { ALOGV("... which means we are not handheld"); sIsHandheld = false; break; } } } }); return true; } #if 0 // a construct to force the above dlopen() to run very early. // goal: so the dlopen() doesn't happen on critical path of latency sensitive apps // failure of this means that cold start of those apps is slower by the time to dlopen() // TODO(b/183454066): tradeoffs between memory of early loading vs latency of late loading // static bool forceEarlyLoadingShaper = connectFormatShaper(); #endif // parse the codec's properties: mapping, whether it meets min quality, etc // and pass them into the video quality code // static void loadCodecProperties(mediaformatshaper::shaperHandle_t shaperHandle, sp codecInfo, AString mediaType) { sp capabilities = codecInfo->getCapabilitiesFor(mediaType.c_str()); if (capabilities == nullptr) { ALOGI("no capabilities as part of the codec?"); } else { const sp &details = capabilities->getDetails(); AString mapTarget; int count = details->countEntries(); for(int ix = 0; ix < count; ix++) { AMessage::Type entryType; const char *mapSrc = details->getEntryNameAt(ix, &entryType); // XXX: re-use ix from getEntryAt() to avoid additional findXXX() invocation // static const char *featurePrefix = "feature-"; static const int featurePrefixLen = strlen(featurePrefix); static const char *tuningPrefix = "tuning-"; static const int tuningPrefixLen = strlen(tuningPrefix); static const char *mappingPrefix = "mapping-"; static const int mappingPrefixLen = strlen(mappingPrefix); if (mapSrc == NULL) { continue; } else if (!strncmp(mapSrc, featurePrefix, featurePrefixLen)) { int32_t intValue; if (details->findInt32(mapSrc, &intValue)) { ALOGV("-- feature '%s' -> %d", mapSrc, intValue); (void)(sShaperOps->setFeature)(shaperHandle, &mapSrc[featurePrefixLen], intValue); } continue; } else if (!strncmp(mapSrc, tuningPrefix, tuningPrefixLen)) { AString value; if (details->findString(mapSrc, &value)) { ALOGV("-- tuning '%s' -> '%s'", mapSrc, value.c_str()); (void)(sShaperOps->setTuning)(shaperHandle, &mapSrc[tuningPrefixLen], value.c_str()); } continue; } else if (!strncmp(mapSrc, mappingPrefix, mappingPrefixLen)) { AString target; if (details->findString(mapSrc, &target)) { ALOGV("-- mapping %s: map %s to %s", mapSrc, &mapSrc[mappingPrefixLen], target.c_str()); // key is really "kind-key" // separate that, so setMap() sees the triple kind, key, value const char *kind = &mapSrc[mappingPrefixLen]; const char *sep = strchr(kind, '-'); const char *key = sep+1; if (sep != NULL) { std::string xkind = std::string(kind, sep-kind); (void)(sShaperOps->setMap)(shaperHandle, xkind.c_str(), key, target.c_str()); } } } } } // we also carry in the codec description whether we are on a handheld device. // this info is eventually used by both the Codec and the C2 machinery to inform // the underlying codec whether to do any shaping. // if (sIsHandheld) { // set if we are indeed a handheld device (or in future 'any eligible device' // missing on devices that aren't eligible for minimum quality enforcement. (void)(sShaperOps->setFeature)(shaperHandle, "_vq_eligible.device", 1); // strictly speaking, it's a tuning, but those are strings and feature stores int (void)(sShaperOps->setFeature)(shaperHandle, "_quality.target", 1 /* S_HANDHELD */); } } status_t MediaCodec::setupFormatShaper(AString mediaType) { ALOGV("setupFormatShaper: initializing shaper data for codec %s mediaType %s", mComponentName.c_str(), mediaType.c_str()); nsecs_t mapping_started = systemTime(SYSTEM_TIME_MONOTONIC); // someone might have beaten us to it. mediaformatshaper::shaperHandle_t shaperHandle; shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str()); if (shaperHandle != nullptr) { ALOGV("shaperhandle %p -- no initialization needed", shaperHandle); return OK; } // we get to build & register one shaperHandle = sShaperOps->createShaper(mComponentName.c_str(), mediaType.c_str()); if (shaperHandle == nullptr) { ALOGW("unable to create a shaper for cocodec %s mediaType %s", mComponentName.c_str(), mediaType.c_str()); return OK; } (void) loadCodecProperties(shaperHandle, mCodecInfo, mediaType); shaperHandle = sShaperOps->registerShaper(shaperHandle, mComponentName.c_str(), mediaType.c_str()); nsecs_t mapping_finished = systemTime(SYSTEM_TIME_MONOTONIC); ALOGV("setupFormatShaper: populated shaper node for codec %s: %" PRId64 " us", mComponentName.c_str(), (mapping_finished - mapping_started)/1000); return OK; } // Format Shaping // Mapping and Manipulation of encoding parameters // // All of these decisions are pushed into the shaper instead of here within MediaCodec. // this includes decisions based on whether the codec implements minimum quality bars // itself or needs to be shaped outside of the codec. // This keeps all those decisions in one place. // It also means that we push some extra decision information (is this a handheld device // or one that is otherwise eligible for minimum quality manipulation, which generational // quality target is in force, etc). This allows those values to be cached in the // per-codec structures that are done 1 time within a process instead of for each // codec instantiation. // status_t MediaCodec::shapeMediaFormat( const sp &format, uint32_t flags, mediametrics_handle_t metricsHandle) { ALOGV("shapeMediaFormat entry"); if (!(flags & CONFIGURE_FLAG_ENCODE)) { ALOGW("shapeMediaFormat: not encoder"); return OK; } if (mCodecInfo == NULL) { ALOGW("shapeMediaFormat: no codecinfo"); return OK; } AString mediaType; if (!format->findString("mime", &mediaType)) { ALOGW("shapeMediaFormat: no mediaType information"); return OK; } // make sure we have the function entry points for the shaper library // connectFormatShaper(); if (sShaperOps == nullptr) { ALOGW("shapeMediaFormat: no MediaFormatShaper hooks available"); return OK; } // find the shaper information for this codec+mediaType pair // mediaformatshaper::shaperHandle_t shaperHandle; shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str()); if (shaperHandle == nullptr) { setupFormatShaper(mediaType); shaperHandle = sShaperOps->findShaper(mComponentName.c_str(), mediaType.c_str()); } if (shaperHandle == nullptr) { ALOGW("shapeMediaFormat: no handler for codec %s mediatype %s", mComponentName.c_str(), mediaType.c_str()); return OK; } // run the shaper // ALOGV("Shaping input: %s", format->debugString(0).c_str()); sp updatedFormat = format->dup(); AMediaFormat *updatedNdkFormat = AMediaFormat_fromMsg(&updatedFormat); int result = (*sShaperOps->shapeFormat)(shaperHandle, updatedNdkFormat, flags); if (result == 0) { AMediaFormat_getFormat(updatedNdkFormat, &updatedFormat); sp deltas = updatedFormat->changesFrom(format, false /* deep */); size_t changeCount = deltas->countEntries(); ALOGD("shapeMediaFormat: deltas(%zu): %s", changeCount, deltas->debugString(2).c_str()); if (metricsHandle != 0) { mediametrics_setInt32(metricsHandle, kCodecShapingEnhanced, changeCount); } if (changeCount > 0) { if (metricsHandle != 0) { // save some old properties before we fold in the new ones int32_t bitrate; if (format->findInt32(KEY_BIT_RATE, &bitrate)) { mediametrics_setInt32(metricsHandle, kCodecOriginalBitrate, bitrate); } int32_t qpIMin = -1; if (format->findInt32("original-video-qp-i-min", &qpIMin)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPIMin, qpIMin); } int32_t qpIMax = -1; if (format->findInt32("original-video-qp-i-max", &qpIMax)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPIMax, qpIMax); } int32_t qpPMin = -1; if (format->findInt32("original-video-qp-p-min", &qpPMin)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPPMin, qpPMin); } int32_t qpPMax = -1; if (format->findInt32("original-video-qp-p-max", &qpPMax)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPPMax, qpPMax); } int32_t qpBMin = -1; if (format->findInt32("original-video-qp-b-min", &qpBMin)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPBMin, qpBMin); } int32_t qpBMax = -1; if (format->findInt32("original-video-qp-b-max", &qpBMax)) { mediametrics_setInt32(metricsHandle, kCodecOriginalVideoQPBMax, qpBMax); } } // NB: for any field in both format and deltas, the deltas copy wins format->extend(deltas); } } AMediaFormat_delete(updatedNdkFormat); return OK; } static void mapFormat(AString componentName, const sp &format, const char *kind, bool reverse) { AString mediaType; if (!format->findString("mime", &mediaType)) { ALOGV("mapFormat: no mediaType information"); return; } ALOGV("mapFormat: codec %s mediatype %s kind %s reverse %d", componentName.c_str(), mediaType.c_str(), kind ? kind : "", reverse); // make sure we have the function entry points for the shaper library // #if 0 // let's play the faster "only do mapping if we've already loaded the library connectFormatShaper(); #endif if (sShaperOps == nullptr) { ALOGV("mapFormat: no MediaFormatShaper hooks available"); return; } // find the shaper information for this codec+mediaType pair // mediaformatshaper::shaperHandle_t shaperHandle; shaperHandle = sShaperOps->findShaper(componentName.c_str(), mediaType.c_str()); if (shaperHandle == nullptr) { ALOGV("mapFormat: no shaper handle"); return; } const char **mappings; if (reverse) mappings = sShaperOps->getReverseMappings(shaperHandle, kind); else mappings = sShaperOps->getMappings(shaperHandle, kind); if (mappings == nullptr) { ALOGV("no mappings returned"); return; } ALOGV("Pre-mapping: %s", format->debugString(2).c_str()); // do the mapping // int entries = format->countEntries(); for (int i = 0; ; i += 2) { if (mappings[i] == nullptr) { break; } size_t ix = format->findEntryByName(mappings[i]); if (ix < entries) { ALOGV("map '%s' to '%s'", mappings[i], mappings[i+1]); status_t status = format->setEntryNameAt(ix, mappings[i+1]); if (status != OK) { ALOGW("Unable to map from '%s' to '%s': status %d", mappings[i], mappings[i+1], status); } } } ALOGV("Post-mapping: %s", format->debugString(2).c_str()); // reclaim the mapping memory for (int i = 0; ; i += 2) { if (mappings[i] == nullptr) { break; } free((void*)mappings[i]); free((void*)mappings[i + 1]); } free(mappings); mappings = nullptr; } // // end of Format Shaping hooks within MediaCodec // status_t MediaCodec::releaseCrypto() { ALOGV("releaseCrypto"); sp msg = new AMessage(kWhatDrmReleaseCrypto, this); sp response; status_t status = msg->postAndAwaitResponse(&response); if (status == OK && response != NULL) { CHECK(response->findInt32("status", &status)); ALOGV("releaseCrypto ret: %d ", status); } else { ALOGE("releaseCrypto err: %d", status); } return status; } void MediaCodec::onReleaseCrypto(const sp& msg) { status_t status = INVALID_OPERATION; if (mCrypto != NULL) { ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount()); mBufferChannel->setCrypto(NULL); // TODO change to ALOGV ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount()); mCrypto.clear(); status = OK; } else { ALOGW("onReleaseCrypto: No mCrypto. err: %d", status); } sp response = new AMessage; response->setInt32("status", status); sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); } status_t MediaCodec::setInputSurface( const sp &surface) { sp msg = new AMessage(kWhatSetInputSurface, this); msg->setObject("input-surface", surface.get()); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::detachOutputSurface() { sp msg = new AMessage(kWhatDetachSurface, this); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::setSurface(const sp &surface) { sp msg = new AMessage(kWhatSetSurface, this); msg->setObject("surface", surface); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::createInputSurface( sp* bufferProducer) { sp msg = new AMessage(kWhatCreateInputSurface, this); sp response; status_t err = PostAndAwaitResponse(msg, &response); if (err == NO_ERROR) { // unwrap the sp sp obj; bool found = response->findObject("input-surface", &obj); CHECK(found); sp wrapper( static_cast(obj.get())); *bufferProducer = wrapper->getBufferProducer(); } else { ALOGW("createInputSurface failed, err=%d", err); } return err; } uint64_t MediaCodec::getGraphicBufferSize() { if (mDomain != DOMAIN_VIDEO && mDomain != DOMAIN_IMAGE) { return 0; } uint64_t size = 0; size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); for (size_t i = 0; i < portNum; ++i) { // TODO: this is just an estimation, we should get the real buffer size from ACodec. size += mPortBuffers[i].size() * mWidth * mHeight * 3 / 2; } return size; } status_t MediaCodec::start() { sp msg = new AMessage(kWhatStart, this); sp callback; status_t err; std::vector resources; resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, toMediaResourceSubType(mIsHardware, mDomain))); if (mDomain == DOMAIN_VIDEO || mDomain == DOMAIN_IMAGE) { // Don't know the buffer size at this point, but it's fine to use 1 because // the reclaimResource call doesn't consider the requester's buffer size for now. resources.push_back(MediaResource::GraphicMemoryResource(1)); } for (int i = 0; i <= kMaxRetry; ++i) { if (i > 0) { // Don't try to reclaim resource for the first time. if (!mResourceManagerProxy->reclaimResource(resources)) { break; } // Recover codec from previous error before retry start. err = reset(); if (err != OK) { ALOGE("retrying start: failed to reset codec"); break; } if (callback != nullptr) { err = setCallback(callback); if (err != OK) { ALOGE("retrying start: failed to set callback"); break; } ALOGD("succeed to set callback for reclaim"); } sp response; err = PostAndAwaitResponse(mConfigureMsg, &response); if (err != OK) { ALOGE("retrying start: failed to configure codec"); break; } } // Keep callback message after the first iteration if necessary. if (i == 0 && mCallback != nullptr && mFlags & kFlagIsAsync) { callback = mCallback; ALOGD("keep callback message for reclaim"); } sp response; err = PostAndAwaitResponse(msg, &response); if (!isResourceError(err)) { break; } } return err; } status_t MediaCodec::stop() { sp msg = new AMessage(kWhatStop, this); sp response; return PostAndAwaitResponse(msg, &response); } bool MediaCodec::hasPendingBuffer(int portIndex) { return std::any_of( mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(), [](const BufferInfo &info) { return info.mOwnedByClient; }); } bool MediaCodec::hasPendingBuffer() { return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput); } status_t MediaCodec::reclaim(bool force) { ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str()); sp msg = new AMessage(kWhatRelease, this); msg->setInt32("reclaimed", 1); msg->setInt32("force", force ? 1 : 0); sp response; status_t ret = PostAndAwaitResponse(msg, &response); if (ret == -ENOENT) { ALOGD("MediaCodec looper is gone, skip reclaim"); ret = OK; } return ret; } status_t MediaCodec::release() { sp msg = new AMessage(kWhatRelease, this); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::releaseAsync(const sp ¬ify) { sp msg = new AMessage(kWhatRelease, this); msg->setMessage("async", notify); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::reset() { /* When external-facing MediaCodec object is created, it is already initialized. Thus, reset is essentially release() followed by init(), plus clearing the state */ status_t err = release(); // unregister handlers if (mCodec != NULL) { if (mCodecLooper != NULL) { mCodecLooper->unregisterHandler(mCodec->id()); } else { mLooper->unregisterHandler(mCodec->id()); } mCodec = NULL; } mLooper->unregisterHandler(id()); mFlags = 0; // clear all flags mStickyError = OK; // reset state not reset by setState(UNINITIALIZED) mDequeueInputReplyID = 0; mDequeueOutputReplyID = 0; mDequeueInputTimeoutGeneration = 0; mDequeueOutputTimeoutGeneration = 0; mHaveInputSurface = false; if (err == OK) { err = init(mInitName); } return err; } status_t MediaCodec::queueInputBuffer( size_t index, size_t offset, size_t size, int64_t presentationTimeUs, uint32_t flags, AString *errorDetailMsg) { if (errorDetailMsg != NULL) { errorDetailMsg->clear(); } sp msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); msg->setSize("offset", offset); msg->setSize("size", size); msg->setInt64("timeUs", presentationTimeUs); msg->setInt32("flags", flags); msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::queueInputBuffers( size_t index, size_t offset, size_t size, const sp &infos, AString *errorDetailMsg) { sp msg = new AMessage(kWhatQueueInputBuffer, this); uint32_t bufferFlags = 0; uint32_t flagsinAllAU = BUFFER_FLAG_DECODE_ONLY | BUFFER_FLAG_CODECCONFIG; uint32_t andFlags = flagsinAllAU; if (infos == nullptr || infos->value.empty()) { ALOGE("ERROR: Large Audio frame with no BufferInfo"); return BAD_VALUE; } int infoIdx = 0; std::vector &accessUnitInfo = infos->value; int64_t minTimeUs = accessUnitInfo.front().mTimestamp; bool foundEndOfStream = false; for ( ; infoIdx < accessUnitInfo.size() && !foundEndOfStream; ++infoIdx) { bufferFlags |= accessUnitInfo[infoIdx].mFlags; andFlags &= accessUnitInfo[infoIdx].mFlags; if (bufferFlags & BUFFER_FLAG_END_OF_STREAM) { foundEndOfStream = true; } } bufferFlags = bufferFlags & (andFlags | (~flagsinAllAU)); if (infoIdx != accessUnitInfo.size()) { ALOGE("queueInputBuffers has incorrect access-units"); return -EINVAL; } msg->setSize("index", index); msg->setSize("offset", offset); msg->setSize("size", size); msg->setInt64("timeUs", minTimeUs); // Make this represent flags for the entire buffer // decodeOnly Flag is set only when all buffers are decodeOnly msg->setInt32("flags", bufferFlags); msg->setObject("accessUnitInfo", infos); msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::queueSecureInputBuffer( size_t index, size_t offset, const CryptoPlugin::SubSample *subSamples, size_t numSubSamples, const uint8_t key[16], const uint8_t iv[16], CryptoPlugin::Mode mode, const CryptoPlugin::Pattern &pattern, int64_t presentationTimeUs, uint32_t flags, AString *errorDetailMsg) { if (errorDetailMsg != NULL) { errorDetailMsg->clear(); } sp msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); msg->setSize("offset", offset); msg->setPointer("subSamples", (void *)subSamples); msg->setSize("numSubSamples", numSubSamples); msg->setPointer("key", (void *)key); msg->setPointer("iv", (void *)iv); msg->setInt32("mode", mode); msg->setInt32("encryptBlocks", pattern.mEncryptBlocks); msg->setInt32("skipBlocks", pattern.mSkipBlocks); msg->setInt64("timeUs", presentationTimeUs); msg->setInt32("flags", flags); msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; status_t err = PostAndAwaitResponse(msg, &response); return err; } status_t MediaCodec::queueSecureInputBuffers( size_t index, size_t offset, size_t size, const sp &auInfo, const sp &cryptoInfos, AString *errorDetailMsg) { if (errorDetailMsg != NULL) { errorDetailMsg->clear(); } sp msg = new AMessage(kWhatQueueInputBuffer, this); uint32_t bufferFlags = 0; uint32_t flagsinAllAU = BUFFER_FLAG_DECODE_ONLY | BUFFER_FLAG_CODECCONFIG; uint32_t andFlags = flagsinAllAU; if (auInfo == nullptr || auInfo->value.empty() || cryptoInfos == nullptr || cryptoInfos->value.empty()) { ALOGE("ERROR: Large Audio frame with no BufferInfo/CryptoInfo"); return BAD_VALUE; } int infoIdx = 0; std::vector &accessUnitInfo = auInfo->value; int64_t minTimeUs = accessUnitInfo.front().mTimestamp; bool foundEndOfStream = false; for ( ; infoIdx < accessUnitInfo.size() && !foundEndOfStream; ++infoIdx) { bufferFlags |= accessUnitInfo[infoIdx].mFlags; andFlags &= accessUnitInfo[infoIdx].mFlags; if (bufferFlags & BUFFER_FLAG_END_OF_STREAM) { foundEndOfStream = true; } } bufferFlags = bufferFlags & (andFlags | (~flagsinAllAU)); if (infoIdx != accessUnitInfo.size()) { ALOGE("queueInputBuffers has incorrect access-units"); return -EINVAL; } msg->setSize("index", index); msg->setSize("offset", offset); msg->setSize("ssize", size); msg->setInt64("timeUs", minTimeUs); msg->setInt32("flags", bufferFlags); msg->setObject("accessUnitInfo", auInfo); msg->setObject("cryptoInfos", cryptoInfos); msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; status_t err = PostAndAwaitResponse(msg, &response); return err; } status_t MediaCodec::queueBuffer( size_t index, const std::shared_ptr &buffer, const sp &bufferInfos, const sp &tunings, AString *errorDetailMsg) { if (errorDetailMsg != NULL) { errorDetailMsg->clear(); } if (bufferInfos == nullptr || bufferInfos->value.empty()) { return BAD_VALUE; } status_t err = OK; sp msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); sp>> obj{ new WrapperObject>{buffer}}; msg->setObject("c2buffer", obj); if (OK != (err = generateFlagsFromAccessUnitInfo(msg, bufferInfos))) { return err; } msg->setObject("accessUnitInfo", bufferInfos); if (tunings && tunings->countEntries() > 0) { msg->setMessage("tunings", tunings); } msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; err = PostAndAwaitResponse(msg, &response); return err; } status_t MediaCodec::queueEncryptedBuffer( size_t index, const sp &buffer, size_t offset, size_t size, const sp &bufferInfos, const sp &cryptoInfos, const sp &tunings, AString *errorDetailMsg) { if (errorDetailMsg != NULL) { errorDetailMsg->clear(); } if (bufferInfos == nullptr || bufferInfos->value.empty()) { return BAD_VALUE; } status_t err = OK; sp msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); sp>> memory{ new WrapperObject>{buffer}}; msg->setObject("memory", memory); msg->setSize("offset", offset); if (cryptoInfos != nullptr) { msg->setSize("ssize", size); msg->setObject("cryptoInfos", cryptoInfos); } else { msg->setSize("size", size); } msg->setObject("accessUnitInfo", bufferInfos); if (OK != (err = generateFlagsFromAccessUnitInfo(msg, bufferInfos))) { return err; } if (tunings && tunings->countEntries() > 0) { msg->setMessage("tunings", tunings); } msg->setPointer("errorDetailMsg", errorDetailMsg); sp response; err = PostAndAwaitResponse(msg, &response); return err; } status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { sp msg = new AMessage(kWhatDequeueInputBuffer, this); msg->setInt64("timeoutUs", timeoutUs); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findSize("index", index)); return OK; } status_t MediaCodec::dequeueOutputBuffer( size_t *index, size_t *offset, size_t *size, int64_t *presentationTimeUs, uint32_t *flags, int64_t timeoutUs) { sp msg = new AMessage(kWhatDequeueOutputBuffer, this); msg->setInt64("timeoutUs", timeoutUs); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findSize("index", index)); CHECK(response->findSize("offset", offset)); CHECK(response->findSize("size", size)); CHECK(response->findInt64("timeUs", presentationTimeUs)); CHECK(response->findInt32("flags", (int32_t *)flags)); return OK; } status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { sp msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); msg->setInt32("render", true); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { sp msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); msg->setInt32("render", true); msg->setInt64("timestampNs", timestampNs); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::releaseOutputBuffer(size_t index) { sp msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::signalEndOfInputStream() { sp msg = new AMessage(kWhatSignalEndOfInputStream, this); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::getOutputFormat(sp *format) const { sp msg = new AMessage(kWhatGetOutputFormat, this); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findMessage("format", format)); return OK; } status_t MediaCodec::getInputFormat(sp *format) const { sp msg = new AMessage(kWhatGetInputFormat, this); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findMessage("format", format)); return OK; } status_t MediaCodec::getName(AString *name) const { sp msg = new AMessage(kWhatGetName, this); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findString("name", name)); return OK; } status_t MediaCodec::getCodecInfo(sp *codecInfo) const { sp msg = new AMessage(kWhatGetCodecInfo, this); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } sp obj; CHECK(response->findObject("codecInfo", &obj)); *codecInfo = static_cast(obj.get()); return OK; } // this is the user-callable entry point status_t MediaCodec::getMetrics(mediametrics_handle_t &reply) { reply = 0; sp msg = new AMessage(kWhatGetMetrics, this); sp response; status_t err; if ((err = PostAndAwaitResponse(msg, &response)) != OK) { return err; } CHECK(response->findInt64("metrics", &reply)); return OK; } // runs on the looper thread (for mutex purposes) void MediaCodec::onGetMetrics(const sp& msg) { mediametrics_handle_t results = 0; sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mMetricsHandle != 0) { updateMediametrics(); results = mediametrics_dup(mMetricsHandle); updateEphemeralMediametrics(results); } else if (mLastMetricsHandle != 0) { // After error, mMetricsHandle is cleared, but we keep the last // metrics around so that it can be queried by getMetrics(). results = mediametrics_dup(mLastMetricsHandle); } else { results = mediametrics_dup(mMetricsHandle); } sp response = new AMessage; response->setInt64("metrics", results); response->postReply(replyID); } status_t MediaCodec::getInputBuffers(Vector > *buffers) const { sp msg = new AMessage(kWhatGetBuffers, this); msg->setInt32("portIndex", kPortIndexInput); msg->setPointer("buffers", buffers); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::getOutputBuffers(Vector > *buffers) const { sp msg = new AMessage(kWhatGetBuffers, this); msg->setInt32("portIndex", kPortIndexOutput); msg->setPointer("buffers", buffers); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::getOutputBuffer(size_t index, sp *buffer) { sp format; return getBufferAndFormat(kPortIndexOutput, index, buffer, &format); } status_t MediaCodec::getOutputFormat(size_t index, sp *format) { sp buffer; return getBufferAndFormat(kPortIndexOutput, index, &buffer, format); } status_t MediaCodec::getInputBuffer(size_t index, sp *buffer) { sp format; return getBufferAndFormat(kPortIndexInput, index, buffer, &format); } bool MediaCodec::isExecuting() const { return mState == STARTED || mState == FLUSHED; } status_t MediaCodec::getBufferAndFormat( size_t portIndex, size_t index, sp *buffer, sp *format) { // use mutex instead of a context switch if (mReleasedByResourceManager) { mErrorLog.log(LOG_TAG, "resource already released"); return DEAD_OBJECT; } if (buffer == NULL) { mErrorLog.log(LOG_TAG, "null buffer"); return INVALID_OPERATION; } if (format == NULL) { mErrorLog.log(LOG_TAG, "null format"); return INVALID_OPERATION; } buffer->clear(); format->clear(); if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Invalid to call %s; only valid in Executing states", apiStateString().c_str())); return INVALID_OPERATION; } // we do not want mPortBuffers to change during this section // we also don't want mOwnedByClient to change during this Mutex::Autolock al(mBufferLock); std::vector &buffers = mPortBuffers[portIndex]; if (index >= buffers.size()) { ALOGE("getBufferAndFormat - trying to get buffer with " "bad index (index=%zu buffer_size=%zu)", index, buffers.size()); mErrorLog.log(LOG_TAG, base::StringPrintf("Bad index (index=%zu)", index)); return INVALID_OPERATION; } const BufferInfo &info = buffers[index]; if (!info.mOwnedByClient) { ALOGE("getBufferAndFormat - invalid operation " "(the index %zu is not owned by client)", index); mErrorLog.log(LOG_TAG, base::StringPrintf("index %zu is not owned by client", index)); return INVALID_OPERATION; } *buffer = info.mData; *format = info.mData->format(); return OK; } status_t MediaCodec::flush() { sp msg = new AMessage(kWhatFlush, this); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::requestIDRFrame() { (new AMessage(kWhatRequestIDRFrame, this))->post(); return OK; } status_t MediaCodec::querySupportedVendorParameters(std::vector *names) { return mCodec->querySupportedParameters(names); } status_t MediaCodec::describeParameter(const std::string &name, CodecParameterDescriptor *desc) { return mCodec->describeParameter(name, desc); } status_t MediaCodec::subscribeToVendorParameters(const std::vector &names) { return mCodec->subscribeToParameters(names); } status_t MediaCodec::unsubscribeFromVendorParameters(const std::vector &names) { return mCodec->unsubscribeFromParameters(names); } void MediaCodec::requestActivityNotification(const sp ¬ify) { sp msg = new AMessage(kWhatRequestActivityNotification, this); msg->setMessage("notify", notify); msg->post(); } void MediaCodec::requestCpuBoostIfNeeded() { if (mCpuBoostRequested) { return; } int32_t colorFormat; if (mOutputFormat->contains("hdr-static-info") && mOutputFormat->findInt32("color-format", &colorFormat) // check format for OMX only, for C2 the format is always opaque since the // software rendering doesn't go through client && ((mSoftRenderer != NULL && colorFormat == OMX_COLOR_FormatYUV420Planar16) || mOwnerName.equalsIgnoreCase("codec2::software"))) { int32_t left, top, right, bottom, width, height; int64_t totalPixel = 0; if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { totalPixel = (right - left + 1) * (bottom - top + 1); } else if (mOutputFormat->findInt32("width", &width) && mOutputFormat->findInt32("height", &height)) { totalPixel = width * height; } if (totalPixel >= 1920 * 1080) { mResourceManagerProxy->addResource(MediaResource::CpuBoostResource()); mCpuBoostRequested = true; } } } BatteryChecker::BatteryChecker(const sp &msg, int64_t timeoutUs) : mTimeoutUs(timeoutUs) , mLastActivityTimeUs(-1ll) , mBatteryStatNotified(false) , mBatteryCheckerGeneration(0) , mIsExecuting(false) , mBatteryCheckerMsg(msg) {} void BatteryChecker::onCodecActivity(std::function batteryOnCb) { if (!isExecuting()) { // ignore if not executing return; } if (!mBatteryStatNotified) { batteryOnCb(); mBatteryStatNotified = true; sp msg = mBatteryCheckerMsg->dup(); msg->setInt32("generation", mBatteryCheckerGeneration); // post checker and clear last activity time msg->post(mTimeoutUs); mLastActivityTimeUs = -1ll; } else { // update last activity time mLastActivityTimeUs = ALooper::GetNowUs(); } } void BatteryChecker::onCheckBatteryTimer( const sp &msg, std::function batteryOffCb) { // ignore if this checker already expired because the client resource was removed int32_t generation; if (!msg->findInt32("generation", &generation) || generation != mBatteryCheckerGeneration) { return; } if (mLastActivityTimeUs < 0ll) { // timed out inactive, do not repost checker batteryOffCb(); mBatteryStatNotified = false; } else { // repost checker and clear last activity time msg->post(mTimeoutUs + mLastActivityTimeUs - ALooper::GetNowUs()); mLastActivityTimeUs = -1ll; } } void BatteryChecker::onClientRemoved() { mBatteryStatNotified = false; mBatteryCheckerGeneration++; } //////////////////////////////////////////////////////////////////////////////// void MediaCodec::cancelPendingDequeueOperations() { if (mFlags & kFlagDequeueInputPending) { mErrorLog.log(LOG_TAG, "Pending dequeue input buffer request cancelled"); PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION); ++mDequeueInputTimeoutGeneration; mDequeueInputReplyID = 0; mFlags &= ~kFlagDequeueInputPending; } if (mFlags & kFlagDequeueOutputPending) { mErrorLog.log(LOG_TAG, "Pending dequeue output buffer request cancelled"); PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION); ++mDequeueOutputTimeoutGeneration; mDequeueOutputReplyID = 0; mFlags &= ~kFlagDequeueOutputPending; } } bool MediaCodec::handleDequeueInputBuffer(const sp &replyID, bool newRequest) { if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Invalid to call %s; only valid in executing state", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); } else if (mFlags & kFlagIsAsync) { mErrorLog.log(LOG_TAG, "Invalid to call in async mode"); PostReplyWithError(replyID, INVALID_OPERATION); } else if (newRequest && (mFlags & kFlagDequeueInputPending)) { mErrorLog.log(LOG_TAG, "Invalid to call while another dequeue input request is pending"); PostReplyWithError(replyID, INVALID_OPERATION); return true; } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); return true; } ssize_t index = dequeuePortBuffer(kPortIndexInput); if (index < 0) { CHECK_EQ(index, -EAGAIN); return false; } sp response = new AMessage; response->setSize("index", index); response->postReply(replyID); return true; } // always called from the looper thread MediaCodec::DequeueOutputResult MediaCodec::handleDequeueOutputBuffer( const sp &replyID, bool newRequest) { if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Invalid to call %s; only valid in executing state", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); } else if (mFlags & kFlagIsAsync) { mErrorLog.log(LOG_TAG, "Invalid to call in async mode"); PostReplyWithError(replyID, INVALID_OPERATION); } else if (newRequest && (mFlags & kFlagDequeueOutputPending)) { mErrorLog.log(LOG_TAG, "Invalid to call while another dequeue output request is pending"); PostReplyWithError(replyID, INVALID_OPERATION); } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); } else if (mFlags & kFlagOutputBuffersChanged) { PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED); mFlags &= ~kFlagOutputBuffersChanged; } else { sp response = new AMessage; BufferInfo *info = peekNextPortBuffer(kPortIndexOutput); if (!info) { return DequeueOutputResult::kNoBuffer; } // In synchronous mode, output format change should be handled // at dequeue to put the event at the correct order. const sp &buffer = info->mData; handleOutputFormatChangeIfNeeded(buffer); if (mFlags & kFlagOutputFormatChanged) { PostReplyWithError(replyID, INFO_FORMAT_CHANGED); mFlags &= ~kFlagOutputFormatChanged; return DequeueOutputResult::kRepliedWithError; } ssize_t index = dequeuePortBuffer(kPortIndexOutput); if (discardDecodeOnlyOutputBuffer(index)) { return DequeueOutputResult::kDiscardedBuffer; } response->setSize("index", index); response->setSize("offset", buffer->offset()); response->setSize("size", buffer->size()); int64_t timeUs; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); response->setInt64("timeUs", timeUs); int32_t flags; CHECK(buffer->meta()->findInt32("flags", &flags)); response->setInt32("flags", flags); // NOTE: we must account the stats for an output buffer only after we // already handled a potential output format change that could have // started a new subsession. statsBufferReceived(timeUs, buffer); response->postReply(replyID); return DequeueOutputResult::kSuccess; } return DequeueOutputResult::kRepliedWithError; } inline void MediaCodec::initClientConfigParcel(ClientConfigParcel& clientConfig) { clientConfig.codecType = toMediaResourceSubType(mIsHardware, mDomain); clientConfig.isEncoder = mFlags & kFlagIsEncoder; clientConfig.width = mWidth; clientConfig.height = mHeight; clientConfig.timeStamp = systemTime(SYSTEM_TIME_MONOTONIC) / 1000LL; clientConfig.id = mCodecId; } void MediaCodec::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatCodecNotify: { int32_t what; CHECK(msg->findInt32("what", &what)); AString codecErrorState; switch (what) { case kWhatError: case kWhatCryptoError: { int32_t err, actionCode; CHECK(msg->findInt32("err", &err)); CHECK(msg->findInt32("actionCode", &actionCode)); ALOGE("Codec reported err %#x/%s, actionCode %d, while in state %d/%s", err, StrMediaError(err).c_str(), actionCode, mState, stateString(mState).c_str()); if (err == DEAD_OBJECT) { mFlags |= kFlagSawMediaServerDie; mFlags &= ~kFlagIsComponentAllocated; } bool sendErrorResponse = true; std::string origin; if (what == kWhatCryptoError) { origin = "kWhatCryptoError:"; } else { origin = "kWhatError:"; //TODO: add a new error state } codecErrorState = kCodecErrorState; origin += stateString(mState); if (mCryptoAsync) { //TODO: do some book keeping on the buffers mCryptoAsync->stop(); } switch (mState) { case INITIALIZING: { // Resource error during INITIALIZING state needs to be logged // through metrics, to be able to track such occurrences. if (isResourceError(err)) { mediametrics_setInt32(mMetricsHandle, kCodecError, err); mediametrics_setCString(mMetricsHandle, kCodecErrorState, stateString(mState).c_str()); flushMediametrics(); initMediametrics(); } setState(UNINITIALIZED); break; } case CONFIGURING: { if (actionCode == ACTION_CODE_FATAL) { mediametrics_setInt32(mMetricsHandle, kCodecError, err); mediametrics_setCString(mMetricsHandle, kCodecErrorState, stateString(mState).c_str()); flushMediametrics(); initMediametrics(); } setState(actionCode == ACTION_CODE_FATAL ? UNINITIALIZED : INITIALIZED); break; } case STARTING: { if (actionCode == ACTION_CODE_FATAL) { mediametrics_setInt32(mMetricsHandle, kCodecError, err); mediametrics_setCString(mMetricsHandle, kCodecErrorState, stateString(mState).c_str()); flushMediametrics(); initMediametrics(); } setState(actionCode == ACTION_CODE_FATAL ? UNINITIALIZED : CONFIGURED); break; } case RELEASING: { // Ignore the error, assuming we'll still get // the shutdown complete notification. If we // don't, we'll timeout and force release. sendErrorResponse = false; FALLTHROUGH_INTENDED; } case STOPPING: { if (mFlags & kFlagSawMediaServerDie) { if (mState == RELEASING && !mReplyID) { ALOGD("Releasing asynchronously, so nothing to reply here."); } // MediaServer died, there definitely won't // be a shutdown complete notification after // all. // note that we may be directly going from // STOPPING->UNINITIALIZED, instead of the // usual STOPPING->INITIALIZED state. setState(UNINITIALIZED); if (mState == RELEASING) { mComponentName.clear(); } if (mReplyID) { postPendingRepliesAndDeferredMessages(origin + ":dead"); } else { ALOGD("no pending replies: %s:dead following %s", origin.c_str(), mLastReplyOrigin.c_str()); } sendErrorResponse = false; } else if (!mReplyID) { sendErrorResponse = false; } break; } case FLUSHING: { if (actionCode == ACTION_CODE_FATAL) { mediametrics_setInt32(mMetricsHandle, kCodecError, err); mediametrics_setCString(mMetricsHandle, kCodecErrorState, stateString(mState).c_str()); flushMediametrics(); initMediametrics(); setState(UNINITIALIZED); } else { setState((mFlags & kFlagIsAsync) ? FLUSHED : STARTED); } break; } case FLUSHED: case STARTED: { sendErrorResponse = (mReplyID != nullptr); setStickyError(err); postActivityNotificationIfPossible(); cancelPendingDequeueOperations(); if (mFlags & kFlagIsAsync) { if (what == kWhatError) { onError(err, actionCode); } else if (what == kWhatCryptoError) { onCryptoError(msg); } } switch (actionCode) { case ACTION_CODE_TRANSIENT: break; case ACTION_CODE_RECOVERABLE: setState(INITIALIZED); break; default: mediametrics_setInt32(mMetricsHandle, kCodecError, err); mediametrics_setCString(mMetricsHandle, kCodecErrorState, stateString(mState).c_str()); flushMediametrics(); initMediametrics(); setState(UNINITIALIZED); break; } break; } default: { sendErrorResponse = (mReplyID != nullptr); setStickyError(err); postActivityNotificationIfPossible(); // actionCode in an uninitialized state is always fatal. if (mState == UNINITIALIZED) { actionCode = ACTION_CODE_FATAL; } if (mFlags & kFlagIsAsync) { if (what == kWhatError) { onError(err, actionCode); } else if (what == kWhatCryptoError) { onCryptoError(msg); } } switch (actionCode) { case ACTION_CODE_TRANSIENT: break; case ACTION_CODE_RECOVERABLE: setState(INITIALIZED); break; default: setState(UNINITIALIZED); break; } break; } } if (sendErrorResponse) { // TRICKY: replicate PostReplyWithError logic for // err code override int32_t finalErr = err; if (mReleasedByResourceManager) { // override the err code if MediaCodec has been // released by ResourceManager. finalErr = DEAD_OBJECT; } postPendingRepliesAndDeferredMessages(origin, finalErr); } break; } case kWhatComponentAllocated: { if (mState == RELEASING || mState == UNINITIALIZED) { // In case a kWhatError or kWhatRelease message came in and replied, // we log a warning and ignore. ALOGW("allocate interrupted by error or release, current state %d/%s", mState, stateString(mState).c_str()); break; } CHECK_EQ(mState, INITIALIZING); setState(INITIALIZED); mFlags |= kFlagIsComponentAllocated; CHECK(msg->findString("componentName", &mComponentName)); if (mComponentName.c_str()) { mIsHardware = !MediaCodecList::isSoftwareCodec(mComponentName); mediametrics_setCString(mMetricsHandle, kCodecCodec, mComponentName.c_str()); // Update the codec name. mResourceManagerProxy->setCodecName(mComponentName.c_str()); } const char *owner = mCodecInfo ? mCodecInfo->getOwnerName() : ""; if (mComponentName.startsWith("OMX.google.") && strncmp(owner, "default", 8) == 0) { mFlags |= kFlagUsesSoftwareRenderer; } else { mFlags &= ~kFlagUsesSoftwareRenderer; } mOwnerName = owner; if (mComponentName.endsWith(".secure")) { mFlags |= kFlagIsSecure; mediametrics_setInt32(mMetricsHandle, kCodecSecure, 1); } else { mFlags &= ~kFlagIsSecure; mediametrics_setInt32(mMetricsHandle, kCodecSecure, 0); } mediametrics_setInt32(mMetricsHandle, kCodecHardware, MediaCodecList::isSoftwareCodec(mComponentName) ? 0 : 1); mResourceManagerProxy->addResource(MediaResource::CodecResource( mFlags & kFlagIsSecure, toMediaResourceSubType(mIsHardware, mDomain))); postPendingRepliesAndDeferredMessages("kWhatComponentAllocated"); break; } case kWhatComponentConfigured: { if (mState == RELEASING || mState == UNINITIALIZED || mState == INITIALIZED) { // In case a kWhatError or kWhatRelease message came in and replied, // we log a warning and ignore. ALOGW("configure interrupted by error or release, current state %d/%s", mState, stateString(mState).c_str()); break; } CHECK_EQ(mState, CONFIGURING); // reset input surface flag mHaveInputSurface = false; CHECK(msg->findMessage("input-format", &mInputFormat)); CHECK(msg->findMessage("output-format", &mOutputFormat)); // limit to confirming the opt-in behavior to minimize any behavioral change if (mSurface != nullptr && !mAllowFrameDroppingBySurface) { // signal frame dropping mode in the input format as this may also be // meaningful and confusing for an encoder in a transcoder scenario mInputFormat->setInt32(KEY_ALLOW_FRAME_DROP, mAllowFrameDroppingBySurface); } sp interestingFormat = (mFlags & kFlagIsEncoder) ? mOutputFormat : mInputFormat; ALOGV("[%s] configured as input format: %s, output format: %s", mComponentName.c_str(), mInputFormat->debugString(4).c_str(), mOutputFormat->debugString(4).c_str()); int32_t usingSwRenderer; if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer) && usingSwRenderer) { mFlags |= kFlagUsesSoftwareRenderer; } setState(CONFIGURED); postPendingRepliesAndDeferredMessages("kWhatComponentConfigured"); // augment our media metrics info, now that we know more things // such as what the codec extracted from any CSD passed in. if (mMetricsHandle != 0) { sp format; if (mConfigureMsg != NULL && mConfigureMsg->findMessage("format", &format)) { // format includes: mime AString mime; if (format->findString("mime", &mime)) { mediametrics_setCString(mMetricsHandle, kCodecMime, mime.c_str()); } } // perhaps video only? int32_t profile = 0; if (interestingFormat->findInt32("profile", &profile)) { mediametrics_setInt32(mMetricsHandle, kCodecProfile, profile); } int32_t level = 0; if (interestingFormat->findInt32("level", &level)) { mediametrics_setInt32(mMetricsHandle, kCodecLevel, level); } sp uncompressedFormat = (mFlags & kFlagIsEncoder) ? mInputFormat : mOutputFormat; int32_t componentColorFormat = -1; if (uncompressedFormat->findInt32("android._color-format", &componentColorFormat)) { mediametrics_setInt32(mMetricsHandle, kCodecComponentColorFormat, componentColorFormat); } updateHdrMetrics(true /* isConfig */); int32_t codecMaxInputSize = -1; if (mInputFormat->findInt32(KEY_MAX_INPUT_SIZE, &codecMaxInputSize)) { mApiUsageMetrics.inputBufferSize.codecMax = codecMaxInputSize; } // bitrate and bitrate mode, encoder only if (mFlags & kFlagIsEncoder) { // encoder specific values int32_t bitrate_mode = -1; if (mOutputFormat->findInt32(KEY_BITRATE_MODE, &bitrate_mode)) { mediametrics_setCString(mMetricsHandle, kCodecBitrateMode, asString_BitrateMode(bitrate_mode)); } int32_t bitrate = -1; if (mOutputFormat->findInt32(KEY_BIT_RATE, &bitrate)) { mediametrics_setInt32(mMetricsHandle, kCodecBitrate, bitrate); } } else { // decoder specific values } } break; } case kWhatInputSurfaceCreated: { if (mState != CONFIGURED) { // state transitioned unexpectedly; we should have replied already. ALOGD("received kWhatInputSurfaceCreated message in state %s", stateString(mState).c_str()); break; } // response to initiateCreateInputSurface() status_t err = NO_ERROR; sp response = new AMessage; if (!msg->findInt32("err", &err)) { sp obj; msg->findObject("input-surface", &obj); CHECK(msg->findMessage("input-format", &mInputFormat)); CHECK(msg->findMessage("output-format", &mOutputFormat)); ALOGV("[%s] input surface created as input format: %s, output format: %s", mComponentName.c_str(), mInputFormat->debugString(4).c_str(), mOutputFormat->debugString(4).c_str()); CHECK(obj != NULL); response->setObject("input-surface", obj); mHaveInputSurface = true; } else { response->setInt32("err", err); } postPendingRepliesAndDeferredMessages("kWhatInputSurfaceCreated", response); break; } case kWhatInputSurfaceAccepted: { if (mState != CONFIGURED) { // state transitioned unexpectedly; we should have replied already. ALOGD("received kWhatInputSurfaceAccepted message in state %s", stateString(mState).c_str()); break; } // response to initiateSetInputSurface() status_t err = NO_ERROR; sp response = new AMessage(); if (!msg->findInt32("err", &err)) { CHECK(msg->findMessage("input-format", &mInputFormat)); CHECK(msg->findMessage("output-format", &mOutputFormat)); mHaveInputSurface = true; } else { response->setInt32("err", err); } postPendingRepliesAndDeferredMessages("kWhatInputSurfaceAccepted", response); break; } case kWhatSignaledInputEOS: { if (!isExecuting()) { // state transitioned unexpectedly; we should have replied already. ALOGD("received kWhatSignaledInputEOS message in state %s", stateString(mState).c_str()); break; } // response to signalEndOfInputStream() sp response = new AMessage; status_t err; if (msg->findInt32("err", &err)) { response->setInt32("err", err); } postPendingRepliesAndDeferredMessages("kWhatSignaledInputEOS", response); break; } case kWhatStartCompleted: { if (mState == RELEASING || mState == UNINITIALIZED) { // In case a kWhatRelease message came in and replied, // we log a warning and ignore. ALOGW("start interrupted by release, current state %d/%s", mState, stateString(mState).c_str()); break; } CHECK_EQ(mState, STARTING); if (mDomain == DOMAIN_VIDEO || mDomain == DOMAIN_IMAGE) { mResourceManagerProxy->addResource( MediaResource::GraphicMemoryResource(getGraphicBufferSize())); } // Notify the RM that the codec is in use (has been started). ClientConfigParcel clientConfig; initClientConfigParcel(clientConfig); mResourceManagerProxy->notifyClientStarted(clientConfig); setState(STARTED); postPendingRepliesAndDeferredMessages("kWhatStartCompleted"); // Now that the codec has started, configure, by default, the peek behavior to // be undefined for backwards compatibility with older releases. Later, if an // app explicitly enables or disables peek, the parameter will be turned off and // the legacy undefined behavior is disallowed. // See updateTunnelPeek called in onSetParameters for more details. if (mTunneled && mTunnelPeekState == TunnelPeekState::kLegacyMode) { sp params = new AMessage; params->setInt32("android._tunnel-peek-set-legacy", 1); mCodec->signalSetParameters(params); } break; } case kWhatOutputBuffersChanged: { mFlags |= kFlagOutputBuffersChanged; postActivityNotificationIfPossible(); break; } case kWhatOutputFramesRendered: { // ignore these in all states except running if (mState != STARTED) { break; } TunnelPeekState previousState = mTunnelPeekState; if (mTunnelPeekState != TunnelPeekState::kLegacyMode) { mTunnelPeekState = TunnelPeekState::kBufferRendered; ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(TunnelPeekState::kBufferRendered)); } processRenderedFrames(msg); // check that we have a notification set if (mOnFrameRenderedNotification != NULL) { sp notify = mOnFrameRenderedNotification->dup(); notify->setMessage("data", msg); notify->post(); } break; } case kWhatFirstTunnelFrameReady: { if (mState != STARTED) { break; } TunnelPeekState previousState = mTunnelPeekState; switch(mTunnelPeekState) { case TunnelPeekState::kDisabledNoBuffer: case TunnelPeekState::kDisabledQueued: mTunnelPeekState = TunnelPeekState::kBufferDecoded; ALOGV("First tunnel frame ready"); ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); break; case TunnelPeekState::kEnabledNoBuffer: case TunnelPeekState::kEnabledQueued: { sp parameters = new AMessage(); parameters->setInt32("android._trigger-tunnel-peek", 1); mCodec->signalSetParameters(parameters); } mTunnelPeekState = TunnelPeekState::kBufferRendered; ALOGV("First tunnel frame ready"); ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); break; default: ALOGV("Ignoring first tunnel frame ready, TunnelPeekState: %s", asString(mTunnelPeekState)); break; } if (mOnFirstTunnelFrameReadyNotification != nullptr) { sp notify = mOnFirstTunnelFrameReadyNotification->dup(); notify->setMessage("data", msg); notify->post(); } break; } case kWhatFillThisBuffer: { /* size_t index = */updateBuffers(kPortIndexInput, msg); bool inStateToReturnBuffers = mState == FLUSHING || mState == STOPPING || mState == RELEASING; if (android::media::codec::provider_->codec_buffer_state_cleanup()) { // Late callbacks from the codec could arrive here // after the codec is already stopped or released. inStateToReturnBuffers = mState == FLUSHING || mState == STOPPING || mState == INITIALIZED || mState == RELEASING || mState == UNINITIALIZED; } if (inStateToReturnBuffers) { returnBuffersToCodecOnPort(kPortIndexInput); break; } if (!mCSD.empty()) { ssize_t index = dequeuePortBuffer(kPortIndexInput); CHECK_GE(index, 0); // If codec specific data had been specified as // part of the format in the call to configure and // if there's more csd left, we submit it here // clients only get access to input buffers once // this data has been exhausted. status_t err = queueCSDInputBuffer(index); if (err != OK) { ALOGE("queueCSDInputBuffer failed w/ error %d", err); setStickyError(err); postActivityNotificationIfPossible(); cancelPendingDequeueOperations(); } break; } if (!mLeftover.empty()) { ssize_t index = dequeuePortBuffer(kPortIndexInput); CHECK_GE(index, 0); status_t err = handleLeftover(index); if (err != OK) { setStickyError(err); postActivityNotificationIfPossible(); cancelPendingDequeueOperations(); } break; } if (mFlags & kFlagIsAsync) { if (!mHaveInputSurface) { if (mState == FLUSHED) { mHavePendingInputBuffers = true; } else { onInputBufferAvailable(); } } } else if (mFlags & kFlagDequeueInputPending) { CHECK(handleDequeueInputBuffer(mDequeueInputReplyID)); ++mDequeueInputTimeoutGeneration; mFlags &= ~kFlagDequeueInputPending; mDequeueInputReplyID = 0; } else { postActivityNotificationIfPossible(); } break; } case kWhatDrainThisBuffer: { if ((mFlags & kFlagUseBlockModel) == 0 && mTunneled) { sp obj; CHECK(msg->findObject("buffer", &obj)); sp buffer = static_cast(obj.get()); if (mFlags & kFlagIsAsync) { // In asynchronous mode, output format change is processed immediately. handleOutputFormatChangeIfNeeded(buffer); } else { postActivityNotificationIfPossible(); } mBufferChannel->discardBuffer(buffer); break; } /* size_t index = */updateBuffers(kPortIndexOutput, msg); bool inStateToReturnBuffers = mState == FLUSHING || mState == STOPPING || mState == RELEASING; if (android::media::codec::provider_->codec_buffer_state_cleanup()) { // Late callbacks from the codec could arrive here // after the codec is already stopped or released. inStateToReturnBuffers = mState == FLUSHING || mState == STOPPING || mState == INITIALIZED || mState == RELEASING || mState == UNINITIALIZED; } if (inStateToReturnBuffers) { returnBuffersToCodecOnPort(kPortIndexOutput); break; } if (mFlags & kFlagIsAsync) { sp obj; CHECK(msg->findObject("buffer", &obj)); sp buffer = static_cast(obj.get()); // In asynchronous mode, output format change is processed immediately. handleOutputFormatChangeIfNeeded(buffer); onOutputBufferAvailable(); } else if (mFlags & kFlagDequeueOutputPending) { DequeueOutputResult dequeueResult = handleDequeueOutputBuffer(mDequeueOutputReplyID); switch (dequeueResult) { case DequeueOutputResult::kNoBuffer: TRESPASS(); break; case DequeueOutputResult::kDiscardedBuffer: break; case DequeueOutputResult::kRepliedWithError: [[fallthrough]]; case DequeueOutputResult::kSuccess: { ++mDequeueOutputTimeoutGeneration; mFlags &= ~kFlagDequeueOutputPending; mDequeueOutputReplyID = 0; break; } default: TRESPASS(); } } else { postActivityNotificationIfPossible(); } break; } case kWhatMetricsUpdated: { sp updatedMetrics; CHECK(msg->findMessage("updated-metrics", &updatedMetrics)); size_t numEntries = updatedMetrics->countEntries(); AMessage::Type type; for (size_t i = 0; i < numEntries; ++i) { const char *name = updatedMetrics->getEntryNameAt(i, &type); AMessage::ItemData itemData = updatedMetrics->getEntryAt(i); switch (type) { case AMessage::kTypeInt32: { int32_t metricValue; itemData.find(&metricValue); mediametrics_setInt32(mMetricsHandle, name, metricValue); break; } case AMessage::kTypeInt64: { int64_t metricValue; itemData.find(&metricValue); mediametrics_setInt64(mMetricsHandle, name, metricValue); break; } case AMessage::kTypeDouble: { double metricValue; itemData.find(&metricValue); mediametrics_setDouble(mMetricsHandle, name, metricValue); break; } case AMessage::kTypeString: { AString metricValue; itemData.find(&metricValue); mediametrics_setCString(mMetricsHandle, name, metricValue.c_str()); break; } // ToDo: add support for other types default: ALOGW("Updated metrics type not supported."); } } break; } case kWhatEOS: { // We already notify the client of this by using the // corresponding flag in "onOutputBufferReady". break; } case kWhatStopCompleted: { if (mState != STOPPING) { ALOGW("Received kWhatStopCompleted in state %d/%s", mState, stateString(mState).c_str()); break; } if (mIsSurfaceToDisplay) { mVideoRenderQualityTracker.resetForDiscontinuity(); } // Notify the RM that the codec has been stopped. ClientConfigParcel clientConfig; initClientConfigParcel(clientConfig); mResourceManagerProxy->notifyClientStopped(clientConfig); setState(INITIALIZED); if (mReplyID) { postPendingRepliesAndDeferredMessages("kWhatStopCompleted"); } else { ALOGW("kWhatStopCompleted: presumably an error occurred earlier, " "but the operation completed anyway. (last reply origin=%s)", mLastReplyOrigin.c_str()); } break; } case kWhatReleaseCompleted: { if (mState != RELEASING) { ALOGW("Received kWhatReleaseCompleted in state %d/%s", mState, stateString(mState).c_str()); break; } setState(UNINITIALIZED); mComponentName.clear(); mFlags &= ~kFlagIsComponentAllocated; // off since we're removing all resources including the battery on if (mBatteryChecker != nullptr) { mBatteryChecker->onClientRemoved(); } mResourceManagerProxy->removeClient(); mDetachedSurface.reset(); if (mReplyID != nullptr) { postPendingRepliesAndDeferredMessages("kWhatReleaseCompleted"); } if (mAsyncReleaseCompleteNotification != nullptr) { flushMediametrics(); mAsyncReleaseCompleteNotification->post(); mAsyncReleaseCompleteNotification.clear(); } break; } case kWhatFlushCompleted: { if (mState != FLUSHING) { ALOGW("received FlushCompleted message in state %d/%s", mState, stateString(mState).c_str()); break; } if (mIsSurfaceToDisplay) { mVideoRenderQualityTracker.resetForDiscontinuity(); } if (mFlags & kFlagIsAsync) { setState(FLUSHED); } else { setState(STARTED); mCodec->signalResume(); } mReliabilityContextMetrics.flushCount++; postPendingRepliesAndDeferredMessages("kWhatFlushCompleted"); break; } default: TRESPASS(); } break; } case kWhatInit: { if (mState != UNINITIALIZED) { PostReplyWithError(msg, INVALID_OPERATION); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mReplyID = replyID; setState(INITIALIZING); sp codecInfo; (void)msg->findObject("codecInfo", &codecInfo); AString name; CHECK(msg->findString("name", &name)); sp format = new AMessage; if (codecInfo) { format->setObject("codecInfo", codecInfo); } format->setString("componentName", name); mCodec->initiateAllocateComponent(format); break; } case kWhatSetNotification: { sp notify; if (msg->findMessage("on-frame-rendered", ¬ify)) { mOnFrameRenderedNotification = notify; } if (msg->findMessage("first-tunnel-frame-ready", ¬ify)) { mOnFirstTunnelFrameReadyNotification = notify; } break; } case kWhatSetCallback: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState == UNINITIALIZED || mState == INITIALIZING || isExecuting()) { // callback can't be set after codec is executing, // or before it's initialized (as the callback // will be cleared when it goes to INITIALIZED) mErrorLog.log(LOG_TAG, base::StringPrintf( "Invalid to call %s; only valid at Initialized state", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); break; } sp callback; CHECK(msg->findMessage("callback", &callback)); mCallback = callback; if (mCallback != NULL) { ALOGI("MediaCodec will operate in async mode"); mFlags |= kFlagIsAsync; } else { mFlags &= ~kFlagIsAsync; } sp response = new AMessage; response->postReply(replyID); break; } case kWhatGetMetrics: { onGetMetrics(msg); break; } case kWhatConfigure: { if (mState != INITIALIZED) { mErrorLog.log(LOG_TAG, base::StringPrintf( "configure() is valid only at Initialized state; currently %s", apiStateString().c_str())); PostReplyWithError(msg, INVALID_OPERATION); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); sp obj; CHECK(msg->findObject("surface", &obj)); sp format; CHECK(msg->findMessage("format", &format)); // start with a copy of the passed metrics info for use in this run mediametrics_handle_t handle; CHECK(msg->findInt64("metrics", &handle)); if (handle != 0) { if (mMetricsHandle != 0) { flushMediametrics(); } mMetricsHandle = mediametrics_dup(handle); // and set some additional metrics values initMediametrics(); } // from this point forward, in this configure/use/release lifecycle, we want to // upload our data mMetricsToUpload = true; int32_t push; if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) { mFlags |= kFlagPushBlankBuffersOnShutdown; } uint32_t flags; CHECK(msg->findInt32("flags", (int32_t *)&flags)); if (android::media::codec::provider_->null_output_surface_support()) { if (obj == nullptr && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) && !(flags & CONFIGURE_FLAG_ENCODE)) { sp surface = getOrCreateDetachedSurface(); if (surface == nullptr) { mErrorLog.log( LOG_TAG, "Detached surface mode is not supported by this codec"); PostReplyWithError(replyID, INVALID_OPERATION); } obj = surface; } } if (obj != NULL) { if (!format->findInt32(KEY_ALLOW_FRAME_DROP, &mAllowFrameDroppingBySurface)) { // allow frame dropping by surface by default mAllowFrameDroppingBySurface = true; } format->setObject("native-window", obj); status_t err = handleSetSurface(static_cast(obj.get())); if (err != OK) { PostReplyWithError(replyID, err); break; } uint32_t generation = mSurfaceGeneration; format->setInt32("native-window-generation", generation); } else { // we are not using surface so this variable is not used, but initialize sensibly anyway mAllowFrameDroppingBySurface = false; handleSetSurface(NULL); } mApiUsageMetrics.isUsingOutputSurface = true; if (flags & CONFIGURE_FLAG_USE_BLOCK_MODEL || flags & CONFIGURE_FLAG_USE_CRYPTO_ASYNC) { if (!(mFlags & kFlagIsAsync)) { mErrorLog.log( LOG_TAG, "Block model is only valid with callback set (async mode)"); PostReplyWithError(replyID, INVALID_OPERATION); break; } if (flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) { mFlags |= kFlagUseBlockModel; } if (flags & CONFIGURE_FLAG_USE_CRYPTO_ASYNC) { mFlags |= kFlagUseCryptoAsync; if ((mFlags & kFlagUseBlockModel)) { ALOGW("CrytoAsync not yet enabled for block model, " "falling back to normal"); } } } int32_t largeFrameParamMax = 0, largeFrameParamThreshold = 0; if (format->findInt32(KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE, &largeFrameParamMax) || format->findInt32(KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE, &largeFrameParamThreshold)) { if (largeFrameParamMax > 0 || largeFrameParamThreshold > 0) { if(mComponentName.startsWith("OMX")) { mErrorLog.log(LOG_TAG, "Large Frame params are not supported on OMX codecs." "Currently only supported on C2 audio codec."); PostReplyWithError(replyID, INVALID_OPERATION); break; } AString mime; CHECK(format->findString("mime", &mime)); if (!mime.startsWith("audio")) { mErrorLog.log(LOG_TAG, "Large Frame params only works with audio codec"); PostReplyWithError(replyID, INVALID_OPERATION); break; } if (!(mFlags & kFlagIsAsync)) { mErrorLog.log(LOG_TAG, "Large Frame audio" \ "config works only with async mode"); PostReplyWithError(replyID, INVALID_OPERATION); break; } } } mReplyID = replyID; setState(CONFIGURING); void *crypto; if (!msg->findPointer("crypto", &crypto)) { crypto = NULL; } ALOGV("kWhatConfigure: Old mCrypto: %p (%d)", mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); mCrypto = static_cast(crypto); mBufferChannel->setCrypto(mCrypto); ALOGV("kWhatConfigure: New mCrypto: %p (%d)", mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); void *descrambler; if (!msg->findPointer("descrambler", &descrambler)) { descrambler = NULL; } mDescrambler = static_cast(descrambler); mBufferChannel->setDescrambler(mDescrambler); if ((mFlags & kFlagUseCryptoAsync) && mCrypto) { // set kFlagUseCryptoAsync but do-not use this for block model // this is to propagate the error in onCryptoError() // TODO (b/274628160): Enable Use of CONFIG_FLAG_USE_CRYPTO_ASYNC // with CONFIGURE_FLAG_USE_BLOCK_MODEL) if (!(mFlags & kFlagUseBlockModel)) { mCryptoAsync = new CryptoAsync(mBufferChannel); mCryptoAsync->setCallback( std::make_unique(new AMessage(kWhatCodecNotify, this))); mCryptoLooper = new ALooper(); mCryptoLooper->setName("CryptoAsyncLooper"); mCryptoLooper->registerHandler(mCryptoAsync); status_t err = mCryptoLooper->start(); if (err != OK) { ALOGE("Crypto Looper failed to start"); mCryptoAsync = nullptr; mCryptoLooper = nullptr; } } } format->setInt32("flags", flags); if (flags & CONFIGURE_FLAG_ENCODE) { format->setInt32("encoder", true); mFlags |= kFlagIsEncoder; } extractCSD(format); int32_t tunneled; if (format->findInt32("feature-tunneled-playback", &tunneled) && tunneled != 0) { ALOGI("Configuring TUNNELED video playback."); mTunneled = true; } else { mTunneled = false; } mediametrics_setInt32(mMetricsHandle, kCodecTunneled, mTunneled ? 1 : 0); int32_t background = 0; if (format->findInt32("android._background-mode", &background) && background) { androidSetThreadPriority(gettid(), ANDROID_PRIORITY_BACKGROUND); } mCodec->initiateConfigureComponent(format); break; } case kWhatDetachSurface: { // detach surface is equivalent to setSurface(mDetachedSurface) sp surface = getOrCreateDetachedSurface(); if (surface == nullptr) { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mErrorLog.log(LOG_TAG, "Detaching surface is not supported by the codec."); PostReplyWithError(replyID, INVALID_OPERATION); break; } msg->setObject("surface", surface); } [[fallthrough]]; case kWhatSetSurface: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); status_t err = OK; switch (mState) { case CONFIGURED: case STARTED: case FLUSHED: { sp obj; (void)msg->findObject("surface", &obj); sp surface = static_cast(obj.get()); if (mSurface == NULL) { // do not support setting surface if it was not set mErrorLog.log(LOG_TAG, base::StringPrintf( "Cannot %s surface if the codec is not configured with " "a surface already", msg->what() == kWhatDetachSurface ? "detach" : "set")); err = INVALID_OPERATION; } else if (obj == NULL) { // do not support unsetting surface mErrorLog.log(LOG_TAG, "Unsetting surface is not supported"); err = BAD_VALUE; } else if (android::media::codec::provider_->null_output_surface_support()) { err = handleSetSurface(surface, true /* callCodec */); } else { uint32_t generation; err = connectToSurface(surface, &generation); if (err == ALREADY_EXISTS) { // reconnecting to same surface err = OK; } else { if (err == OK) { if (mFlags & kFlagUsesSoftwareRenderer) { if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { pushBlankBuffersToNativeWindow(mSurface.get()); } surface->setDequeueTimeout(-1); mSoftRenderer = new SoftwareRenderer(surface); // TODO: check if this was successful } else { err = mCodec->setSurface(surface, generation); } } if (err == OK) { (void)disconnectFromSurface(); mSurface = surface; mSurfaceGeneration = generation; } mReliabilityContextMetrics.setOutputSurfaceCount++; } } break; } default: mErrorLog.log(LOG_TAG, base::StringPrintf( "%sSurface() is valid only at Executing states; currently %s", msg->what() == kWhatDetachSurface ? "detach" : "set", apiStateString().c_str())); err = INVALID_OPERATION; break; } PostReplyWithError(replyID, err); break; } case kWhatCreateInputSurface: case kWhatSetInputSurface: { // Must be configured, but can't have been started yet. if (mState != CONFIGURED) { mErrorLog.log(LOG_TAG, base::StringPrintf( "setInputSurface() is valid only at Configured state; currently %s", apiStateString().c_str())); PostReplyWithError(msg, INVALID_OPERATION); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mReplyID = replyID; if (msg->what() == kWhatCreateInputSurface) { mCodec->initiateCreateInputSurface(); } else { sp obj; CHECK(msg->findObject("input-surface", &obj)); mCodec->initiateSetInputSurface( static_cast(obj.get())); } break; } case kWhatStart: { if (mState == FLUSHED) { setState(STARTED); if (mHavePendingInputBuffers) { onInputBufferAvailable(); mHavePendingInputBuffers = false; } mCodec->signalResume(); PostReplyWithError(msg, OK); break; } else if (mState != CONFIGURED) { mErrorLog.log(LOG_TAG, base::StringPrintf( "start() is valid only at Configured state; currently %s", apiStateString().c_str())); PostReplyWithError(msg, INVALID_OPERATION); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); TunnelPeekState previousState = mTunnelPeekState; if (previousState != TunnelPeekState::kLegacyMode) { mTunnelPeekState = mTunnelPeekEnabled ? TunnelPeekState::kEnabledNoBuffer : TunnelPeekState::kDisabledNoBuffer; ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); } mReplyID = replyID; setState(STARTING); mCodec->initiateStart(); break; } case kWhatStop: { if (mReplyID) { mDeferredMessages.push_back(msg); break; } [[fallthrough]]; } case kWhatRelease: { State targetState = (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; if ((mState == RELEASING && targetState == UNINITIALIZED) || (mState == STOPPING && targetState == INITIALIZED)) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mCryptoAsync) { mCryptoAsync->stop(); } sp asyncNotify; (void)msg->findMessage("async", &asyncNotify); // post asyncNotify if going out of scope. struct AsyncNotifyPost { AsyncNotifyPost(const sp &asyncNotify) : mAsyncNotify(asyncNotify) {} ~AsyncNotifyPost() { if (mAsyncNotify) { mAsyncNotify->post(); } } void clear() { mAsyncNotify.clear(); } private: sp mAsyncNotify; } asyncNotifyPost{asyncNotify}; // already stopped/released if (mState == UNINITIALIZED && mReleasedByResourceManager) { sp response = new AMessage; response->setInt32("err", OK); response->postReply(replyID); break; } int32_t reclaimed = 0; msg->findInt32("reclaimed", &reclaimed); if (reclaimed) { if (!mReleasedByResourceManager) { // notify the async client if (mFlags & kFlagIsAsync) { onError(DEAD_OBJECT, ACTION_CODE_FATAL); } mErrorLog.log(LOG_TAG, "Released by resource manager"); mReleasedByResourceManager = true; } int32_t force = 0; msg->findInt32("force", &force); if (!force && hasPendingBuffer()) { ALOGW("Can't reclaim codec right now due to pending buffers."); // return WOULD_BLOCK to ask resource manager to retry later. sp response = new AMessage; response->setInt32("err", WOULD_BLOCK); response->postReply(replyID); break; } } bool isReleasingAllocatedComponent = (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED; if (!isReleasingAllocatedComponent // See 1 && mState != INITIALIZED && mState != CONFIGURED && !isExecuting()) { // 1) Permit release to shut down the component if allocated. // // 2) We may be in "UNINITIALIZED" state already and // also shutdown the encoder/decoder without the // client being aware of this if media server died while // we were being stopped. The client would assume that // after stop() returned, it would be safe to call release() // and it should be in this case, no harm to allow a release() // if we're already uninitialized. sp response = new AMessage; // TODO: we shouldn't throw an exception for stop/release. Change this to wait until // the previous stop/release completes and then reply with OK. status_t err = mState == targetState ? OK : INVALID_OPERATION; response->setInt32("err", err); // TODO: mErrorLog if (err == OK && targetState == UNINITIALIZED) { mComponentName.clear(); } response->postReply(replyID); break; } // If we're flushing, configuring or starting but // received a release request, post the reply for the pending call // first, and consider it done. The reply token will be replaced // after this, and we'll no longer be able to reply. if (mState == FLUSHING || mState == CONFIGURING || mState == STARTING) { // mReply is always set if in these states. postPendingRepliesAndDeferredMessages( std::string("kWhatRelease:") + stateString(mState)); } // If we're stopping but received a release request, post the reply // for the pending call if necessary. Note that the reply may have been // already posted due to an error. if (mState == STOPPING && mReplyID) { postPendingRepliesAndDeferredMessages("kWhatRelease:STOPPING"); } if (mFlags & kFlagSawMediaServerDie) { // It's dead, Jim. Don't expect initiateShutdown to yield // any useful results now... // Any pending reply would have been handled at kWhatError. setState(UNINITIALIZED); if (targetState == UNINITIALIZED) { mComponentName.clear(); } (new AMessage)->postReply(replyID); break; } // If we already have an error, component may not be able to // complete the shutdown properly. If we're stopping, post the // reply now with an error to unblock the client, client can // release after the failure (instead of ANR). if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) { // Any pending reply would have been handled at kWhatError. PostReplyWithError(replyID, getStickyError()); break; } bool forceSync = false; if (asyncNotify != nullptr && mSurface != NULL) { if (android::media::codec::provider_->null_output_surface_support()) { if (handleSetSurface(getOrCreateDetachedSurface(), true /* callCodec */, true /* onShutDown */) != OK) { // We were not able to detach the surface, so force // synchronous release. forceSync = true; } } else { if (!mDetachedSurface) { uint64_t usage = 0; if (mSurface->getConsumerUsage(&usage) != OK) { usage = 0; } mDetachedSurface.reset(new ReleaseSurface(usage)); } if (mSurface != mDetachedSurface->getSurface()) { uint32_t generation; status_t err = connectToSurface(mDetachedSurface->getSurface(), &generation); ALOGW_IF(err != OK, "error connecting to release surface: err = %d", err); if (err == OK && !(mFlags & kFlagUsesSoftwareRenderer)) { err = mCodec->setSurface(mDetachedSurface->getSurface(), generation); ALOGW_IF(err != OK, "error setting release surface: err = %d", err); } if (err == OK) { (void)disconnectFromSurface(); mSurface = mDetachedSurface->getSurface(); mSurfaceGeneration = generation; } else { // We were not able to switch the surface, so force // synchronous release. forceSync = true; } } } } if (mReplyID) { // State transition replies are handled above, so this reply // would not be related to state transition. As we are // shutting down the component, just fail the operation. postPendingRepliesAndDeferredMessages("kWhatRelease:reply", UNKNOWN_ERROR); } mReplyID = replyID; setState(msg->what() == kWhatStop ? STOPPING : RELEASING); mCodec->initiateShutdown( msg->what() == kWhatStop /* keepComponentAllocated */); returnBuffersToCodec(reclaimed); if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { pushBlankBuffersToNativeWindow(mSurface.get()); } if (asyncNotify != nullptr) { if (!forceSync) { mResourceManagerProxy->markClientForPendingRemoval(); postPendingRepliesAndDeferredMessages("kWhatRelease:async"); } asyncNotifyPost.clear(); mAsyncReleaseCompleteNotification = asyncNotify; } break; } case kWhatDequeueInputBuffer: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mFlags & kFlagIsAsync) { mErrorLog.log(LOG_TAG, "dequeueInputBuffer can't be used in async mode"); PostReplyWithError(replyID, INVALID_OPERATION); break; } if (mHaveInputSurface) { mErrorLog.log(LOG_TAG, "dequeueInputBuffer can't be used with input surface"); PostReplyWithError(replyID, INVALID_OPERATION); break; } if (handleDequeueInputBuffer(replyID, true /* new request */)) { break; } int64_t timeoutUs; CHECK(msg->findInt64("timeoutUs", &timeoutUs)); if (timeoutUs == 0LL) { PostReplyWithError(replyID, -EAGAIN); break; } mFlags |= kFlagDequeueInputPending; mDequeueInputReplyID = replyID; if (timeoutUs > 0LL) { sp timeoutMsg = new AMessage(kWhatDequeueInputTimedOut, this); timeoutMsg->setInt32( "generation", ++mDequeueInputTimeoutGeneration); timeoutMsg->post(timeoutUs); } break; } case kWhatDequeueInputTimedOut: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mDequeueInputTimeoutGeneration) { // Obsolete break; } CHECK(mFlags & kFlagDequeueInputPending); PostReplyWithError(mDequeueInputReplyID, -EAGAIN); mFlags &= ~kFlagDequeueInputPending; mDequeueInputReplyID = 0; break; } case kWhatQueueInputBuffer: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "queueInputBuffer() is valid only at Executing states; currently %s", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); break; } status_t err = UNKNOWN_ERROR; if (!mLeftover.empty()) { mLeftover.push_back(msg); size_t index; msg->findSize("index", &index); err = handleLeftover(index); } else { err = onQueueInputBuffer(msg); } PostReplyWithError(replyID, err); break; } case kWhatDequeueOutputBuffer: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mFlags & kFlagIsAsync) { mErrorLog.log(LOG_TAG, "dequeueOutputBuffer can't be used in async mode"); PostReplyWithError(replyID, INVALID_OPERATION); break; } DequeueOutputResult dequeueResult = handleDequeueOutputBuffer(replyID, true /* new request */); switch (dequeueResult) { case DequeueOutputResult::kNoBuffer: [[fallthrough]]; case DequeueOutputResult::kDiscardedBuffer: { int64_t timeoutUs; CHECK(msg->findInt64("timeoutUs", &timeoutUs)); if (timeoutUs == 0LL) { PostReplyWithError(replyID, -EAGAIN); break; } mFlags |= kFlagDequeueOutputPending; mDequeueOutputReplyID = replyID; if (timeoutUs > 0LL) { sp timeoutMsg = new AMessage(kWhatDequeueOutputTimedOut, this); timeoutMsg->setInt32( "generation", ++mDequeueOutputTimeoutGeneration); timeoutMsg->post(timeoutUs); } break; } case DequeueOutputResult::kRepliedWithError: [[fallthrough]]; case DequeueOutputResult::kSuccess: break; default: TRESPASS(); } break; } case kWhatDequeueOutputTimedOut: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mDequeueOutputTimeoutGeneration) { // Obsolete break; } CHECK(mFlags & kFlagDequeueOutputPending); PostReplyWithError(mDequeueOutputReplyID, -EAGAIN); mFlags &= ~kFlagDequeueOutputPending; mDequeueOutputReplyID = 0; break; } case kWhatReleaseOutputBuffer: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "releaseOutputBuffer() is valid only at Executing states; currently %s", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); break; } status_t err = onReleaseOutputBuffer(msg); PostReplyWithError(replyID, err); break; } case kWhatPollForRenderedBuffers: { if (isExecuting()) { mBufferChannel->pollForRenderedBuffers(); } break; } case kWhatSignalEndOfInputStream: { if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "signalEndOfInputStream() is valid only at Executing states; currently %s", apiStateString().c_str())); PostReplyWithError(msg, INVALID_OPERATION); break; } else if (!mHaveInputSurface) { mErrorLog.log( LOG_TAG, "signalEndOfInputStream() called without an input surface set"); PostReplyWithError(msg, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(msg, getStickyError()); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mReplyID = replyID; mCodec->signalEndOfInputStream(); break; } case kWhatGetBuffers: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "getInput/OutputBuffers() is valid only at Executing states; currently %s", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (mFlags & kFlagIsAsync) { mErrorLog.log(LOG_TAG, "getInput/OutputBuffers() is not supported with callbacks"); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); break; } int32_t portIndex; CHECK(msg->findInt32("portIndex", &portIndex)); Vector > *dstBuffers; CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); dstBuffers->clear(); // If we're using input surface (either non-persistent created by // createInputSurface(), or persistent set by setInputSurface()), // give the client an empty input buffers array. if (portIndex != kPortIndexInput || !mHaveInputSurface) { if (portIndex == kPortIndexInput) { mBufferChannel->getInputBufferArray(dstBuffers); } else { mBufferChannel->getOutputBufferArray(dstBuffers); } } mApiUsageMetrics.isArrayMode = true; (new AMessage)->postReply(replyID); break; } case kWhatFlush: { if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "flush() is valid only at Executing states; currently %s", apiStateString().c_str())); PostReplyWithError(msg, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(msg, getStickyError()); break; } if (mReplyID) { mDeferredMessages.push_back(msg); break; } sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mReplyID = replyID; // TODO: skip flushing if already FLUSHED setState(FLUSHING); if (mCryptoAsync) { std::list> pendingBuffers; mCryptoAsync->stop(&pendingBuffers); //TODO: do something with these buffers } mCodec->signalFlush(); returnBuffersToCodec(); TunnelPeekState previousState = mTunnelPeekState; if (previousState != TunnelPeekState::kLegacyMode) { mTunnelPeekState = mTunnelPeekEnabled ? TunnelPeekState::kEnabledNoBuffer : TunnelPeekState::kDisabledNoBuffer; ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); } break; } case kWhatGetInputFormat: case kWhatGetOutputFormat: { sp format = (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState != CONFIGURED && mState != STARTING && mState != STARTED && mState != FLUSHING && mState != FLUSHED) { mErrorLog.log(LOG_TAG, base::StringPrintf( "getInput/OutputFormat() is valid at Executing states " "and Configured state; currently %s", apiStateString().c_str())); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (format == NULL) { mErrorLog.log(LOG_TAG, "Fatal error: format is not initialized"); PostReplyWithError(replyID, INVALID_OPERATION); break; } else if (mFlags & kFlagStickyError) { PostReplyWithError(replyID, getStickyError()); break; } sp response = new AMessage; response->setMessage("format", format); response->postReply(replyID); break; } case kWhatRequestIDRFrame: { mCodec->signalRequestIDRFrame(); break; } case kWhatRequestActivityNotification: { CHECK(mActivityNotify == NULL); CHECK(msg->findMessage("notify", &mActivityNotify)); postActivityNotificationIfPossible(); break; } case kWhatGetName: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mComponentName.empty()) { mErrorLog.log(LOG_TAG, "Fatal error: name is not set"); PostReplyWithError(replyID, INVALID_OPERATION); break; } sp response = new AMessage; response->setString("name", mComponentName.c_str()); response->postReply(replyID); break; } case kWhatGetCodecInfo: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); sp response = new AMessage; response->setObject("codecInfo", mCodecInfo); response->postReply(replyID); break; } case kWhatSetParameters: { sp replyID; CHECK(msg->senderAwaitsResponse(&replyID)); sp params; CHECK(msg->findMessage("params", ¶ms)); status_t err = onSetParameters(params); PostReplyWithError(replyID, err); break; } case kWhatDrmReleaseCrypto: { onReleaseCrypto(msg); break; } case kWhatCheckBatteryStats: { if (mBatteryChecker != nullptr) { mBatteryChecker->onCheckBatteryTimer(msg, [this] () { mResourceManagerProxy->removeResource( MediaResource::VideoBatteryResource(mIsHardware)); }); } break; } default: TRESPASS(); } } // always called from the looper thread void MediaCodec::handleOutputFormatChangeIfNeeded(const sp &buffer) { sp format = buffer->format(); if (mOutputFormat == format) { return; } if (mFlags & kFlagUseBlockModel) { sp diff1 = mOutputFormat->changesFrom(format); sp diff2 = format->changesFrom(mOutputFormat); std::set keys; size_t numEntries = diff1->countEntries(); AMessage::Type type; for (size_t i = 0; i < numEntries; ++i) { keys.emplace(diff1->getEntryNameAt(i, &type)); } numEntries = diff2->countEntries(); for (size_t i = 0; i < numEntries; ++i) { keys.emplace(diff2->getEntryNameAt(i, &type)); } sp>> changedKeys{ new WrapperObject>{std::move(keys)}}; buffer->meta()->setObject("changedKeys", changedKeys); } mOutputFormat = format; mapFormat(mComponentName, format, nullptr, true); ALOGV("[%s] output format changed to: %s", mComponentName.c_str(), mOutputFormat->debugString(4).c_str()); if (mSoftRenderer == NULL && mSurface != NULL && (mFlags & kFlagUsesSoftwareRenderer)) { AString mime; CHECK(mOutputFormat->findString("mime", &mime)); // TODO: propagate color aspects to software renderer to allow better // color conversion to RGB. For now, just mark dataspace for YUV // rendering. int32_t dataSpace; if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) { ALOGD("[%s] setting dataspace on output surface to %#x", mComponentName.c_str(), dataSpace); int err = native_window_set_buffers_data_space( mSurface.get(), (android_dataspace)dataSpace); ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err); } if (mOutputFormat->contains("hdr-static-info")) { HDRStaticInfo info; if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) { setNativeWindowHdrMetadata(mSurface.get(), &info); } } sp hdr10PlusInfo; if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo) && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) { native_window_set_buffers_hdr10_plus_metadata(mSurface.get(), hdr10PlusInfo->size(), hdr10PlusInfo->data()); } if (mime.startsWithIgnoreCase("video/")) { mSurface->setDequeueTimeout(-1); mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees); } } requestCpuBoostIfNeeded(); if (mFlags & kFlagIsEncoder) { // Before we announce the format change we should // collect codec specific data and amend the output // format as necessary. int32_t flags = 0; (void) buffer->meta()->findInt32("flags", &flags); if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure) && !mOwnerName.startsWith("codec2::")) { status_t err = amendOutputFormatWithCodecSpecificData(buffer); if (err != OK) { ALOGE("Codec spit out malformed codec " "specific data!"); } } } // Update the width and the height. int32_t left = 0, top = 0, right = 0, bottom = 0, width = 0, height = 0; bool newSubsession = false; if (android::media::codec::provider_->subsession_metrics() && mOutputFormat->findInt32("width", &width) && mOutputFormat->findInt32("height", &height) && (width != mWidth || height != mHeight)) { // consider a new subsession if the width or height changes. newSubsession = true; } // TODO: properly detect new audio subsession // Only consider a new subsession if we already have output (from a previous subsession). if (newSubsession && mMetricsToUpload && mBytesEncoded > 0) { handleStartingANewSubsession(); } if (mFlags & kFlagIsAsync) { onOutputFormatChanged(); } else { mFlags |= kFlagOutputFormatChanged; postActivityNotificationIfPossible(); } bool resolutionChanged = false; if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { mWidth = right - left + 1; mHeight = bottom - top + 1; resolutionChanged = true; } else if (mOutputFormat->findInt32("width", &width) && mOutputFormat->findInt32("height", &height)) { mWidth = width; mHeight = height; resolutionChanged = true; } // Notify mCrypto and the RM of video resolution changes if (resolutionChanged) { if (mCrypto != NULL) { mCrypto->notifyResolution(mWidth, mHeight); } ClientConfigParcel clientConfig; initClientConfigParcel(clientConfig); mResourceManagerProxy->notifyClientConfigChanged(clientConfig); mReliabilityContextMetrics.resolutionChangeCount++; } updateHdrMetrics(false /* isConfig */); } // always called from the looper thread (and therefore not mutexed) void MediaCodec::handleStartingANewSubsession() { // create a new metrics item for the subsession with the new resolution. // TODO: properly account input counts for the previous and the new // subsessions. We only find out that a new subsession started from the // output format, but by that time we already accounted the input counts // to the previous subsession. flushMediametrics(); // this deletes mMetricsHandle, but stores it in mLastMetricsHandle // hence mLastMetricsHandle has the metrics item for the previous subsession. if ((mFlags & kFlagIsAsync) && mCallback != nullptr) { sp msg = mCallback->dup(); msg->setInt32("callbackID", CB_METRICS_FLUSHED); std::unique_ptr flushedMetrics( mediametrics::Item::convert(mediametrics_dup(mLastMetricsHandle))); msg->setObject("metrics", new WrapperObject>( std::move(flushedMetrics))); msg->post(); } // reuse/continue old metrics item for the new subsession. mMetricsHandle = mediametrics_dup(mLastMetricsHandle); mMetricsToUpload = true; // TODO: configured width/height for the new subsession should be the // previous width/height. mSubsessionCount++; resetSubsessionMetricsFields(); } void MediaCodec::extractCSD(const sp &format) { mCSD.clear(); size_t i = 0; for (;;) { sp csd; if (!format->findBuffer(base::StringPrintf("csd-%zu", i).c_str(), &csd)) { break; } if (csd->size() == 0) { ALOGW("csd-%zu size is 0", i); } else { mCSD.push_back(csd); } ++i; } ALOGV("Found %zu pieces of codec specific data.", mCSD.size()); } status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { CHECK(!mCSD.empty()); sp csd = *mCSD.begin(); mCSD.erase(mCSD.begin()); std::shared_ptr c2Buffer; sp memory; if (mFlags & kFlagUseBlockModel) { if (hasCryptoOrDescrambler()) { constexpr size_t kInitialDealerCapacity = 1048576; // 1MB thread_local sp sDealer = new MemoryDealer( kInitialDealerCapacity, "CSD(1MB)"); sp mem = sDealer->allocate(csd->size()); if (mem == nullptr) { size_t newDealerCapacity = sDealer->getMemoryHeap()->getSize() * 2; while (csd->size() * 2 > newDealerCapacity) { newDealerCapacity *= 2; } sDealer = new MemoryDealer( newDealerCapacity, base::StringPrintf("CSD(%zuMB)", newDealerCapacity / 1048576).c_str()); mem = sDealer->allocate(csd->size()); } memcpy(mem->unsecurePointer(), csd->data(), csd->size()); ssize_t heapOffset; memory = hardware::fromHeap(mem->getMemory(&heapOffset, nullptr)); } else { std::shared_ptr block = FetchLinearBlock(csd->size(), {std::string{mComponentName.c_str()}}); C2WriteView view{block->map().get()}; if (view.error() != C2_OK) { mErrorLog.log(LOG_TAG, "Fatal error: failed to allocate and map a block"); return -EINVAL; } if (csd->size() > view.capacity()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Fatal error: allocated block is too small " "(csd size %zu; block cap %u)", csd->size(), view.capacity())); return -EINVAL; } memcpy(view.base(), csd->data(), csd->size()); c2Buffer = C2Buffer::CreateLinearBuffer(block->share(0, csd->size(), C2Fence{})); } } else { const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex]; const sp &codecInputData = info.mData; if (csd->size() > codecInputData->capacity()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "CSD is too large to fit in input buffer " "(csd size %zu; buffer cap %zu)", csd->size(), codecInputData->capacity())); return -EINVAL; } if (codecInputData->data() == NULL) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Fatal error: input buffer %zu is not properly allocated", bufferIndex)); return -EINVAL; } memcpy(codecInputData->data(), csd->data(), csd->size()); } AString errorDetailMsg; sp msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", bufferIndex); msg->setSize("offset", 0); msg->setSize("size", csd->size()); msg->setInt64("timeUs", 0LL); msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG); msg->setPointer("errorDetailMsg", &errorDetailMsg); if (c2Buffer) { sp>> obj{ new WrapperObject>{c2Buffer}}; msg->setObject("c2buffer", obj); } else if (memory) { sp>> obj{ new WrapperObject>{memory}}; msg->setObject("memory", obj); } return onQueueInputBuffer(msg); } void MediaCodec::setState(State newState) { if (newState == INITIALIZED || newState == UNINITIALIZED) { delete mSoftRenderer; mSoftRenderer = NULL; if ( mCrypto != NULL ) { ALOGV("setState: ~mCrypto: %p (%d)", mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); } mCrypto.clear(); mDescrambler.clear(); handleSetSurface(NULL); mInputFormat.clear(); mOutputFormat.clear(); if (android::media::codec::provider_->codec_buffer_state_cleanup()) { mCSD.clear(); mLeftover.clear(); } mFlags &= ~kFlagOutputFormatChanged; mFlags &= ~kFlagOutputBuffersChanged; mFlags &= ~kFlagStickyError; mFlags &= ~kFlagIsEncoder; mFlags &= ~kFlagIsAsync; mStickyError = OK; mActivityNotify.clear(); mCallback.clear(); mErrorLog.clear(); } if (android::media::codec::provider_->set_state_early()) { mState = newState; } if (newState == UNINITIALIZED) { // return any straggling buffers, e.g. if we got here on an error returnBuffersToCodec(); // The component is gone, mediaserver's probably back up already // but should definitely be back up should we try to instantiate // another component.. and the cycle continues. mFlags &= ~kFlagSawMediaServerDie; } if (!android::media::codec::provider_->set_state_early()) { mState = newState; } if (mBatteryChecker != nullptr) { mBatteryChecker->setExecuting(isExecuting()); } cancelPendingDequeueOperations(); } void MediaCodec::returnBuffersToCodec(bool isReclaim) { returnBuffersToCodecOnPort(kPortIndexInput, isReclaim); returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim); } void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) { CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); Mutex::Autolock al(mBufferLock); if (portIndex == kPortIndexInput) { mLeftover.clear(); } for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) { BufferInfo *info = &mPortBuffers[portIndex][i]; if (info->mData != nullptr) { sp buffer = info->mData; if (isReclaim && info->mOwnedByClient) { ALOGD("port %d buffer %zu still owned by client when codec is reclaimed", portIndex, i); } else { ALOGV("returnBuffersToCodecOnPort: mPortBuffers[%s][%zu] NOT owned by client", portIndex == kPortIndexInput ? "in" : "out", i); info->mOwnedByClient = false; info->mData.clear(); } mBufferChannel->discardBuffer(buffer); } } mAvailPortBuffers[portIndex].clear(); } size_t MediaCodec::updateBuffers( int32_t portIndex, const sp &msg) { CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); size_t index; CHECK(msg->findSize("index", &index)); sp obj; CHECK(msg->findObject("buffer", &obj)); sp buffer = static_cast(obj.get()); { Mutex::Autolock al(mBufferLock); if (mPortBuffers[portIndex].size() <= index) { mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign)); } mPortBuffers[portIndex][index].mData = buffer; } mAvailPortBuffers[portIndex].push_back(index); return index; } status_t MediaCodec::onQueueInputBuffer(const sp &msg) { size_t index; size_t offset = 0; size_t size = 0; int64_t timeUs = 0; uint32_t flags = 0; CHECK(msg->findSize("index", &index)); CHECK(msg->findInt64("timeUs", &timeUs)); CHECK(msg->findInt32("flags", (int32_t *)&flags)); std::shared_ptr c2Buffer; sp memory; sp obj; if (msg->findObject("c2buffer", &obj)) { CHECK(obj); c2Buffer = static_cast> *>(obj.get())->value; } else if (msg->findObject("memory", &obj)) { CHECK(obj); memory = static_cast> *>(obj.get())->value; CHECK(msg->findSize("offset", &offset)); } else { CHECK(msg->findSize("offset", &offset)); } const CryptoPlugin::SubSample *subSamples; size_t numSubSamples = 0; const uint8_t *key = NULL; const uint8_t *iv = NULL; CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted; // We allow the simpler queueInputBuffer API to be used even in // secure mode, by fabricating a single unencrypted subSample. CryptoPlugin::SubSample ss; CryptoPlugin::Pattern pattern; if (android::media::codec::provider_->secure_codecs_require_crypto() && (mFlags & kFlagIsSecure) && !hasCryptoOrDescrambler()) { mErrorLog.log(LOG_TAG, "Crypto or descrambler must be given for secure codec"); return INVALID_OPERATION; } if (msg->findSize("size", &size)) { if (hasCryptoOrDescrambler()) { ss.mNumBytesOfClearData = size; ss.mNumBytesOfEncryptedData = 0; subSamples = &ss; numSubSamples = 1; pattern.mEncryptBlocks = 0; pattern.mSkipBlocks = 0; } } else if (!c2Buffer) { if (!hasCryptoOrDescrambler()) { ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!", mComponentName.c_str()); mErrorLog.log(LOG_TAG, "queuing secure buffer without mCrypto or mDescrambler!"); return -EINVAL; } sp obj; if (msg->findObject("cryptoInfos", &obj)) { CHECK(msg->findSize("ssize", &size)); } else { CHECK(msg->findPointer("subSamples", (void **)&subSamples)); CHECK(msg->findSize("numSubSamples", &numSubSamples)); CHECK(msg->findPointer("key", (void **)&key)); CHECK(msg->findPointer("iv", (void **)&iv)); CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks)); CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks)); int32_t tmp; CHECK(msg->findInt32("mode", &tmp)); mode = (CryptoPlugin::Mode)tmp; size = 0; for (size_t i = 0; i < numSubSamples; ++i) { size += subSamples[i].mNumBytesOfClearData; size += subSamples[i].mNumBytesOfEncryptedData; } } } if (index >= mPortBuffers[kPortIndexInput].size()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "index out of range (index=%zu)", mPortBuffers[kPortIndexInput].size())); return -ERANGE; } BufferInfo *info = &mPortBuffers[kPortIndexInput][index]; sp buffer = info->mData; if (buffer == nullptr) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Fatal error: failed to fetch buffer for index %zu", index)); return -EACCES; } if (!info->mOwnedByClient) { mErrorLog.log(LOG_TAG, base::StringPrintf( "client does not own the buffer #%zu", index)); return -EACCES; } auto setInputBufferParams = [this, &msg, &buffer] (int64_t timeUs, uint32_t flags = 0) -> status_t { status_t err = OK; sp obj; if (msg->findObject("accessUnitInfo", &obj)) { buffer->meta()->setObject("accessUnitInfo", obj); } buffer->meta()->setInt64("timeUs", timeUs); if (flags & BUFFER_FLAG_EOS) { buffer->meta()->setInt32("eos", true); } if (flags & BUFFER_FLAG_CODECCONFIG) { buffer->meta()->setInt32("csd", true); } bool isBufferDecodeOnly = ((flags & BUFFER_FLAG_DECODE_ONLY) != 0); if (isBufferDecodeOnly) { buffer->meta()->setInt32("decode-only", true); } if (mTunneled && !isBufferDecodeOnly && !(flags & BUFFER_FLAG_CODECCONFIG)) { TunnelPeekState previousState = mTunnelPeekState; switch(mTunnelPeekState){ case TunnelPeekState::kEnabledNoBuffer: buffer->meta()->setInt32("tunnel-first-frame", 1); mTunnelPeekState = TunnelPeekState::kEnabledQueued; ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); break; case TunnelPeekState::kDisabledNoBuffer: buffer->meta()->setInt32("tunnel-first-frame", 1); mTunnelPeekState = TunnelPeekState::kDisabledQueued; ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState)); break; default: break; } } return err; }; auto buildCryptoInfoAMessage = [&](const sp & cryptoInfo, int32_t action) { // set decrypt Action cryptoInfo->setInt32("action", action); cryptoInfo->setObject("buffer", buffer); cryptoInfo->setInt32("secure", mFlags & kFlagIsSecure); sp obj; if (msg->findObject("cryptoInfos", &obj)) { // this object is a standalone object when created (no copy requied here) buffer->meta()->setObject("cryptoInfos", obj); } else { size_t key_len = (key != nullptr)? 16 : 0; size_t iv_len = (iv != nullptr)? 16 : 0; sp shared_key; sp shared_iv; if (key_len > 0) { shared_key = ABuffer::CreateAsCopy((void*)key, key_len); } if (iv_len > 0) { shared_iv = ABuffer::CreateAsCopy((void*)iv, iv_len); } sp subSamples_buffer = new ABuffer(sizeof(CryptoPlugin::SubSample) * numSubSamples); CryptoPlugin::SubSample * samples = (CryptoPlugin::SubSample *)(subSamples_buffer.get()->data()); for (int s = 0 ; s < numSubSamples ; s++) { samples[s].mNumBytesOfClearData = subSamples[s].mNumBytesOfClearData; samples[s].mNumBytesOfEncryptedData = subSamples[s].mNumBytesOfEncryptedData; } cryptoInfo->setBuffer("key", shared_key); cryptoInfo->setBuffer("iv", shared_iv); cryptoInfo->setInt32("mode", (int)mode); cryptoInfo->setInt32("encryptBlocks", pattern.mEncryptBlocks); cryptoInfo->setInt32("skipBlocks", pattern.mSkipBlocks); cryptoInfo->setBuffer("subSamples", subSamples_buffer); cryptoInfo->setSize("numSubSamples", numSubSamples); } }; if (c2Buffer || memory) { sp tunings = NULL; if (msg->findMessage("tunings", &tunings) && tunings != NULL) { onSetParameters(tunings); } status_t err = OK; if (c2Buffer) { err = mBufferChannel->attachBuffer(c2Buffer, buffer); // to prevent unnecessary copy for single info case. if (msg->findObject("accessUnitInfo", &obj)) { sp infos{(BufferInfosWrapper*)(obj.get())}; if (infos->value.size() == 1) { msg->removeEntryByName("accessUnitInfo"); } } } else if (memory) { AString errorDetailMsg; if (msg->findObject("cryptoInfos", &obj)) { buffer->meta()->setSize("ssize", size); buffer->meta()->setObject("cryptoInfos", obj); if (msg->findObject("accessUnitInfo", &obj)) { // the reference will be same here and // setBufferParams buffer->meta()->setObject("accessUnitInfo", obj); } err = mBufferChannel->attachEncryptedBuffers( memory, offset, buffer, (mFlags & kFlagIsSecure), &errorDetailMsg); } else { err = mBufferChannel->attachEncryptedBuffer( memory, (mFlags & kFlagIsSecure), key, iv, mode, pattern, offset, subSamples, numSubSamples, buffer, &errorDetailMsg); } if (err != OK && hasCryptoOrDescrambler() && (mFlags & kFlagUseCryptoAsync)) { // create error detail sp cryptoErrorInfo = new AMessage(); buildCryptoInfoAMessage(cryptoErrorInfo, CryptoAsync::kActionDecrypt); cryptoErrorInfo->setInt32("err", err); cryptoErrorInfo->setInt32("actionCode", ACTION_CODE_FATAL); cryptoErrorInfo->setString("errorDetail", errorDetailMsg); onCryptoError(cryptoErrorInfo); // we want cryptoError to be in the callback // but Codec IllegalStateException to be triggered. err = INVALID_OPERATION; } } else { mErrorLog.log(LOG_TAG, "Fatal error: invalid queue request without a buffer"); err = UNKNOWN_ERROR; } if (err == OK && !buffer->asC2Buffer() && c2Buffer && c2Buffer->data().type() == C2BufferData::LINEAR) { C2ConstLinearBlock block{c2Buffer->data().linearBlocks().front()}; if (block.size() > buffer->size()) { C2ConstLinearBlock leftover = block.subBlock( block.offset() + buffer->size(), block.size() - buffer->size()); sp>> obj{ new WrapperObject>{ C2Buffer::CreateLinearBuffer(leftover)}}; msg->setObject("c2buffer", obj); mLeftover.push_front(msg); // Not sending EOS if we have leftovers flags &= ~BUFFER_FLAG_EOS; } } offset = buffer->offset(); size = buffer->size(); if (err != OK) { ALOGE("block model buffer attach failed: err = %s (%d)", StrMediaError(err).c_str(), err); return err; } } if (offset + size > buffer->capacity()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "buffer offset and size goes beyond the capacity: " "offset=%zu, size=%zu, cap=%zu", offset, size, buffer->capacity())); return -EINVAL; } buffer->setRange(offset, size); status_t err = OK; err = setInputBufferParams(timeUs, flags); if (err != OK) { return -EINVAL; } int32_t usedMaxInputSize = mApiUsageMetrics.inputBufferSize.usedMax; mApiUsageMetrics.inputBufferSize.usedMax = size > usedMaxInputSize ? size : usedMaxInputSize; if (hasCryptoOrDescrambler() && !c2Buffer && !memory) { AString *errorDetailMsg; CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg)); // Notify mCrypto of video resolution changes if (mTunneled && mCrypto != NULL) { int32_t width, height; if (mInputFormat->findInt32("width", &width) && mInputFormat->findInt32("height", &height) && width > 0 && height > 0) { if (width != mTunneledInputWidth || height != mTunneledInputHeight) { mTunneledInputWidth = width; mTunneledInputHeight = height; mCrypto->notifyResolution(width, height); } } } if (mCryptoAsync) { // prepare a message and enqueue sp cryptoInfo = new AMessage(); buildCryptoInfoAMessage(cryptoInfo, CryptoAsync::kActionDecrypt); mCryptoAsync->decrypt(cryptoInfo); } else if (msg->findObject("cryptoInfos", &obj)) { buffer->meta()->setObject("cryptoInfos", obj); err = mBufferChannel->queueSecureInputBuffers( buffer, (mFlags & kFlagIsSecure), errorDetailMsg); } else { err = mBufferChannel->queueSecureInputBuffer( buffer, (mFlags & kFlagIsSecure), key, iv, mode, pattern, subSamples, numSubSamples, errorDetailMsg); } if (err != OK) { mediametrics_setInt32(mMetricsHandle, kCodecQueueSecureInputBufferError, err); ALOGW("Log queueSecureInputBuffer error: %d", err); } } else { err = mBufferChannel->queueInputBuffer(buffer); if (err != OK) { mediametrics_setInt32(mMetricsHandle, kCodecQueueInputBufferError, err); ALOGW("Log queueInputBuffer error: %d", err); } } if (err == OK) { if (mTunneled && (flags & (BUFFER_FLAG_DECODE_ONLY | BUFFER_FLAG_END_OF_STREAM)) == 0) { mVideoRenderQualityTracker.onTunnelFrameQueued(timeUs); } // synchronization boundary for getBufferAndFormat Mutex::Autolock al(mBufferLock); ALOGV("onQueueInputBuffer: mPortBuffers[in][%zu] NOT owned by client", index); info->mOwnedByClient = false; info->mData.clear(); statsBufferSent(timeUs, buffer); } return err; } status_t MediaCodec::handleLeftover(size_t index) { if (mLeftover.empty()) { return OK; } sp msg = mLeftover.front(); mLeftover.pop_front(); msg->setSize("index", index); ALOGV("handleLeftover(%zu)", index); return onQueueInputBuffer(msg); } template static size_t CreateFramesRenderedMessageInternal(const std::list &done, sp &msg) { size_t index = 0; for (typename std::list::const_iterator it = done.cbegin(); it != done.cend(); ++it) { if (it->getRenderTimeNs() < 0) { continue; // dropped frame from tracking } msg->setInt64(base::StringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); msg->setInt64(base::StringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); ++index; } return index; } //static size_t MediaCodec::CreateFramesRenderedMessage( const std::list &done, sp &msg) { return CreateFramesRenderedMessageInternal(done, msg); } //static size_t MediaCodec::CreateFramesRenderedMessage( const std::list &done, sp &msg) { return CreateFramesRenderedMessageInternal(done, msg); } status_t MediaCodec::onReleaseOutputBuffer(const sp &msg) { size_t index; CHECK(msg->findSize("index", &index)); int32_t render; if (!msg->findInt32("render", &render)) { render = 0; } if (!isExecuting()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "releaseOutputBuffer() is valid at Executing states; currently %s", apiStateString().c_str())); return -EINVAL; } if (index >= mPortBuffers[kPortIndexOutput].size()) { mErrorLog.log(LOG_TAG, base::StringPrintf( "index out of range (index=%zu)", mPortBuffers[kPortIndexOutput].size())); return -ERANGE; } BufferInfo *info = &mPortBuffers[kPortIndexOutput][index]; if (!info->mOwnedByClient) { mErrorLog.log(LOG_TAG, base::StringPrintf( "client does not own the buffer #%zu", index)); return -EACCES; } if (info->mData == nullptr) { mErrorLog.log(LOG_TAG, base::StringPrintf( "Fatal error: null buffer for index %zu", index)); return -EACCES; } // synchronization boundary for getBufferAndFormat sp buffer; { Mutex::Autolock al(mBufferLock); ALOGV("onReleaseOutputBuffer: mPortBuffers[out][%zu] NOT owned by client", index); info->mOwnedByClient = false; buffer = info->mData; info->mData.clear(); } if (render && buffer->size() != 0) { int64_t mediaTimeUs = INT64_MIN; buffer->meta()->findInt64("timeUs", &mediaTimeUs); bool noRenderTime = false; int64_t renderTimeNs = 0; if (!msg->findInt64("timestampNs", &renderTimeNs)) { // use media timestamp if client did not request a specific render timestamp ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs); renderTimeNs = mediaTimeUs * 1000; noRenderTime = true; } if (mSoftRenderer != NULL) { std::list doneFrames = mSoftRenderer->render( buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs, mPortBuffers[kPortIndexOutput].size(), buffer->format()); // if we are running, notify rendered frames if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { sp notify = mOnFrameRenderedNotification->dup(); sp data = new AMessage; if (CreateFramesRenderedMessage(doneFrames, data)) { notify->setMessage("data", data); notify->post(); } } } // If rendering to the screen, then schedule a time in the future to poll to see if this // frame was ever rendered to seed onFrameRendered callbacks. if (mAreRenderMetricsEnabled && mIsSurfaceToDisplay) { if (mediaTimeUs != INT64_MIN) { noRenderTime ? mVideoRenderQualityTracker.onFrameReleased(mediaTimeUs) : mVideoRenderQualityTracker.onFrameReleased(mediaTimeUs, renderTimeNs); } // can't initialize this in the constructor because the Looper parent class needs to be // initialized first if (mMsgPollForRenderedBuffers == nullptr) { mMsgPollForRenderedBuffers = new AMessage(kWhatPollForRenderedBuffers, this); } // Schedule the poll to occur 100ms after the render time - should be safe for // determining if the frame was ever rendered. If no render time was specified, the // presentation timestamp is used instead, which almost certainly occurs in the past, // since it's almost always a zero-based offset from the start of the stream. In these // scenarios, we expect the frame to be rendered with no delay. int64_t nowUs = ALooper::GetNowUs(); int64_t renderTimeUs = renderTimeNs / 1000; int64_t delayUs = renderTimeUs < nowUs ? 0 : renderTimeUs - nowUs; delayUs += 100 * 1000; /* 100ms in microseconds */ status_t err = mMsgPollForRenderedBuffers->postUnique(/* token= */ mMsgPollForRenderedBuffers, delayUs); if (err != OK) { ALOGE("unexpected failure to post pollForRenderedBuffers: %d", err); } } status_t err = mBufferChannel->renderOutputBuffer(buffer, renderTimeNs); if (err == NO_INIT) { mErrorLog.log(LOG_TAG, "rendering to non-initialized(obsolete) surface"); return err; } if (err != OK) { ALOGI("rendring output error %d", err); } } else { if (mIsSurfaceToDisplay && buffer->size() != 0) { int64_t mediaTimeUs = INT64_MIN; if (buffer->meta()->findInt64("timeUs", &mediaTimeUs)) { mVideoRenderQualityTracker.onFrameSkipped(mediaTimeUs); } } mBufferChannel->discardBuffer(buffer); } return OK; } MediaCodec::BufferInfo *MediaCodec::peekNextPortBuffer(int32_t portIndex) { CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); std::list *availBuffers = &mAvailPortBuffers[portIndex]; if (availBuffers->empty()) { return nullptr; } return &mPortBuffers[portIndex][*availBuffers->begin()]; } ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) { CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); BufferInfo *info = peekNextPortBuffer(portIndex); if (!info) { return -EAGAIN; } std::list *availBuffers = &mAvailPortBuffers[portIndex]; size_t index = *availBuffers->begin(); CHECK_EQ(info, &mPortBuffers[portIndex][index]); availBuffers->erase(availBuffers->begin()); { Mutex::Autolock al(mBufferLock); ALOGV("dequeuePortBuffer: mPortBuffers[%s][%zu] checking if not owned by client", portIndex == kPortIndexInput ? "in" : "out", index); CHECK(!info->mOwnedByClient); info->mOwnedByClient = true; // set image-data if (info->mData->format() != NULL) { sp imageData; if (info->mData->format()->findBuffer("image-data", &imageData)) { info->mData->meta()->setBuffer("image-data", imageData); } int32_t left, top, right, bottom; if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) { info->mData->meta()->setRect("crop-rect", left, top, right, bottom); } } } return index; } sp MediaCodec::getOrCreateDetachedSurface() { if (mDomain != DOMAIN_VIDEO || (mFlags & kFlagIsEncoder)) { return nullptr; } if (!mDetachedSurface) { uint64_t usage = 0; if (!mSurface || mSurface->getConsumerUsage(&usage) != OK) { // By default prepare buffer to be displayed on any of the common surfaces usage = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER); } mDetachedSurface.reset(new ReleaseSurface(usage)); } return mDetachedSurface->getSurface(); } status_t MediaCodec::connectToSurface(const sp &surface, uint32_t *generation) { status_t err = OK; if (surface != NULL) { uint64_t oldId, newId; if (mSurface != NULL && surface->getUniqueId(&newId) == NO_ERROR && mSurface->getUniqueId(&oldId) == NO_ERROR && newId == oldId) { ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str()); return ALREADY_EXISTS; } // in case we don't connect, ensure that we don't signal the surface is // connected to the screen mIsSurfaceToDisplay = false; err = nativeWindowConnect(surface.get(), "connectToSurface"); if (err == OK) { // Require a fresh set of buffers after each connect by using a unique generation // number. Rely on the fact that max supported process id by Linux is 2^22. // PID is never 0 so we don't have to worry that we use the default generation of 0. // TODO: come up with a unique scheme if other producers also set the generation number. static uint32_t sSurfaceGeneration = 0; *generation = (getpid() << 10) | (++sSurfaceGeneration & ((1 << 10) - 1)); surface->setGenerationNumber(*generation); ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), *generation); // HACK: clear any free buffers. Remove when connect will automatically do this. // This is needed as the consumer may be holding onto stale frames that it can reattach // to this surface after disconnect/connect, and those free frames would inherit the new // generation number. Disconnecting after setting a unique generation prevents this. nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)"); sp listener = new OnBufferReleasedListener(*generation, mBufferChannel); err = surfaceConnectWithListener( surface, listener, "connectToSurface(reconnect-with-listener)"); } if (err != OK) { *generation = 0; ALOGE("nativeWindowConnect/surfaceConnectWithListener returned an error: %s (%d)", strerror(-err), err); } else { if (!mAllowFrameDroppingBySurface) { disableLegacyBufferDropPostQ(surface); } // keep track whether or not the buffers of the connected surface go to the screen int result = 0; surface->query(NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER, &result); mIsSurfaceToDisplay = result != 0; } } // do not return ALREADY_EXISTS unless surfaces are the same return err == ALREADY_EXISTS ? BAD_VALUE : err; } status_t MediaCodec::disconnectFromSurface() { status_t err = OK; if (mSurface != NULL) { // Resetting generation is not technically needed, but there is no need to keep it either mSurface->setGenerationNumber(0); err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface"); if (err != OK) { ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err); } // assume disconnected even on error mSurface.clear(); mSurfaceGeneration = 0; mIsSurfaceToDisplay = false; } return err; } status_t MediaCodec::handleSetSurface(const sp &surface, bool callCodec, bool onShutDown) { uint32_t generation; status_t err = OK; if (surface != nullptr) { err = connectToSurface(surface, &generation); if (err == ALREADY_EXISTS) { // reconnecting to same surface return OK; } if (err == OK && callCodec) { if (mFlags & kFlagUsesSoftwareRenderer) { if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { pushBlankBuffersToNativeWindow(mSurface.get()); } // do not create a new software renderer on shutdown (release) // as it will not be used anyway if (!onShutDown) { surface->setDequeueTimeout(-1); mSoftRenderer = new SoftwareRenderer(surface); // TODO: check if this was successful } } else { err = mCodec->setSurface(surface, generation); } mReliabilityContextMetrics.setOutputSurfaceCount++; } } if (err == OK) { if (mSurface != NULL) { (void)disconnectFromSurface(); } if (surface != NULL) { mSurface = surface; mSurfaceGeneration = generation; } } return err; } status_t MediaCodec::handleSetSurface(const sp &surface) { if (android::media::codec::provider_->null_output_surface_support()) { return handleSetSurface(surface, false /* callCodec */); } status_t err = OK; if (mSurface != NULL) { (void)disconnectFromSurface(); } if (surface != NULL) { uint32_t generation; err = connectToSurface(surface, &generation); if (err == OK) { mSurface = surface; mSurfaceGeneration = generation; } } return err; } void MediaCodec::onInputBufferAvailable() { int32_t index; while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) { sp msg = mCallback->dup(); msg->setInt32("callbackID", CB_INPUT_AVAILABLE); msg->setInt32("index", index); msg->post(); } } void MediaCodec::onOutputBufferAvailable() { int32_t index; while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) { if (discardDecodeOnlyOutputBuffer(index)) { continue; } sp msg = mCallback->dup(); const sp &buffer = mPortBuffers[kPortIndexOutput][index].mData; int32_t outputCallbackID = CB_OUTPUT_AVAILABLE; sp accessUnitInfoObj; msg->setInt32("index", index); msg->setSize("offset", buffer->offset()); msg->setSize("size", buffer->size()); int64_t timeUs; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); msg->setInt64("timeUs", timeUs); int32_t flags; CHECK(buffer->meta()->findInt32("flags", &flags)); msg->setInt32("flags", flags); buffer->meta()->findObject("accessUnitInfo", &accessUnitInfoObj); if (accessUnitInfoObj) { outputCallbackID = CB_LARGE_FRAME_OUTPUT_AVAILABLE; msg->setObject("accessUnitInfo", accessUnitInfoObj); sp auInfo( (decltype(auInfo.get()))accessUnitInfoObj.get()); auInfo->value.back().mFlags |= flags & BUFFER_FLAG_END_OF_STREAM; } msg->setInt32("callbackID", outputCallbackID); statsBufferReceived(timeUs, buffer); msg->post(); } } void MediaCodec::onCryptoError(const sp & msg) { if (mCallback != NULL) { sp cb_msg = mCallback->dup(); cb_msg->setInt32("callbackID", CB_CRYPTO_ERROR); cb_msg->extend(msg); cb_msg->post(); } } void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) { if (mCallback != NULL) { sp msg = mCallback->dup(); msg->setInt32("callbackID", CB_ERROR); msg->setInt32("err", err); msg->setInt32("actionCode", actionCode); if (detail != NULL) { msg->setString("detail", detail); } msg->post(); } } void MediaCodec::onOutputFormatChanged() { if (mCallback != NULL) { sp msg = mCallback->dup(); msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED); msg->setMessage("format", mOutputFormat); msg->post(); } } void MediaCodec::onRequiredResourcesChanged( const std::vector& resourceInfo) { mRequiredResourceInfo = resourceInfo; // Make sure codec availability feature is on. if (mCallback != nullptr && android::media::codec::codec_availability()) { // Post the callback sp msg = mCallback->dup(); msg->setInt32("callbackID", CB_REQUIRED_RESOURCES_CHANGED); msg->post(); } } void MediaCodec::postActivityNotificationIfPossible() { if (mActivityNotify == NULL) { return; } bool isErrorOrOutputChanged = (mFlags & (kFlagStickyError | kFlagOutputBuffersChanged | kFlagOutputFormatChanged)); if (isErrorOrOutputChanged || !mAvailPortBuffers[kPortIndexInput].empty() || !mAvailPortBuffers[kPortIndexOutput].empty()) { mActivityNotify->setInt32("input-buffers", mAvailPortBuffers[kPortIndexInput].size()); if (isErrorOrOutputChanged) { // we want consumer to dequeue as many times as it can mActivityNotify->setInt32("output-buffers", INT32_MAX); } else { mActivityNotify->setInt32("output-buffers", mAvailPortBuffers[kPortIndexOutput].size()); } mActivityNotify->post(); mActivityNotify.clear(); } } status_t MediaCodec::setParameters(const sp ¶ms) { sp msg = new AMessage(kWhatSetParameters, this); msg->setMessage("params", params); sp response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::onSetParameters(const sp ¶ms) { if (mState == UNINITIALIZED || mState == INITIALIZING) { return NO_INIT; } updateLowLatency(params); updateCodecImportance(params); mapFormat(mComponentName, params, nullptr, false); updateTunnelPeek(params); mCodec->signalSetParameters(params); return OK; } status_t MediaCodec::amendOutputFormatWithCodecSpecificData( const sp &buffer) { AString mime; CHECK(mOutputFormat->findString("mime", &mime)); if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { // Codec specific data should be SPS and PPS in a single buffer, // each prefixed by a startcode (0x00 0x00 0x00 0x01). // We separate the two and put them into the output format // under the keys "csd-0" and "csd-1". unsigned csdIndex = 0; const uint8_t *data = buffer->data(); size_t size = buffer->size(); const uint8_t *nalStart; size_t nalSize; while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { sp csd = new ABuffer(nalSize + 4); memcpy(csd->data(), "\x00\x00\x00\x01", 4); memcpy(csd->data() + 4, nalStart, nalSize); mOutputFormat->setBuffer( base::StringPrintf("csd-%u", csdIndex).c_str(), csd); ++csdIndex; } if (csdIndex != 2) { mErrorLog.log(LOG_TAG, base::StringPrintf( "codec config data contains %u NAL units; expected 2.", csdIndex)); return ERROR_MALFORMED; } } else { // For everything else we just stash the codec specific data into // the output format as a single piece of csd under "csd-0". sp csd = new ABuffer(buffer->size()); memcpy(csd->data(), buffer->data(), buffer->size()); csd->setRange(0, buffer->size()); mOutputFormat->setBuffer("csd-0", csd); } return OK; } void MediaCodec::postPendingRepliesAndDeferredMessages( std::string origin, status_t err /* = OK */) { sp response{new AMessage}; if (err != OK) { response->setInt32("err", err); } postPendingRepliesAndDeferredMessages(origin, response); } void MediaCodec::postPendingRepliesAndDeferredMessages( std::string origin, const sp &response) { LOG_ALWAYS_FATAL_IF( !mReplyID, "postPendingRepliesAndDeferredMessages: mReplyID == null, from %s following %s", origin.c_str(), mLastReplyOrigin.c_str()); mLastReplyOrigin = origin; response->postReply(mReplyID); mReplyID.clear(); ALOGV_IF(!mDeferredMessages.empty(), "posting %zu deferred messages", mDeferredMessages.size()); for (sp msg : mDeferredMessages) { msg->post(); } mDeferredMessages.clear(); } std::string MediaCodec::apiStateString() { const char *rval = NULL; char rawbuffer[16]; // room for "%d" switch (mState) { case UNINITIALIZED: rval = (mFlags & kFlagStickyError) ? "at Error state" : "at Released state"; break; case INITIALIZING: rval = "while constructing"; break; case INITIALIZED: rval = "at Uninitialized state"; break; case CONFIGURING: rval = "during configure()"; break; case CONFIGURED: rval = "at Configured state"; break; case STARTING: rval = "during start()"; break; case STARTED: rval = "at Running state"; break; case FLUSHING: rval = "during flush()"; break; case FLUSHED: rval = "at Flushed state"; break; case STOPPING: rval = "during stop()"; break; case RELEASING: rval = "during release()"; break; default: snprintf(rawbuffer, sizeof(rawbuffer), "at %d", mState); rval = rawbuffer; break; } return rval; } std::string MediaCodec::stateString(State state) { const char *rval = NULL; char rawbuffer[16]; // room for "%d" switch (state) { case UNINITIALIZED: rval = "UNINITIALIZED"; break; case INITIALIZING: rval = "INITIALIZING"; break; case INITIALIZED: rval = "INITIALIZED"; break; case CONFIGURING: rval = "CONFIGURING"; break; case CONFIGURED: rval = "CONFIGURED"; break; case STARTING: rval = "STARTING"; break; case STARTED: rval = "STARTED"; break; case FLUSHING: rval = "FLUSHING"; break; case FLUSHED: rval = "FLUSHED"; break; case STOPPING: rval = "STOPPING"; break; case RELEASING: rval = "RELEASING"; break; default: snprintf(rawbuffer, sizeof(rawbuffer), "%d", state); rval = rawbuffer; break; } return rval; } // static status_t MediaCodec::CanFetchLinearBlock( const std::vector &names, bool *isCompatible) { *isCompatible = false; if (names.size() == 0) { *isCompatible = true; return OK; } const CodecListCache &cache = GetCodecListCache(); for (const std::string &name : names) { auto it = cache.mCodecInfoMap.find(name); if (it == cache.mCodecInfoMap.end()) { return NAME_NOT_FOUND; } const char *owner = it->second->getOwnerName(); if (owner == nullptr || strncmp(owner, "default", 8) == 0) { *isCompatible = false; return OK; } else if (strncmp(owner, "codec2::", 8) != 0) { return NAME_NOT_FOUND; } } return CCodec::CanFetchLinearBlock(names, kDefaultReadWriteUsage, isCompatible); } // static std::shared_ptr MediaCodec::FetchLinearBlock( size_t capacity, const std::vector &names) { return CCodec::FetchLinearBlock(capacity, kDefaultReadWriteUsage, names); } // static status_t MediaCodec::CanFetchGraphicBlock( const std::vector &names, bool *isCompatible) { *isCompatible = false; if (names.size() == 0) { *isCompatible = true; return OK; } const CodecListCache &cache = GetCodecListCache(); for (const std::string &name : names) { auto it = cache.mCodecInfoMap.find(name); if (it == cache.mCodecInfoMap.end()) { return NAME_NOT_FOUND; } const char *owner = it->second->getOwnerName(); if (owner == nullptr || strncmp(owner, "default", 8) == 0) { *isCompatible = false; return OK; } else if (strncmp(owner, "codec2.", 7) != 0) { return NAME_NOT_FOUND; } } return CCodec::CanFetchGraphicBlock(names, isCompatible); } // static std::shared_ptr MediaCodec::FetchGraphicBlock( int32_t width, int32_t height, int32_t format, uint64_t usage, const std::vector &names) { return CCodec::FetchGraphicBlock(width, height, format, usage, names); } } // namespace android