[{"id":16878,"web_url":"https://patchwork.libcamera.org/comment/16878/","msgid":"","date":"2021-05-11T04:24:59","subject":"Re: [libcamera-devel] [PATCH 1/8] android: Rework request\n\tcompletion notification","submitter":{"id":63,"url":"https://patchwork.libcamera.org/api/people/63/","name":"Hirokazu Honda","email":"hiroh@chromium.org"},"content":"Hi Jacopo, thank you for the patch.\n\n\nOn Mon, May 10, 2021 at 7:51 PM Jacopo Mondi wrote:\n\n> The current implementation of CameraDevice::requestComplete() which\n> handles event notification and calls the framework capture result\n> callback does not handle error notification precisely enough.\n>\n> In detail:\n> - Error notification is an asynchronous callback that has to be notified\n> to the framework as soon as an error condition is detected, and it\n> independent from the process_capture_result() callback\n>\n> - Error notification requires the HAL to report the precise error cause,\n> by specifying the correct CAMERA3_MSG_ERROR_* error code.\n>\n> The current implementation only notifies errors of type\n> CAMERA3_MSG_ERROR_REQUEST at the end of the procedure, before the\n> callback invocation.\n>\n> Rework the procedure to:\n>\n> - Notify CAMERA3_MSG_ERROR_DEVICE and perform library tear-down in case\n> a Fatal error is detected\n>\n> - Notify CAMERA3_MSG_ERROR_REQUEST if the libcamera::Request::status is\n> different than RequestCompleted and immediately call\n> process_capture_result() with all buffers in error state.\n>\n> - Notify the shutter event as soon as possible\n>\n> - Notify CAMERA3_MSG_ERROR_RESULT in case the metadata cannot be\n> generated correctly and call process_capture_result() with the right\n> buffer state regardless of metadata availability.\n>\n> - Notify CAMERA3_MSG_ERROR_BUFFER for buffers whose post-processing\n> failed\n>\n> While at it, return the CameraStream buffer by calling\n> cameraStream->putBuffer() regardless of the post-processing result.\n>\n> No regression detected when running CTS in LIMITED mode.\n>\n> Signed-off-by: Jacopo Mondi \n>\n\nReviewed-by: Hirokazu Honda \n\n\n> ---\n> src/android/camera_device.cpp | 138 +++++++++++++++++++++-------------\n> src/android/camera_device.h | 3 +-\n> 2 files changed, 86 insertions(+), 55 deletions(-)\n>\n> diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\n> index f1f26c775611..d8b7e62ef896 100644\n> --- a/src/android/camera_device.cpp\n> +++ b/src/android/camera_device.cpp\n> @@ -2025,17 +2025,20 @@ int\n> CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n>\n> void CameraDevice::requestComplete(Request *request)\n> {\n> - camera3_buffer_status status = CAMERA3_BUFFER_STATUS_OK;\n> - std::unique_ptr resultMetadata;\n> -\n> decltype(descriptors_)::node_type node;\n> {\n> std::scoped_lock lock(mutex_);\n> auto it = descriptors_.find(request->cookie());\n> if (it == descriptors_.end()) {\n> + /*\n> + * \\todo Clarify if the Camera has to be closed on\n> + * ERROR_DEVICE and possibly demote the Fatal to\n> simple\n> + * Error.\n> + */\n> + notifyError(0, nullptr, CAMERA3_MSG_ERROR_DEVICE);\n> LOG(HAL, Fatal)\n> << \"Unknown request: \" <<\n> request->cookie();\n> - status = CAMERA3_BUFFER_STATUS_ERROR;\n> +\n> return;\n> }\n>\n> @@ -2043,16 +2046,72 @@ void CameraDevice::requestComplete(Request\n> *request)\n> }\n> Camera3RequestDescriptor &descriptor = node.mapped();\n>\n> + /*\n> + * Prepare the capture result for the Android camera stack.\n> + *\n> + * The buffer status is set to OK and later changed to ERROR if\n> + * post-processing/compression fails.\n> + */\n> + camera3_capture_result_t captureResult = {};\n> + captureResult.frame_number = descriptor.frameNumber_;\n> + captureResult.num_output_buffers = descriptor.buffers_.size();\n> + for (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n> + buffer.acquire_fence = -1;\n> + buffer.release_fence = -1;\n> + buffer.status = CAMERA3_BUFFER_STATUS_OK;\n> + }\n> + captureResult.output_buffers = descriptor.buffers_.data();\n> + captureResult.partial_result = 1;\n> +\n> + /*\n> + * If the Request has failed, abort the request by notifying the\n> error\n> + * and complete the request with all buffers in error state.\n> + */\n> if (request->status() != Request::RequestComplete) {\n> - LOG(HAL, Error) << \"Request not successfully completed: \"\n> + LOG(HAL, Error) << \"Request \" << request->cookie()\n> + << \" not successfully completed: \"\n> << request->status();\n> - status = CAMERA3_BUFFER_STATUS_ERROR;\n> +\n> + notifyError(descriptor.frameNumber_,\n> + descriptor.buffers_[0].stream,\n> + CAMERA3_MSG_ERROR_REQUEST);\n> +\n> + captureResult.partial_result = 0;\n> + for (camera3_stream_buffer_t &buffer : descriptor.buffers_)\n> + buffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n> + callbacks_->process_capture_result(callbacks_,\n> &captureResult);\n> +\n> + return;\n> }\n>\n> + /*\n> + * Notify shutter as soon as we have verified we have a valid\n> request.\n> + *\n> + * \\todo The shutter event notification should be sent to the\n> framework\n> + * as soon as possible, earlier than request completion time.\n> + */\n> + uint64_t sensorTimestamp =\n> static_cast(request->metadata()\n> +\n> .get(controls::SensorTimestamp));\n> + notifyShutter(descriptor.frameNumber_, sensorTimestamp);\n> +\n> LOG(HAL, Debug) << \"Request \" << request->cookie() << \" completed\n> with \"\n> << descriptor.buffers_.size() << \" streams\";\n>\n> - resultMetadata = getResultMetadata(descriptor);\n> + /*\n> + * Generate the metadata associated with the captured buffers.\n> + *\n> + * Notify if the metadata generation has failed, but continue\n> processing\n> + * buffers and return an empty metadata pack.\n> + */\n> + std::unique_ptr resultMetadata =\n> getResultMetadata(descriptor);\n> + if (!resultMetadata) {\n> + notifyError(descriptor.frameNumber_,\n> descriptor.buffers_[0].stream,\n> + CAMERA3_MSG_ERROR_RESULT);\n> +\n> + /* The camera framework expects an empy metadata pack on\n> error. */\n> + resultMetadata = std::make_unique(0, 0);\n> + }\n> + captureResult.result = resultMetadata->get();\n>\n> /* Handle any JPEG compression. */\n> for (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n> @@ -2065,56 +2124,27 @@ void CameraDevice::requestComplete(Request\n> *request)\n> FrameBuffer *src =\n> request->findBuffer(cameraStream->stream());\n> if (!src) {\n> LOG(HAL, Error) << \"Failed to find a source stream\n> buffer\";\n> + buffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n> + notifyError(descriptor.frameNumber_, buffer.stream,\n> + CAMERA3_MSG_ERROR_BUFFER);\n> continue;\n> }\n>\n> - int ret = cameraStream->process(*src,\n> - *buffer.buffer,\n> + int ret = cameraStream->process(*src, *buffer.buffer,\n> descriptor.settings_,\n> resultMetadata.get());\n> - if (ret) {\n> - status = CAMERA3_BUFFER_STATUS_ERROR;\n> - continue;\n> - }\n> -\n> /*\n> * Return the FrameBuffer to the CameraStream now that\n> we're\n> * done processing it.\n> */\n> if (cameraStream->type() == CameraStream::Type::Internal)\n> cameraStream->putBuffer(src);\n> - }\n>\n> - /* Prepare to call back the Android camera stack. */\n> - camera3_capture_result_t captureResult = {};\n> - captureResult.frame_number = descriptor.frameNumber_;\n> - captureResult.num_output_buffers = descriptor.buffers_.size();\n> - for (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n> - buffer.acquire_fence = -1;\n> - buffer.release_fence = -1;\n> - buffer.status = status;\n> - }\n> - captureResult.output_buffers = descriptor.buffers_.data();\n> -\n> - if (status == CAMERA3_BUFFER_STATUS_OK) {\n> - uint64_t timestamp =\n> - static_cast(request->metadata()\n> -\n> .get(controls::SensorTimestamp));\n> - notifyShutter(descriptor.frameNumber_, timestamp);\n> -\n> - captureResult.partial_result = 1;\n> - captureResult.result = resultMetadata->get();\n> - }\n> -\n> - if (status == CAMERA3_BUFFER_STATUS_ERROR ||\n> !captureResult.result) {\n> - /* \\todo Improve error handling. In case we notify an error\n> - * because the metadata generation fails, a shutter event\n> has\n> - * already been notified for this frame number before the\n> error\n> - * is here signalled. Make sure the error path plays well\n> with\n> - * the camera stack state machine.\n> - */\n> - notifyError(descriptor.frameNumber_,\n> - descriptor.buffers_[0].stream);\n> + if (ret) {\n> + buffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n> + notifyError(descriptor.frameNumber_, buffer.stream,\n> + CAMERA3_MSG_ERROR_BUFFER);\n> + }\n> }\n>\n> callbacks_->process_capture_result(callbacks_, &captureResult);\n> @@ -2136,23 +2166,23 @@ void CameraDevice::notifyShutter(uint32_t\n> frameNumber, uint64_t timestamp)\n> callbacks_->notify(callbacks_, ¬ify);\n> }\n>\n> -void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t\n> *stream)\n> +void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t\n> *stream,\n> + camera3_error_msg_code code)\n> {\n> camera3_notify_msg_t notify = {};\n>\n> - /*\n> - * \\todo Report and identify the stream number or configuration to\n> - * clarify the stream that failed.\n> - */\n> - LOG(HAL, Error) << \"Error occurred on frame \" << frameNumber << \"\n> (\"\n> - << toPixelFormat(stream->format).toString() << \")\";\n> -\n> notify.type = CAMERA3_MSG_ERROR;\n> notify.message.error.error_stream = stream;\n> notify.message.error.frame_number = frameNumber;\n> - notify.message.error.error_code = CAMERA3_MSG_ERROR_REQUEST;\n> + notify.message.error.error_code = code;\n>\n> callbacks_->notify(callbacks_, ¬ify);\n> +\n> + if (stream)\n> + LOG(HAL, Error) << \"Error occurred on frame \" <<\n> frameNumber << \" (\"\n> + <<\n> toPixelFormat(stream->format).toString() << \")\";\n> + else\n> + LOG(HAL, Error) << \"Fatal error occurred on device\";\n> }\n>\n> /*\n> diff --git a/src/android/camera_device.h b/src/android/camera_device.h\n> index 23457e47767a..8d5da8bc59e1 100644\n> --- a/src/android/camera_device.h\n> +++ b/src/android/camera_device.h\n> @@ -101,7 +101,8 @@ private:\n> std::tuple calculateStaticMetadataSize();\n> libcamera::FrameBuffer *createFrameBuffer(const buffer_handle_t\n> camera3buffer);\n> void notifyShutter(uint32_t frameNumber, uint64_t timestamp);\n> - void notifyError(uint32_t frameNumber, camera3_stream_t *stream);\n> + void notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n> + camera3_error_msg_code code);\n> std::unique_ptr requestTemplatePreview();\n> std::unique_ptr requestTemplateVideo();\n> libcamera::PixelFormat toPixelFormat(int format) const;\n> --\n> 2.31.1\n>\n>","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 02D8DBF829\n\tfor ;\n\tTue, 11 May 2021 04:25:13 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 508AF6891F;\n\tTue, 11 May 2021 06:25:12 +0200 (CEST)","from mail-ej1-x631.google.com (mail-ej1-x631.google.com\n\t[IPv6:2a00:1450:4864:20::631])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id D7D4561538\n\tfor ;\n\tTue, 11 May 2021 06:25:10 +0200 (CEST)","by mail-ej1-x631.google.com with SMTP id n2so27723021ejy.7\n\tfor ;\n\tMon, 10 May 2021 21:25:10 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=chromium.org header.i=@chromium.org\n\theader.b=\"oBvBiD/g\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed; d=chromium.org;\n\ts=google; \n\th=mime-version:references:in-reply-to:from:date:message-id:subject:to\n\t:cc; bh=n+ej3LtHczZfK0ArWV9WNrnDGsxfUvkEFI2yxSnl10Y=;\n\tb=oBvBiD/g04Qjn0HWKnoEMCtxthP6VCP4ZF+zWJAPUoaVaUWBYYFyq1NMBdm27sKoZ5\n\tDOKgxUBCvQa74l8WotVgcyk/j/8GKr9//k6Hi0d2zOuZZMqhYpacP+nXrnuB8oQtEdXp\n\tiSASXHjgDZQXtMOJbJRccOV0pVzvSay5Qkmcg=","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:references:in-reply-to:from:date\n\t:message-id:subject:to:cc;\n\tbh=n+ej3LtHczZfK0ArWV9WNrnDGsxfUvkEFI2yxSnl10Y=;\n\tb=GkAEUJyh5K0ssyVCI/ZVJE8nWUyTtwJJ8Uir/mrwJSapVoJvFVSrQyywgDfnP0gq8p\n\tAuQZMu7jDAHer+S87cW9v0h3poxbOSSIm11pqSusvCIKivSsKmz2ld1y+TIQG3x8GmDH\n\tHxSEigfRGv4USaDtPc7Ve5bfoCDA7aNcuCOJBi+0MSYVGe1+1Ovuot8Y1cvyJDPZyIvX\n\tZDV4nh1mhEDIL9UokPhfFgfMSH52sEmRWj99fh5+4EKFJP2Pzoz3Lb70UJzW8eo8DZs/\n\taL2jNg+rd5Na3X4H64n2XYot2MTylMCKr0pZ1lvWXDh7D6Xgl4Toimcai4WojMmGJ5bI\n\tSFXQ==","X-Gm-Message-State":"AOAM530QP35Ga1bBAwZQXx+N+ec0/hRYSvqwKUgNrCLZNySrFuJju2zi\n\tuYzm8xltZnlIg2K94GdCqfVeVYGEkEgz2pf60l1Krzc25Cs=","X-Google-Smtp-Source":"ABdhPJx2R1bTqdo6L9Li0sWlwhABywu150jzqa2EYX8uq45IcjHXEc/6UpYd9DmywkH9CSwZuh1NUJCfAbGW0kIiM9A=","X-Received":"by 2002:a17:907:209b:: with SMTP id\n\tpv27mr29510148ejb.475.1620707110478; \n\tMon, 10 May 2021 21:25:10 -0700 (PDT)","MIME-Version":"1.0","References":"<20210510105235.28319-1-jacopo@jmondi.org>\n\t<20210510105235.28319-2-jacopo@jmondi.org>","In-Reply-To":"<20210510105235.28319-2-jacopo@jmondi.org>","From":"Hirokazu Honda ","Date":"Tue, 11 May 2021 13:24:59 +0900","Message-ID":"","To":"Jacopo Mondi ","Subject":"Re: [libcamera-devel] [PATCH 1/8] android: Rework request\n\tcompletion notification","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera devel ","Content-Type":"multipart/mixed;\n\tboundary=\"===============5795574184583036882==\"","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]