Patch Detail
Show a patch.
GET /api/1.1/patches/12208/?format=api
{ "id": 12208, "url": "https://patchwork.libcamera.org/api/1.1/patches/12208/?format=api", "web_url": "https://patchwork.libcamera.org/patch/12208/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20210506150106.28470-1-jacopo@jmondi.org>", "date": "2021-05-06T15:01:06", "name": "[libcamera-devel,v2] android: Rework request completion notification", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "0f82bbd2b008567913d27f7381b8522beeeab23b", "submitter": { "id": 3, "url": "https://patchwork.libcamera.org/api/1.1/people/3/?format=api", "name": "Jacopo Mondi", "email": "jacopo@jmondi.org" }, "delegate": { "id": 15, "url": "https://patchwork.libcamera.org/api/1.1/users/15/?format=api", "username": "jmondi", "first_name": "Jacopo", "last_name": "Mondi", "email": "jacopo@jmondi.org" }, "mbox": "https://patchwork.libcamera.org/patch/12208/mbox/", "series": [ { "id": 2004, "url": "https://patchwork.libcamera.org/api/1.1/series/2004/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2004", "date": "2021-05-06T15:01:06", "name": "[libcamera-devel,v2] android: Rework request completion notification", "version": 2, "mbox": "https://patchwork.libcamera.org/series/2004/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/12208/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/12208/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 51502BDE7F\n\tfor <parsemail@patchwork.libcamera.org>;\n\tThu, 6 May 2021 15:00:29 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id A00E96891A;\n\tThu, 6 May 2021 17:00:28 +0200 (CEST)", "from relay11.mail.gandi.net (relay11.mail.gandi.net\n\t[217.70.178.231])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id DB758602BA\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tThu, 6 May 2021 17:00:27 +0200 (CEST)", "from uno.LocalDomain (93-61-96-190.ip145.fastwebnet.it\n\t[93.61.96.190]) (Authenticated sender: jacopo@jmondi.org)\n\tby relay11.mail.gandi.net (Postfix) with ESMTPSA id 52596100018;\n\tThu, 6 May 2021 15:00:26 +0000 (UTC)" ], "From": "Jacopo Mondi <jacopo@jmondi.org>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Thu, 6 May 2021 17:01:06 +0200", "Message-Id": "<20210506150106.28470-1-jacopo@jmondi.org>", "X-Mailer": "git-send-email 2.31.1", "MIME-Version": "1.0", "Subject": "[libcamera-devel] [PATCH v2] android: Rework request completion\n\tnotification", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Content-Type": "text/plain; charset=\"us-ascii\"", "Content-Transfer-Encoding": "7bit", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "The current implementation of CameraDevice::requestComplete() which\nhandles event notification and calls the framework capture result\ncallback does not handle error notification precisely enough.\n\nIn detail:\n- Error notification is an asynchronous callback that has to be notified\n to the framework as soon as an error condition is detected, and it\n independent from the process_capture_result() callback\n\n- Error notification requires the HAL to report the precise error cause,\n by specifying the correct CAMERA3_MSG_ERROR_* error code.\n\nThe current implementation only notifies errors of type\nCAMERA3_MSG_ERROR_REQUEST at the end of the procedure, before the\ncallback invocation.\n\nRework the procedure to:\n\n- Notify CAMERA3_MSG_ERROR_DEVICE and close the camera in case a Fatal\n error is detected (Fatal does not perform library teardown in\n production builds)\n\n- Notify CAMERA3_MSG_ERROR_REQUEST if the libcamera::Request::status is\n different than RequestCompleted and immediately call\n process_capture_result() with all buffers in error state.\n\n- Notify the shutter event as soon as possible\n\n- Notify CAMERA3_MSG_ERROR_RESULT in case the metadata cannot be\n generated correctly and call process_capture_result() with the right\n buffer state regardless of metadata availability.\n\n- Notify CAMERA3_MSG_ERROR_BUFFER for buffers whose post-processing\n failed\n\nWhile at it, return the CameraStream buffer by calling\ncameraStream->putBuffer() regardless of the post-processing result.\n\nNo regression detected when running CTS in LIMITED mode.\n\nSigned-off-by: Jacopo Mondi <jacopo@jmondi.org>\n---\n\nv2:\n- Rework a comment as suggested by Niklas\n- As Niklas reported, dereferencing a unique_ptr<> which owns a nullptr has\n an undefined behaviour. Replace that pattern by resetting the resultMetadata\n unique_ptr<> with a new one holding an empty CameraMetadata, as the framework\n requires on error.\n\n camera3.h:\n * If there was an error producing the result metadata, result must be an\n * empty metadata buffer, and notify() must be called with ERROR_RESULT.\n\n\n std::unique_ptr<CameraMetadata> resultMetadata = getResultMetadata(descriptor);\n if (!resultMetadata) {\n \t\tnotifyError(descriptor.frameNumber_, descriptor.buffers_[0].stream,\n \t\t\t CAMERA3_MSG_ERROR_RESULT);\n\n \t\t/* The camera framework expects an empy metadata pack on error. */\n \t\tresultMetadata = std::make_unique<CameraMetadata>(0, 0);\n }\n captureResult.result = resultMetadata->get();\n\n---\n src/android/camera_device.cpp | 132 ++++++++++++++++++++--------------\n src/android/camera_device.h | 3 +-\n 2 files changed, 80 insertions(+), 55 deletions(-)\n\n--\n2.31.1", "diff": "diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp\nindex f1f26c775611..ea7ec8c05f88 100644\n--- a/src/android/camera_device.cpp\n+++ b/src/android/camera_device.cpp\n@@ -2025,9 +2025,6 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques\n\n void CameraDevice::requestComplete(Request *request)\n {\n-\tcamera3_buffer_status status = CAMERA3_BUFFER_STATUS_OK;\n-\tstd::unique_ptr<CameraMetadata> resultMetadata;\n-\n \tdecltype(descriptors_)::node_type node;\n \t{\n \t\tstd::scoped_lock<std::mutex> lock(mutex_);\n@@ -2035,7 +2032,8 @@ void CameraDevice::requestComplete(Request *request)\n \t\tif (it == descriptors_.end()) {\n \t\t\tLOG(HAL, Fatal)\n \t\t\t\t<< \"Unknown request: \" << request->cookie();\n-\t\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t\tclose();\n+\t\t\tnotifyError(0, nullptr, CAMERA3_MSG_ERROR_DEVICE);\n \t\t\treturn;\n \t\t}\n\n@@ -2043,16 +2041,71 @@ void CameraDevice::requestComplete(Request *request)\n \t}\n \tCamera3RequestDescriptor &descriptor = node.mapped();\n\n+\t/*\n+\t * Prepare the capture result for the Android camera stack.\n+\t *\n+\t * The buffer status is set to OK and later changed to ERROR if\n+\t * post-processing/compression fails.\n+\t */\n+\tcamera3_capture_result_t captureResult = {};\n+\tcaptureResult.frame_number = descriptor.frameNumber_;\n+\tcaptureResult.num_output_buffers = descriptor.buffers_.size();\n+\tfor (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n+\t\tbuffer.acquire_fence = -1;\n+\t\tbuffer.release_fence = -1;\n+\t\tbuffer.status = CAMERA3_BUFFER_STATUS_OK;\n+\t}\n+\tcaptureResult.output_buffers = descriptor.buffers_.data();\n+\tcaptureResult.partial_result = 1;\n+\n+\t/*\n+\t * If the Request has failed, abort the request by notifying the error\n+\t * and complete the request with all buffers in error state.\n+\t */\n \tif (request->status() != Request::RequestComplete) {\n-\t\tLOG(HAL, Error) << \"Request not successfully completed: \"\n+\t\tLOG(HAL, Error) << \"Request \" << request->cookie()\n+\t\t\t\t<< \" not successfully completed: \"\n \t\t\t\t<< request->status();\n-\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n+\n+\t\tnotifyError(descriptor.frameNumber_,\n+\t\t\t descriptor.buffers_[0].stream,\n+\t\t\t CAMERA3_MSG_ERROR_REQUEST);\n+\n+\t\tfor (camera3_stream_buffer_t &buffer : descriptor.buffers_)\n+\t\t\tbuffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\tcallbacks_->process_capture_result(callbacks_, &captureResult);\n+\n+\t\treturn;\n \t}\n\n+\t/*\n+\t * Notify shutter as soon as we have verified we have a valid request.\n+\t *\n+\t * \\todo The shutter event notification should be sent to the framework\n+\t * as soon as possible, earlier than request completion time.\n+\t */\n+\tuint64_t sensorTimestamp = static_cast<uint64_t>(request->metadata()\n+\t\t\t\t\t\t\t .get(controls::SensorTimestamp));\n+\tnotifyShutter(descriptor.frameNumber_, sensorTimestamp);\n+\n \tLOG(HAL, Debug) << \"Request \" << request->cookie() << \" completed with \"\n \t\t\t<< descriptor.buffers_.size() << \" streams\";\n\n-\tresultMetadata = getResultMetadata(descriptor);\n+\t/*\n+\t * Generate the metadata associated with the captured buffers.\n+\t *\n+\t * Notify if the metadata generation has failed, but continue processing\n+\t * buffers and return an empty metadata pack.\n+\t */\n+\tstd::unique_ptr<CameraMetadata> resultMetadata = getResultMetadata(descriptor);\n+\tif (!resultMetadata) {\n+\t\tnotifyError(descriptor.frameNumber_, descriptor.buffers_[0].stream,\n+\t\t\t CAMERA3_MSG_ERROR_RESULT);\n+\n+\t\t/* The camera framework expects an empy metadata pack on error. */\n+\t\tresultMetadata = std::make_unique<CameraMetadata>(0, 0);\n+\t}\n+\tcaptureResult.result = resultMetadata->get();\n\n \t/* Handle any JPEG compression. */\n \tfor (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n@@ -2065,56 +2118,27 @@ void CameraDevice::requestComplete(Request *request)\n \t\tFrameBuffer *src = request->findBuffer(cameraStream->stream());\n \t\tif (!src) {\n \t\t\tLOG(HAL, Error) << \"Failed to find a source stream buffer\";\n+\t\t\tbuffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t\tnotifyError(descriptor.frameNumber_, buffer.stream,\n+\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n \t\t\tcontinue;\n \t\t}\n\n-\t\tint ret = cameraStream->process(*src,\n-\t\t\t\t\t\t*buffer.buffer,\n+\t\tint ret = cameraStream->process(*src, *buffer.buffer,\n \t\t\t\t\t\tdescriptor.settings_,\n \t\t\t\t\t\tresultMetadata.get());\n-\t\tif (ret) {\n-\t\t\tstatus = CAMERA3_BUFFER_STATUS_ERROR;\n-\t\t\tcontinue;\n-\t\t}\n-\n \t\t/*\n \t\t * Return the FrameBuffer to the CameraStream now that we're\n \t\t * done processing it.\n \t\t */\n \t\tif (cameraStream->type() == CameraStream::Type::Internal)\n \t\t\tcameraStream->putBuffer(src);\n-\t}\n\n-\t/* Prepare to call back the Android camera stack. */\n-\tcamera3_capture_result_t captureResult = {};\n-\tcaptureResult.frame_number = descriptor.frameNumber_;\n-\tcaptureResult.num_output_buffers = descriptor.buffers_.size();\n-\tfor (camera3_stream_buffer_t &buffer : descriptor.buffers_) {\n-\t\tbuffer.acquire_fence = -1;\n-\t\tbuffer.release_fence = -1;\n-\t\tbuffer.status = status;\n-\t}\n-\tcaptureResult.output_buffers = descriptor.buffers_.data();\n-\n-\tif (status == CAMERA3_BUFFER_STATUS_OK) {\n-\t\tuint64_t timestamp =\n-\t\t\tstatic_cast<uint64_t>(request->metadata()\n-\t\t\t\t\t .get(controls::SensorTimestamp));\n-\t\tnotifyShutter(descriptor.frameNumber_, timestamp);\n-\n-\t\tcaptureResult.partial_result = 1;\n-\t\tcaptureResult.result = resultMetadata->get();\n-\t}\n-\n-\tif (status == CAMERA3_BUFFER_STATUS_ERROR || !captureResult.result) {\n-\t\t/* \\todo Improve error handling. In case we notify an error\n-\t\t * because the metadata generation fails, a shutter event has\n-\t\t * already been notified for this frame number before the error\n-\t\t * is here signalled. Make sure the error path plays well with\n-\t\t * the camera stack state machine.\n-\t\t */\n-\t\tnotifyError(descriptor.frameNumber_,\n-\t\t\t descriptor.buffers_[0].stream);\n+\t\tif (ret) {\n+\t\t\tbuffer.status = CAMERA3_BUFFER_STATUS_ERROR;\n+\t\t\tnotifyError(descriptor.frameNumber_, buffer.stream,\n+\t\t\t\t CAMERA3_MSG_ERROR_BUFFER);\n+\t\t}\n \t}\n\n \tcallbacks_->process_capture_result(callbacks_, &captureResult);\n@@ -2136,23 +2160,23 @@ void CameraDevice::notifyShutter(uint32_t frameNumber, uint64_t timestamp)\n \tcallbacks_->notify(callbacks_, ¬ify);\n }\n\n-void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream)\n+void CameraDevice::notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n+\t\t\t camera3_error_msg_code code)\n {\n \tcamera3_notify_msg_t notify = {};\n\n-\t/*\n-\t * \\todo Report and identify the stream number or configuration to\n-\t * clarify the stream that failed.\n-\t */\n-\tLOG(HAL, Error) << \"Error occurred on frame \" << frameNumber << \" (\"\n-\t\t\t<< toPixelFormat(stream->format).toString() << \")\";\n-\n \tnotify.type = CAMERA3_MSG_ERROR;\n \tnotify.message.error.error_stream = stream;\n \tnotify.message.error.frame_number = frameNumber;\n-\tnotify.message.error.error_code = CAMERA3_MSG_ERROR_REQUEST;\n+\tnotify.message.error.error_code = code;\n\n \tcallbacks_->notify(callbacks_, ¬ify);\n+\n+\tif (stream)\n+\t\tLOG(HAL, Error) << \"Error occurred on frame \" << frameNumber << \" (\"\n+\t\t\t\t<< toPixelFormat(stream->format).toString() << \")\";\n+\telse\n+\t\tLOG(HAL, Error) << \"Fatal error occurred on device\";\n }\n\n /*\ndiff --git a/src/android/camera_device.h b/src/android/camera_device.h\nindex 23457e47767a..8d5da8bc59e1 100644\n--- a/src/android/camera_device.h\n+++ b/src/android/camera_device.h\n@@ -101,7 +101,8 @@ private:\n \tstd::tuple<uint32_t, uint32_t> calculateStaticMetadataSize();\n \tlibcamera::FrameBuffer *createFrameBuffer(const buffer_handle_t camera3buffer);\n \tvoid notifyShutter(uint32_t frameNumber, uint64_t timestamp);\n-\tvoid notifyError(uint32_t frameNumber, camera3_stream_t *stream);\n+\tvoid notifyError(uint32_t frameNumber, camera3_stream_t *stream,\n+\t\t\t camera3_error_msg_code code);\n \tstd::unique_ptr<CameraMetadata> requestTemplatePreview();\n \tstd::unique_ptr<CameraMetadata> requestTemplateVideo();\n \tlibcamera::PixelFormat toPixelFormat(int format) const;\n", "prefixes": [ "libcamera-devel", "v2" ] }