Patch Detail
Show a patch.
GET /api/1.1/patches/18948/?format=api
{ "id": 18948, "url": "https://patchwork.libcamera.org/api/1.1/patches/18948/?format=api", "web_url": "https://patchwork.libcamera.org/patch/18948/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20230821131039.127370-6-gabbymg94@gmail.com>", "date": "2023-08-21T13:10:39", "name": "[libcamera-devel,RFC,v2,5/5] libcamera: pipeline: uvcvideo: Handle metadata stream", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "1efa4d332d6a298875d82b2fc207387af9a2a90c", "submitter": { "id": 160, "url": "https://patchwork.libcamera.org/api/1.1/people/160/?format=api", "name": "Gabrielle George", "email": "gabbymg94@gmail.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/18948/mbox/", "series": [ { "id": 4004, "url": "https://patchwork.libcamera.org/api/1.1/series/4004/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4004", "date": "2023-08-21T13:10:34", "name": "Add UVC Metadata buffer timestamp support", "version": 2, "mbox": "https://patchwork.libcamera.org/series/4004/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/18948/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/18948/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 7C2BAC32B2\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 21 Aug 2023 13:10:53 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AD39B628BE;\n\tMon, 21 Aug 2023 15:10:52 +0200 (CEST)", "from mail-oa1-x33.google.com (mail-oa1-x33.google.com\n\t[IPv6:2001:4860:4864:20::33])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2B045628C1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 21 Aug 2023 15:10:48 +0200 (CEST)", "by mail-oa1-x33.google.com with SMTP id\n\t586e51a60fabf-1a1fa977667so2192314fac.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 21 Aug 2023 06:10:48 -0700 (PDT)", "from localhost.localdomain (97-115-76-16.ptld.qwest.net.\n\t[97.115.76.16]) by smtp.gmail.com with ESMTPSA id\n\tci1-20020a056871c48100b001bb51450d85sm4160283oac.4.2023.08.21.06.10.45\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tMon, 21 Aug 2023 06:10:46 -0700 (PDT)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1692623452;\n\tbh=6AxJ8c4c7HXmRYYCXvoO2RunB2yaegrKd1K9gPzMqbM=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=hOcVR+7i3cY2jtZz4cF5jmOesLQs0yZGpfgJRaGYbLnc8buuJcqLgKF5jumxbdSN+\n\tJLcetcxLc5qFdOO2frZqmrDVJ/ZKhdVFo6X8d2PWPODmChTVLDvw5rGI9XZJVLw5aN\n\tQBfo7rN3vSg2kUxbYqj38uxvaUwJ3o2Kr/MC2a3qebsr3BFck7Lruipm4uxC2aVB9p\n\t06YqpG+tBcDvXpZJxLTyRWw/yTfPtRcUc0IG1WzZGwB7CPxUx+15bERQdt2qTP8U7r\n\tL0ptpZy81VgBipbuegr+tYrmQ0eMxE7vtC+1vq4KYIeMli3mzIvVB2EtMBZrJva9US\n\tXPs9jLJE5/x1w==", "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20221208; t=1692623446; x=1693228246;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:to:from:from:to:cc:subject:date:message-id\n\t:reply-to; bh=a0EOWnwLCT3Q7nV/PP4nZpbXd+qQfFMqXt5oWOXOEyI=;\n\tb=UeDUtaXLyVOBWsrk5nBz33ik5I6NwdjnamKH1yQTIQWc1ArsUp040jQ1qqR6qewcOw\n\t6AjdXSvJOYQDt7a5MVS3OCLscInRiZG7afpM8N2KEuplo9WbP0cjwDR2SrxK1yFotjPA\n\t4NDtnzZEsp5kegb6QVmQnUKgRC0o4fPedIO2NtlDmGclhO5ZF78IRtlqtdsj+7UeYiT5\n\tdZ4j0Oov+B7bwUs5p8Ee6PnWiMp91aTVHmNsgOjk80cfix83KZ+5S7/u7sgiIOTPGIDy\n\tlBOzGagvRBV/kJZUTSgjSR0qfferWrngvMqInM5hBbFAymC0qHwuslspY1k6lJblU9RW\n\toXhw==" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"UeDUtaXL\"; dkim-atps=neutral", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20221208; t=1692623446; x=1693228246;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=a0EOWnwLCT3Q7nV/PP4nZpbXd+qQfFMqXt5oWOXOEyI=;\n\tb=lXgmSx4v1UdvlIwHk03Th2fR7YHMyVs4OPSPzOQguZjI7AOTpNiSG7lH97lm/UUkJp\n\t+NOr26TxB1s3LJOKpHWZ0MQrqtDVp8Vl8YgCuDdZEGgNqRpnuZ87nrjrHEtNeDUxg23M\n\tYrrBHiHUOOm9YTXKAzta9AXw89PAovjhzdEY+Tdfg5r39OuRyHQScjUrK3v8FV8hPYo7\n\tQP+kzmRtW6MoP4TOWfCo2PPjPJBgyS3gDTcsvUBRwcSFas6zA8T9WFh3fKEjkdSLK3N9\n\to6CoRTjVOAFSY6BA/JvqSozpcKJIGXYLbKmfr4jKNnUCMHQfxEGEwbjgay6wBdooyl0X\n\t+k4g==", "X-Gm-Message-State": "AOJu0Yz0fNU81rQxEGmxkR2DpsO7wVsn7/u63Ztj04LY/uTSVZs3A0GF\n\ttgwgCFj1TpPo5MbqtxS3ddK/Gl7ZyLQ=", "X-Google-Smtp-Source": "AGHT+IFbiiJai62GYO2Je+gLgIVtGay20sCY9TAG7cudF32BI3DCpfOX8tEzaey4OcO00oUdMPXchQ==", "X-Received": "by 2002:a05:6870:4193:b0:1c0:2e8f:17fd with SMTP id\n\ty19-20020a056870419300b001c02e8f17fdmr10123197oac.40.1692623446638; \n\tMon, 21 Aug 2023 06:10:46 -0700 (PDT)", "To": "libcamera-devel@lists.libcamera.org, kieran.bingham@ideasonboard.com,\n\tvedantparanjape160201@gmail.com, gabbymg94@gmail.com", "Date": "Mon, 21 Aug 2023 06:10:39 -0700", "Message-Id": "<20230821131039.127370-6-gabbymg94@gmail.com>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20230821131039.127370-1-gabbymg94@gmail.com>", "References": "<20230821131039.127370-1-gabbymg94@gmail.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [RFC PATCH v2 5/5] libcamera: pipeline: uvcvideo:\n\tHandle metadata stream", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Gabby George via libcamera-devel <libcamera-devel@lists.libcamera.org>", "Reply-To": "Gabby George <gabbymg94@gmail.com>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Register the metadata stream's buffer ready callback and start\nprocessing metadata buffers. Use the timestamp from the metadata\nbuffer as the corresponding video buffer Requests' timestamp. Metadata\nbuffers are synchronized with frames coming into the video stream\nusing the sequence field of the buffers. They may come in either order\n(video buffer first or metadata buffer first), so store relevant\ninformation about the buffer required to set the metadata timestamp or\ncomplete the buffer request as soon as possible.\n\nThe timestamp will be improved upon in the next patch. For now, use\nthe driver-provided metadata timestamp.\n\nSigned-off-by: Gabby George <gabbymg94@gmail.com>\n---\n src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 168 ++++++++++++++++++-\n 1 file changed, 162 insertions(+), 6 deletions(-)", "diff": "diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\nindex c8d6633f..215435ec 100644\n--- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\n+++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\n@@ -12,6 +12,8 @@\n #include <memory>\n #include <tuple>\n \n+#include <linux/uvcvideo.h>\n+\n #include <libcamera/base/log.h>\n #include <libcamera/base/utils.h>\n \n@@ -34,6 +36,17 @@ namespace libcamera {\n \n LOG_DEFINE_CATEGORY(UVC)\n \n+/*\n+ * The UVCH buffer contains an unsigned char array\n+ * encoding UVC timing data that needs to be recast\n+ * into usable data.\n+ */\n+struct UVCTimingBuf {\n+\t__u32 pts;\n+\t__u32 stc;\n+\t__u16 sofDevice;\n+} __attribute__((packed));\n+\n class UVCCameraData : public Camera::Private\n {\n public:\n@@ -46,6 +59,8 @@ public:\n \tvoid addControl(uint32_t cid, const ControlInfo &v4l2info,\n \t\t\tControlInfoMap::Map *ctrls);\n \tvoid bufferReady(FrameBuffer *buffer);\n+\tvoid bufferReadyMetadata(FrameBuffer *buffer);\n+\tvoid handleUnfinishedRequests();\n \n \tconst std::string &id() const { return id_; }\n \n@@ -56,9 +71,16 @@ public:\n \tstd::map<unsigned int, MappedFrameBuffer> mappedMetadataBuffers_;\n \n \tstd::map<PixelFormat, std::vector<SizeRange>> formats_;\n+\tstd::queue<FrameBuffer *> pendingVideoBuffers_;\n+\tstd::queue<std::pair<unsigned int, uint64_t>> pendingMetadata_;\n \n private:\n \tint initMetadata(MediaDevice *media);\n+\tvoid completeRequest(FrameBuffer *buffer, uint64_t timestamp);\n+\tvoid endCorruptedStream();\n+\n+\tconst unsigned int frameStart_ = 1;\n+\tconst unsigned int maxVidBuffersInQueue_ = 1;\n \n \tbool generateId();\n \n@@ -242,7 +264,7 @@ int PipelineHandlerUVC::cleanupMetadataBuffers(Camera *camera)\n \tUVCCameraData *data = cameraData(camera);\n \n \tif (data->metadata_)\n-\t\tdata->metadata_->releaseBuffers();\n+\t\tret = data->metadata_->releaseBuffers();\n \tdata->metadataBuffers_.clear();\n \tdata->mappedMetadataBuffers_.clear();\n \tdata->metadata_ = nullptr;\n@@ -253,7 +275,9 @@ int PipelineHandlerUVC::cleanupMetadataBuffers(Camera *camera)\n int PipelineHandlerUVC::cleanup(Camera *camera)\n {\n \tUVCCameraData *data = cameraData(camera);\n+\n \tcleanupMetadataBuffers(camera);\n+\n \tdata->video_->releaseBuffers();\n \treturn 0;\n }\n@@ -354,6 +378,8 @@ void PipelineHandlerUVC::stopDevice(Camera *camera)\n \tif (data->metadata_)\n \t\tdata->metadata_->streamOff();\n \n+\tdata->handleUnfinishedRequests();\n+\n \tcleanup(camera);\n }\n \n@@ -646,8 +672,11 @@ int UVCCameraData::init(MediaDevice *media)\n \tif (ret) {\n \t\tmetadata_ = nullptr;\n \t\tLOG(UVC, Error) << \"Could not find a metadata video device.\";\n+\t\treturn 0;\n \t}\n \n+\tmetadata_->bufferReady.connect(this, &UVCCameraData::bufferReadyMetadata);\n+\n \treturn 0;\n }\n \n@@ -833,18 +862,145 @@ void UVCCameraData::addControl(uint32_t cid, const ControlInfo &v4l2Info,\n \tctrls->emplace(id, info);\n }\n \n-void UVCCameraData::bufferReady(FrameBuffer *buffer)\n+void UVCCameraData::completeRequest(FrameBuffer *buffer, uint64_t timestamp)\n {\n \tRequest *request = buffer->request();\n-\n-\t/* \\todo Use the UVC metadata to calculate a more precise timestamp */\n-\trequest->metadata().set(controls::SensorTimestamp,\n-\t\t\t\tbuffer->metadata().timestamp);\n+\trequest->metadata().set(controls::SensorTimestamp, timestamp);\n \n \tpipe()->completeBuffer(request, buffer);\n \tpipe()->completeRequest(request);\n }\n \n+void UVCCameraData::handleUnfinishedRequests()\n+{\n+\twhile (!pendingVideoBuffers_.empty()) {\n+\t\tFrameBuffer *oldBuffer = pendingVideoBuffers_.front();\n+\t\tRequest *oldRequest = oldBuffer->request();\n+\n+\t\toldRequest->metadata().set(controls::SensorTimestamp,\n+\t\t\t\t\t oldBuffer->metadata().timestamp);\n+\n+\t\tpipe()->completeBuffer(oldRequest, oldBuffer);\n+\t\tpipe()->completeRequest(oldRequest);\n+\t\tpendingVideoBuffers_.pop();\n+\t}\n+}\n+\n+void UVCCameraData::endCorruptedStream()\n+{\n+\thandleUnfinishedRequests();\n+\t/* Close the metadata node so we don't get inaccurate timestamps*/\n+\tmetadata_ = nullptr;\n+\tLOG(UVC, Error)\n+\t\t<< \"UVC metadata stream corrupted. Reverting to driver timestamps.\";\n+}\n+\n+/*\n+ * If there is a metadata buffer that hasn't been matched with a\n+ * video buffer, check to see if it matches this video buffer.\n+ *\n+ * If there is a match, use the timestamp stored in the metadata queue\n+ * for this video buffer's request. Complete this video buffer\n+ * and its request.\n+ *\n+ * If there are no metadata buffers available to check for a match,\n+ * push this video buffer's request object to the queue. It may\n+ * be that the metadata buffer has not yet arrived.\n+ * When the matching metadata buffer does come in, it will handle\n+ * completion of the buffer and request.\n+ *\n+ * If more than maxVidBuffersInQueue_ video buffers have been added\n+ * to the queue, something is wrong with the metadata stream and\n+ * we can no longer use UVC metadata packets for timestamps.\n+ * Complete all of the outstanding requests and turn off metadata\n+ * stream use.\n+ */\n+void UVCCameraData::bufferReady(FrameBuffer *buffer)\n+{\n+\t/* \\todo Use the UVC metadata to calculate a more precise timestamp */\n+\tif (!metadata_ || buffer->metadata().sequence < frameStart_) {\n+\t\tcompleteRequest(buffer, buffer->metadata().timestamp);\n+\t\treturn;\n+\t}\n+\n+\tif (!pendingMetadata_.empty()) {\n+\t\t/* A metadata buffer was ready first. */\n+\t\tunsigned int mdSequence = std::get<0>(pendingMetadata_.front()) + frameStart_;\n+\t\tif (mdSequence == buffer->metadata().sequence) {\n+\t\t\tcompleteRequest(buffer, std::get<1>(pendingMetadata_.front()));\n+\t\t\tpendingMetadata_.pop();\n+\t\t\treturn;\n+\t\t} else {\n+\t\t\t/* \\todo: Is there a reason metadata buffers can arrive out of order? */\n+\t\t\tendCorruptedStream();\n+\t\t\treturn;\n+\t\t}\n+\t}\n+\n+\tpendingVideoBuffers_.push(buffer);\n+\t/*\n+\t * Deal with video buffers that haven't been completed, and with\n+\t * buffers whose metadata information arrived out of order.\n+\t */\n+\tif (pendingVideoBuffers_.size() > maxVidBuffersInQueue_) {\n+\t\tendCorruptedStream();\n+\t}\n+}\n+\n+void UVCCameraData::bufferReadyMetadata(FrameBuffer *buffer)\n+{\n+\tif (!metadata_ ||\n+\t buffer->metadata().status != FrameMetadata::Status::FrameSuccess) {\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * The metadata stream seems to start at seq 1 and libcamera\n+\t * sets the start sequence to 0, so it's necessary to add one\n+\t * to match this buffer with the correct video frame buffer.\n+\t *\n+\t * \\todo: Is there a better way to do this? What is the root cause?\n+\t */\n+\tunsigned int mdSequence = buffer->metadata().sequence + frameStart_;\n+\tint pos = buffer->cookie();\n+\n+\tSpan<uint8_t> memMeta = mappedMetadataBuffers_.at(pos).planes()[0];\n+\tuvc_meta_buf *metaBuf = reinterpret_cast<uvc_meta_buf *>(memMeta.data());\n+\n+\t//Span<uint8_t> memTime = mappedMetadataBuffers_.at(pos).planes()[0];\n+\t//UVCTimingBuf * timeBuf = reinterpret_cast<UVCTimingBuf *>(&memTime.data()[sizeof(uvc_meta_buf)]);\n+\n+\tsize_t UVCPayloadHeaderSize = sizeof(metaBuf->length) +\n+\t\t\t\t sizeof(metaBuf->flags) + sizeof(UVCTimingBuf);\n+\tif (metaBuf->length < UVCPayloadHeaderSize) {\n+\t\tendCorruptedStream();\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * Match a pending video buffer with this buffer's sequence. If\n+\t * there is none available, put this timestamp information on the\n+\t * queue. When the matching video buffer does come in, it will use\n+\t * a timestamp from this metadata.\n+\t */\n+\tif (!pendingVideoBuffers_.empty()) {\n+\t\tFrameBuffer *vidBuffer = pendingVideoBuffers_.front();\n+\t\tunsigned int vidSequence = vidBuffer->metadata().sequence;\n+\n+\t\tif (vidSequence == mdSequence) {\n+\t\t\tcompleteRequest(vidBuffer, static_cast<uint64_t>(metaBuf->ns));\n+\t\t\tpendingVideoBuffers_.pop();\n+\t\t} else {\n+\t\t\tendCorruptedStream();\n+\t\t}\n+\t} else {\n+\t\tpendingMetadata_.push(\n+\t\t\tstd::make_pair(buffer->metadata().sequence,\n+\t\t\t\t static_cast<uint64_t>(metaBuf->ns)));\n+\t}\n+\tmetadata_->queueBuffer(buffer);\n+}\n+\n REGISTER_PIPELINE_HANDLER(PipelineHandlerUVC)\n \n } /* namespace libcamera */\n", "prefixes": [ "libcamera-devel", "RFC", "v2", "5/5" ] }