Patch Detail
Show a patch.
GET /api/1.1/patches/18942/?format=api
{ "id": 18942, "url": "https://patchwork.libcamera.org/api/1.1/patches/18942/?format=api", "web_url": "https://patchwork.libcamera.org/patch/18942/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20230814112849.176943-6-gabbymg94@gmail.com>", "date": "2023-08-14T11:28:49", "name": "[libcamera-devel,RFC,5/5] libcamera: pipeline: uvcvideo: Handle metadata stream", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "7100e958d6a5832d9d90c99d313d7953c345a57d", "submitter": { "id": 160, "url": "https://patchwork.libcamera.org/api/1.1/people/160/?format=api", "name": "Gabrielle George", "email": "gabbymg94@gmail.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/18942/mbox/", "series": [ { "id": 4003, "url": "https://patchwork.libcamera.org/api/1.1/series/4003/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4003", "date": "2023-08-14T11:28:44", "name": "RFC:Add UVC Metadata buffer timestamp support", "version": 1, "mbox": "https://patchwork.libcamera.org/series/4003/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/18942/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/18942/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A228FC32B2\n\tfor <parsemail@patchwork.libcamera.org>;\n\tMon, 14 Aug 2023 11:29:03 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id AB059628E6;\n\tMon, 14 Aug 2023 13:29:01 +0200 (CEST)", "from mail-ot1-x333.google.com (mail-ot1-x333.google.com\n\t[IPv6:2607:f8b0:4864:20::333])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 86309628DF\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 14 Aug 2023 13:28:58 +0200 (CEST)", "by mail-ot1-x333.google.com with SMTP id\n\t46e09a7af769-6bcd4b5ebbaso3730565a34.1\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tMon, 14 Aug 2023 04:28:58 -0700 (PDT)", "from localhost.localdomain (97-115-76-16.ptld.qwest.net.\n\t[97.115.76.16]) by smtp.gmail.com with ESMTPSA id\n\tc3-20020a9d6c83000000b006b92509e76esm4163817otr.32.2023.08.14.04.28.56\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tMon, 14 Aug 2023 04:28:56 -0700 (PDT)" ], "DKIM-Signature": [ "v=1; a=rsa-sha256; c=relaxed/simple; d=libcamera.org;\n\ts=mail; t=1692012541;\n\tbh=ABhfgmo5PUBSXwT8Jvvta5ZvStQKy++jlNCWwuPYcIg=;\n\th=To:Date:In-Reply-To:References:Subject:List-Id:List-Unsubscribe:\n\tList-Archive:List-Post:List-Help:List-Subscribe:From:Reply-To:\n\tFrom;\n\tb=sjDmf/IXHhTAslxv8iS45hmh5ECWzequXqFhFj+QNY29un+b0NMAqX+KPXni6LBbU\n\tMPX7FxuY3xXD73upwg6EAhvwHolUDZOaQp7KUCRO9qgrl2Ga7FFY/BEHcDywYltd4E\n\t8d79Mtrazxj5lFEuTn5+sJuZWV0P+ce6bN5Vg8MiEAk6YyxxIxwq73HCyOPtj3bec/\n\tumhQklBVaGvEuFgvOp7OPhJT9cWqaf1/xrOB99fTKB9iJ63tdLCGNH4TAnFtXT99Rl\n\tdGBhz04DiQek49Ex36ADTOXFzlwKyN96k1rSH28IrYjlhfd0FlcnmKBAsvy5jSflUd\n\tVgxejjCRYt7fg==", "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20221208; t=1692012537; x=1692617337;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:to:from:from:to:cc:subject:date:message-id\n\t:reply-to; bh=pShnn9eKyZPi5uoMChY9/RWSzW6FXWMZIhCveXimmHI=;\n\tb=aR/UQLMLg62UNlaUiwZ/2c/PRPvyh3uYqNgtuaRz57pmC8GPoA4FurJLnD7vm+ZH8z\n\tCyvKDln+NXeKUNhp8KqXxODEY1KLRkq83uQQ3Y59F6ktcJkc0ISnERavZNNJiYve8KTx\n\tMvH4huLxfq6ggYWiMFq+7qlwNwv1BSM1cGry1d2oYVfxJKwO8IWstPfBe6YKxuaQ9EZn\n\toAhezKTCit1syeo9FEvtFPU23OOYfgUKVFqZAmNGvuTlimDHMlvsWPMPRJno1AaSdnPZ\n\tW3VwbxnlgoFM9A4Vv4434Ocgnn5s9AW4KcatdncvlK2tH8ssjfb+bUYMML9V1Yc00vCB\n\tFaeA==" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (2048-bit key; \n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"aR/UQLML\"; dkim-atps=neutral", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20221208; t=1692012537; x=1692617337;\n\th=content-transfer-encoding:mime-version:references:in-reply-to\n\t:message-id:date:subject:to:from:x-gm-message-state:from:to:cc\n\t:subject:date:message-id:reply-to;\n\tbh=pShnn9eKyZPi5uoMChY9/RWSzW6FXWMZIhCveXimmHI=;\n\tb=cw3c/qU65fdAH78oPdD8bijZ048y402n7TAe6EdOqRM4y+U4lLmEd2G0d1lGnb717Y\n\tM+yZOV+nRs21mn2ZZK8RXczz3ktvFYzYSClXcI69Z+0nlgMxCSsC0VCNtTxeHBrZkIWk\n\tluKS55uCCwra5Uk0wjaxmKxXNJHMLxl+DFLSvjUXu1bQ+Qid4IwpMBCX36rjsI5z5o2j\n\th0hWQd4sxg3ELG87XPBKKz5xM8d13gryCEkfcUYC1eZzf8R/khaIW1osdGDqVIIYTQMJ\n\tqEXxTGEGljX50bwd41vhGrBHfG55AsvxhJ/i2fSETGowXk57wqjw9ZGG10JbW0gdVV78\n\tBhrg==", "X-Gm-Message-State": "AOJu0YxkxLsUQta6bo5+iKdwAjcAtCfv2D3N9IdBTckHvcLM9ss5OGCP\n\tNhlXtvz4uxB9l7ZYEgv26z9cQNaR7nK24w==", "X-Google-Smtp-Source": "AGHT+IHsDct/sp9xSgzkJchQVT/FA8dlD5GvKjGS0USkEzX+SzBk8fJ5qd8p6u6A8HAkXH0Jo2BHng==", "X-Received": "by 2002:a9d:63ce:0:b0:6b9:696d:716a with SMTP id\n\te14-20020a9d63ce000000b006b9696d716amr9676790otl.18.1692012536904; \n\tMon, 14 Aug 2023 04:28:56 -0700 (PDT)", "To": "libcamera-devel@lists.libcamera.org, kieran.bingham@ideasonboard.com,\n\tvedantparanjape160201@gmail.com, gabbymg94@gmail.com", "Date": "Mon, 14 Aug 2023 04:28:49 -0700", "Message-Id": "<20230814112849.176943-6-gabbymg94@gmail.com>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20230814112849.176943-1-gabbymg94@gmail.com>", "References": "<20230814112849.176943-1-gabbymg94@gmail.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [RFC PATCH 5/5] libcamera: pipeline: uvcvideo:\n\tHandle metadata stream", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "From": "Gabby George via libcamera-devel <libcamera-devel@lists.libcamera.org>", "Reply-To": "Gabby George <gabbymg94@gmail.com>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "Register the metadata stream's buffer ready callback and start\nprocessing metadata buffers. Use the timestamp from the metadata\nbuffer as the corresponding video buffer Requests' timestamp. Metadata\nbuffers are synchronized with frames coming into the video stream\nusing the sequence field of the buffers. They may come in either order\n(video buffer first or metadata buffer first), so store relevant\ninformation about the buffer required to set the metadata timestamp or\ncomplete the buffer request as soon as possible.\n\nThe timestamp will improved upon in the next patch. For now, use the\ndriver-provided metadata timestamp.\n\nSigned-off-by: Gabby George <gabbymg94@gmail.com>\n---\n src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 157 ++++++++++++++++++-\n 1 file changed, 152 insertions(+), 5 deletions(-)", "diff": "diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\nindex 51f30187..5c7ae064 100644\n--- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\n+++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp\n@@ -12,6 +12,8 @@\n #include <memory>\n #include <tuple>\n \n+#include <linux/uvcvideo.h>\n+\n #include <libcamera/base/log.h>\n #include <libcamera/base/utils.h>\n \n@@ -34,6 +36,13 @@ namespace libcamera {\n \n LOG_DEFINE_CATEGORY(UVC)\n \n+/* This is used to memcpy */\n+struct UVCMetadataPacked {\n+\t__u32 pts;\n+\t__u32 scr;\n+\t__u16 sofDevice;\n+} __attribute__((packed));\n+\n class UVCCameraData : public Camera::Private\n {\n public:\n@@ -46,6 +55,7 @@ public:\n \tvoid addControl(uint32_t cid, const ControlInfo &v4l2info,\n \t\t\tControlInfoMap::Map *ctrls);\n \tvoid bufferReady(FrameBuffer *buffer);\n+\tvoid bufferReadyMetadata(FrameBuffer *buffer);\n \n \tconst std::string &id() const { return id_; }\n \n@@ -57,10 +67,14 @@ public:\n \tbool useMetadataStream_;\n \n \tstd::map<PixelFormat, std::vector<SizeRange>> formats_;\n+\tstd::queue<std::pair<Request *, FrameBuffer *>> waitingForVideoBuffer_;\n+\tstd::queue<std::pair<unsigned int, uint64_t>> waitingForMDBuffer_;\n \n private:\n \tint initMetadata(MediaDevice *media);\n \n+\tconst unsigned int frameStart_ = 1;\n+\tconst unsigned int maxVidBuffersInQueue_ = 2;\n \tconst unsigned int minLengthHeaderBuf_ = 10;\n \n \tbool generateId();\n@@ -638,8 +652,16 @@ int UVCCameraData::init(MediaDevice *media)\n \t}\n \n \tcontrolInfo_ = ControlInfoMap(std::move(ctrls), controls::controls);\n+\tret = initMetadata(media);\n+\n+\tif (!ret) {\n+\t\tmetadata_->bufferReady.connect(this, &UVCCameraData::bufferReadyMetadata);\n+\t\tuseMetadataStream_ = true;\n+\t} else {\n+\t\tuseMetadataStream_ = false;\n+\t}\n \n-\treturn initMetadata(media);\n+\treturn 0;\n }\n \n bool UVCCameraData::generateId()\n@@ -824,16 +846,141 @@ void UVCCameraData::addControl(uint32_t cid, const ControlInfo &v4l2Info,\n \tctrls->emplace(id, info);\n }\n \n+/*\n+ * If there is a metadata buffer that hasn't been matched with a\n+ * video buffer, check to see if it matches this video buffer.\n+ *\n+ * If there is a match, use the timestamp stored in the metadata queue\n+ * for this video buffer's request. Complete this video buffer\n+ * and its request.\n+ *\n+ * If there are no metadata buffers available to check for a match,\n+ * push this video buffer's request object to the queue. It may\n+ * be that the metadata buffer has not yet arrived.\n+ * When the matching metadata buffer does come in, it will handle\n+ * completion of the buffer and request.\n+ *\n+ * If more than maxVidBuffersInQueue_ video buffers have been added\n+ * to the queue, something is wrong with the metadata stream and\n+ * we can no longer use UVC metadata packets for timestamps.\n+ * Complete all of the outstanding requests and turn off metadata\n+ * stream use.\n+ */\n void UVCCameraData::bufferReady(FrameBuffer *buffer)\n {\n \tRequest *request = buffer->request();\n-\n-\t/* \\todo Use the UVC metadata to calculate a more precise timestamp */\n \trequest->metadata().set(controls::SensorTimestamp,\n \t\t\t\tbuffer->metadata().timestamp);\n \n-\tpipe()->completeBuffer(request, buffer);\n-\tpipe()->completeRequest(request);\n+\tif (useMetadataStream_) {\n+\t\tif (buffer->metadata().sequence == 0) {\n+\t\t\t/* \\todo: we do not expect the first frame to have a\n+\t\t\t* metadata buffer associated with it. Why?\n+\t\t\t*/\n+\t\t\tpipe()->completeBuffer(request, buffer);\n+\t\t\tpipe()->completeRequest(request);\n+\t\t\treturn;\n+\t\t}\n+\n+\t\tif (!waitingForMDBuffer_.empty()) {\n+\t\t\tunsigned int mdSequence =\n+\t\t\t\tstd::get<0>(waitingForMDBuffer_.front()) + frameStart_;\n+\t\t\tif (mdSequence == buffer->metadata().sequence) {\n+\t\t\t\trequest->metadata().set(controls::SensorTimestamp,\n+\t\t\t\t\t\t\tstd::get<1>(waitingForMDBuffer_.front()));\n+\t\t\t\tpipe()->completeBuffer(request, buffer);\n+\t\t\t\tpipe()->completeRequest(request);\n+\t\t\t\twaitingForMDBuffer_.pop();\n+\t\t\t\treturn;\n+\t\t\t}\n+\t\t} else {\n+\t\t\twaitingForVideoBuffer_.push(std::make_pair(request, buffer));\n+\t\t}\n+\n+\t\tif (waitingForVideoBuffer_.size() > maxVidBuffersInQueue_) {\n+\t\t\twhile (!waitingForVideoBuffer_.empty()) {\n+\t\t\t\tRequest *oldRequest = std::get<0>(waitingForVideoBuffer_.front());\n+\t\t\t\tFrameBuffer *oldBuffer = std::get<1>(waitingForVideoBuffer_.front());\n+\t\t\t\toldRequest->metadata().set(controls::SensorTimestamp,\n+\t\t\t\t\t\t\t oldBuffer->metadata().timestamp);\n+\t\t\t\tpipe()->completeBuffer(oldRequest, oldBuffer);\n+\t\t\t\tpipe()->completeRequest(oldRequest);\n+\t\t\t\twaitingForVideoBuffer_.pop();\n+\t\t\t}\n+\t\t}\n+\t} else {\n+\t\tpipe()->completeBuffer(request, buffer);\n+\t\tpipe()->completeRequest(request);\n+\t}\n+}\n+\n+void UVCCameraData::bufferReadyMetadata(FrameBuffer *buffer)\n+{\n+\tif (!useMetadataStream_ || buffer->metadata().status != FrameMetadata::Status::FrameSuccess) {\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * The metadata stream always starts at seq 1 and libcamera sets the start sequence to 0,\n+\t * so it's necessary to add one to match this buffer with the correct\n+\t * video frame buffer.\n+\t *\n+\t * \\todo: Is there a better way to do this? What is the root cause?\n+\t */\n+\tunsigned int mdSequence = buffer->metadata().sequence + frameStart_;\n+\tint pos = buffer->cookie();\n+\t/*\n+\t * A UVC Metadata Block length field contains size of\n+\t * the header buf, length field, and flags field.\n+\t */\n+\tuvc_meta_buf metadataBuf;\n+\t__u8 minLength = minLengthHeaderBuf_ +\n+\t\t\t sizeof(metadataBuf.length) +\n+\t\t\t sizeof(metadataBuf.flags);\n+\tsize_t lenMDPacket = minLength + sizeof(metadataBuf.ns) + sizeof(metadataBuf.sof);\n+\tmemcpy(&metadataBuf, mappedMetadataBuffers_.at(pos).planes()[0].data(), lenMDPacket);\n+\n+\tif (metadataBuf.length < minLength) {\n+\t\tLOG(UVC, Error) << \"Received improper metadata packet. Using default timestamps.\";\n+\t\tuseMetadataStream_ = false;\n+\t\treturn;\n+\t}\n+\n+\t/*\n+\t * If there is a video buffer that hasn't been matched with a\n+\t * metadata buffer, check to see if it matches this metadata buffer.\n+\t *\n+\t * If there is a match, use the timestamp associated with this\n+\t * metadata buffer as the timestamp for the video buffer's request.\n+\t * Complete that video buffer and its request.\n+\t *\n+\t * If there are no video buffers, push this metadata buffer's\n+\t * sequence number and timestamp to a shared queue. It may be that\n+\t * the metadata buffer came in before the video buffer.\n+\t * When the matching video buffer does come in, it will use this\n+\t * metadata buffer's timestamp.\n+\t */\n+\t__u64 timestamp = metadataBuf.ns;\n+\n+\tif (!waitingForVideoBuffer_.empty()) {\n+\t\tRequest *request = std::get<0>(waitingForVideoBuffer_.front());\n+\t\tFrameBuffer *vidBuffer = std::get<1>(waitingForVideoBuffer_.front());\n+\t\tunsigned int vidSequence = vidBuffer->metadata().sequence;\n+\n+\t\tif (vidSequence == mdSequence) {\n+\t\t\trequest->metadata().set(controls::SensorTimestamp,\n+\t\t\t\t\t\ttimestamp);\n+\n+\t\t\tpipe()->completeBuffer(request, vidBuffer);\n+\t\t\tpipe()->completeRequest(request);\n+\t\t\twaitingForVideoBuffer_.pop();\n+\t\t}\n+\t} else {\n+\t\twaitingForMDBuffer_.push(\n+\t\t\tstd::make_pair(buffer->metadata().sequence,\n+\t\t\t\t timestamp));\n+\t}\n+\tmetadata_->queueBuffer(buffer);\n }\n \n REGISTER_PIPELINE_HANDLER(PipelineHandlerUVC)\n", "prefixes": [ "libcamera-devel", "RFC", "5/5" ] }