Patch Detail
Show a patch.
GET /api/patches/13189/?format=api
{ "id": 13189, "url": "https://patchwork.libcamera.org/api/patches/13189/?format=api", "web_url": "https://patchwork.libcamera.org/patch/13189/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<tencent_5B12F44775EB76DC77A7C123AD000ADB3905@qq.com>", "date": "2021-08-04T07:33:45", "name": "[libcamera-devel,RFC,v1,1/3] pipeline: isp: The software ISP-based pipeline handler", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "4382b923391b628371c61777c6f93b147d0a6d4c", "submitter": { "id": 88, "url": "https://patchwork.libcamera.org/api/people/88/?format=api", "name": "Siyuan Fan", "email": "siyuan.fan@foxmail.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/13189/mbox/", "series": [ { "id": 2299, "url": "https://patchwork.libcamera.org/api/series/2299/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=2299", "date": "2021-08-04T07:33:44", "name": "pipeline: isp: The software ISP Module", "version": 1, "mbox": "https://patchwork.libcamera.org/series/2299/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/13189/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/13189/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id 657EEC3236\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 4 Aug 2021 07:34:06 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 7BF8468822;\n\tWed, 4 Aug 2021 09:34:05 +0200 (CEST)", "from out162-62-58-211.mail.qq.com (out162-62-58-211.mail.qq.com\n\t[162.62.58.211])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id EF45668810\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 4 Aug 2021 09:33:58 +0200 (CEST)", "from localhost.localdomain ([123.126.82.9])\n\tby newxmesmtplogicsvrsza5.qq.com (NewEsmtp) with SMTP\n\tid 8722689A; Wed, 04 Aug 2021 15:33:50 +0800" ], "Authentication-Results": "lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (1024-bit key;\n\tunprotected) header.d=foxmail.com header.i=@foxmail.com\n\theader.b=\"ADVV/K1S\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=foxmail.com;\n\ts=s201512; t=1628062434;\n\tbh=bjp+htc5ywvGucIThHbIlSty4pziM5OG38ZKNJgVSDs=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References;\n\tb=ADVV/K1SL0MK94+nrRvwsW5UL0g6HRnprYPnJhOgMXKzJ/uyyvLwAJSQrx7Tx1nQC\n\tlJzBqq0lFGkwbc+9vzdp7s9KPW+rIddxidAWCq+Mj/Gcpk/xac+w3kXsOq5aPYF132\n\t95YJ28LwYV2y2DZcB82xYHyw5tvJc2NrSJUuqLJ8=", "X-QQ-mid": "xmsmtpt1628062431tgzczbxa0", "Message-ID": "<tencent_5B12F44775EB76DC77A7C123AD000ADB3905@qq.com>", "X-QQ-XMAILINFO": "ONalwM1aM/hi0gK1Uu1MxmqXNSAy/G+LXtilntyUerbhZ40hqZbN/ozEWm2PI6\n\tEelJosgv80hynUarzVmM7B4NtoswZoWxVwZMWguTs2lf6L9dUa39jZaMJL1h0m1buS/r9qsxq9ql\n\tIJUe//39PcBB7K3zrOpnj8IUk5JnzJaIRhNEQgIYPp/BQ73zNrVp1ngj+gc6dGlVcHUzE4y6icle\n\tbjsSiPnrOVOimbqvN3IgdYr9eCybeekfrp4msW+G2muEfcFR6yJt6SPCezRNcIDuaAAlrh91hsEF\n\t4h/BbQzMFbfWNMR3qWWIaCpMNCgo7pDYlHGlk2jhgDArnU3PhSFhmqgRFziQlQKaPsqJ0qdY3IlC\n\tPL1Lzyx9v76NTmigxKCQ12jIsbY3heSms0EufSLyhNNnv0JbDfYr073Jk2bVNEoENo0Gp0GNwl15\n\tBUCbQDd8Re54eIfkm62uRot8ivSODsTSbX+8zDPE2X6JZM1YD5mvJXScyFHMI7DQym1OsMa6BA7g\n\t5/J3WTqP+jqu+H0v+ECVYg8co4Ck0f7srHGSK7xPGikqzsVakg3CtG4isEFE0IqJTZb8Zyx7+nAi\n\tFm9sIPVPZWsTdczCT4IfHKvH8VL8qNqQr9wIBARQD3/OVNRybb3RmMFOIpxj2C86q++Sc0MbZjYx\n\tis/WDikg1gSvfNcZUsmvLOnoZZ96gajzH3JJgrsTQoTPW5wZWg/5Q3sOJy7LWI0iz7It7AknvRrG\n\tGCaVrue4TwlTASxh46KbTGQkjFnABzSR2+ZfnA498rW6xfrCoTtw2QCg4OPuEgfus7zGbx3BFVij\n\tjAh3zp8O3OS2CSHwTbqtxRcXRf2hTeRFRbsqzJqiw0hsTj+K+MSrROxzaCFxQWRaIUkEAk6fWX1J\n\tPn/LXnqRQSnpD1v/du/Vmj4CmwviQI0sV1QzwuHgzZSSusZDF0/5nN8o56ej0t0PV9F/uIjSaFE4\n\tc6FR0ZVLZOTprnNGlphseT6DsD2nMvodnauGG35rUC5KdKg4dZjw==", "From": "Siyuan Fan <siyuan.fan@foxmail.com>", "To": "libcamera-devel@lists.libcamera.org", "Date": "Wed, 4 Aug 2021 08:33:45 +0100", "X-OQ-MSGID": "<20210804073347.1368-2-siyuan.fan@foxmail.com>", "X-Mailer": "git-send-email 2.20.1", "In-Reply-To": "<20210804073347.1368-1-siyuan.fan@foxmail.com>", "References": "<20210804073347.1368-1-siyuan.fan@foxmail.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[libcamera-devel] [RFC PATCH v1 1/3] pipeline: isp: The software\n\tISP-based pipeline handler", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "From: Fan Siyuan <siyuan.fan@foxmail.com>\n\nAdd class PipelineHandlerISP to connect three parts, including application, Camera Sensor\nand ISP processing. The process is roughly as follows.\n\nThe application requests the format and size of rgb image(e.x. RGB888 640x480),\nthen pipeline handler configures the format and size of raw image (e.x. BGGR10 640X480)\nand calls V4L2VideoDevice::allocateBuffers and V4L2VideoDevice::queueBuffer to queue raw buffer. \nWhen bufferReady signal is emitted, which means that the raw buffer is filled with data captured by camera sensor.\nLater, pipeline handler calls the bufferReady function to pass the raw buffer to class isp::processing\nand the raw buffer is processed by isp pipeline. When processing done, the rgb buffer is exported to application\nby PipelineHandlerISP::exportFrameBuffers.\n\nSigned-off-by: Fan Siyuan <siyuan.fan@foxmail.com>\n\n---\n src/libcamera/pipeline/isp/isp.cpp | 323 +++++++++++++++++++++++++++++\n 1 file changed, 323 insertions(+)\n create mode 100644 src/libcamera/pipeline/isp/isp.cpp", "diff": "diff --git a/src/libcamera/pipeline/isp/isp.cpp b/src/libcamera/pipeline/isp/isp.cpp\nnew file mode 100644\nindex 00000000..e5fbd536\n--- /dev/null\n+++ b/src/libcamera/pipeline/isp/isp.cpp\n@@ -0,0 +1,323 @@\n+/* SPDX-License-Identifier: LGPL-2.1-or-later */\n+/*\n+ * Copyright (C) 2021, Siyuan Fan <siyuan.fan@foxmail.com>\n+ *\n+ * isp.cpp - The software ISP-based pipeline handler\n+ */\n+\n+#include \"isp_processing.h\"\n+\n+#include <math.h>\n+#include <stdlib.h>\n+#include <sys/mman.h>\n+#include <unistd.h>\n+\n+#include <libcamera/camera.h>\n+#include <libcamera/control_ids.h>\n+#include <libcamera/controls.h>\n+#include <libcamera/formats.h>\n+\n+#include \"libcamera/base/thread.h\"\n+#include \"libcamera/internal/device_enumerator.h\"\n+#include \"libcamera/internal/media_device.h\"\n+#include \"libcamera/internal/pipeline_handler.h\"\n+#include \"libcamera/internal/v4l2_videodevice.h\"\n+\n+namespace libcamera {\n+\n+LOG_DEFINE_CATEGORY(ISP)\n+\n+class ISPCameraData : public CameraData\n+{\n+public:\n+ ISPCameraData(PipelineHandler *pipe, MediaDevice *media)\n+ : CameraData(pipe), media_(media), video_(nullptr)\n+ {\n+ }\n+\n+ ~ISPCameraData()\n+ {\n+ delete video_;\n+ delete rawBuffers;\n+ }\n+\n+ int init();\n+ void bufferReady(FrameBuffer *buffer);\n+ void ISPCompleted(FrameBuffer *buffer);\n+\n+ Stream stream_;\n+ ISP isp_;\n+ Thread threadISP_;\n+ int width;\n+ int height;\n+\n+ MediaDevice *media_;\n+ V4L2VideoDevice *video_;\n+ std::vector<std::unique_ptr<FrameBuffer>> *tempCopyExportBuffers;\n+ std::vector<std::unique_ptr<FrameBuffer>> *rawBuffers;\n+ std::map<FrameBuffer *, FrameBuffer *> bufferPair;\n+};\n+\n+\n+class ISPCameraConfiguration : public CameraConfiguration\n+{\n+public:\n+ ISPCameraConfiguration();\n+\n+ Status validate() override;\n+};\n+\n+class PipelineHandlerISP : public PipelineHandler\n+{\n+public:\n+ PipelineHandlerISP(CameraManager *manager);\n+\n+ CameraConfiguration *generateConfiguration(Camera *camera,\n+ const StreamRoles &roles) override;\n+ int configure(Camera *camera, CameraConfiguration *config) override;\n+\n+ int exportFrameBuffers(Camera *camera, Stream *stream,\n+ std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;\n+\n+ int start(Camera *camera, const ControlList *controls) override;\n+ void stop(Camera *camera) override;\n+\n+ int queueRequestDevice(Camera *camera, Request *request) override;\n+\n+ bool match(DeviceEnumerator *enumerator) override;\n+\n+private:\n+ ISPCameraData *cameraData(const Camera *camera)\n+ {\n+ return static_cast<ISPCameraData *>(\n+ PipelineHandler::cameraData(camera));\n+ }\n+};\n+\n+ISPCameraConfiguration::ISPCameraConfiguration()\n+ : CameraConfiguration()\n+{\n+}\n+\n+CameraConfiguration::Status ISPCameraConfiguration::validate()\n+{\n+ Status status = Valid;\n+\n+ return status;\n+}\n+\n+PipelineHandlerISP::PipelineHandlerISP(CameraManager *manager)\n+ : PipelineHandler(manager)\n+{\n+}\n+\n+CameraConfiguration *PipelineHandlerISP::generateConfiguration(Camera *camera,\n+ const StreamRoles &roles)\n+{\n+ ISPCameraData *data = cameraData(camera);\n+ CameraConfiguration *config = new ISPCameraConfiguration();\n+\n+ if (roles.empty())\n+ return config;\n+\n+ std::map<V4L2PixelFormat, std::vector<SizeRange>> v4l2Formats =\n+ data->video_->formats();\n+ std::map<PixelFormat, std::vector<SizeRange>> deviceFormats;\n+ std::transform(v4l2Formats.begin(), v4l2Formats.end(),\n+ std::inserter(deviceFormats, deviceFormats.begin()),\n+ [&](const decltype(v4l2Formats)::value_type &format) {\n+ return decltype(deviceFormats)::value_type{\n+ format.first.toPixelFormat(),\n+ format.second\n+ };\n+ });\n+\n+ StreamFormats formats(deviceFormats);\n+ StreamConfiguration cfg(formats);\n+\n+ cfg.pixelFormat = formats::RGB888;\n+ cfg.size = { 640, 480 };\n+ cfg.bufferCount = 4;\n+\n+ config->addConfiguration(cfg);\n+\n+ config->validate();\n+\n+ return config;\n+}\n+\n+int PipelineHandlerISP::configure(Camera *camera, CameraConfiguration *config)\n+{\n+ ISPCameraData *data = cameraData(camera);\n+ StreamConfiguration &cfg = config->at(0);\n+\n+ V4L2VideoDevice::Formats fmts = data->video_->formats();\n+ V4L2PixelFormat v4l2Format = fmts.begin()->first;\n+\n+ V4L2DeviceFormat format = {};\n+ format.fourcc = v4l2Format;\n+ format.size = cfg.size;\n+\n+ data->width = format.size.width;\n+ data->height = format.size.height;\n+\n+ int ret = data->video_->setFormat(&format);\n+ if (ret)\n+ return ret;\n+\n+ cfg.setStream(&data->stream_);\n+ cfg.stride = format.planes[0].bpl;\n+\n+ return 0;\n+}\n+\n+int PipelineHandlerISP::exportFrameBuffers(Camera *camera, Stream *stream,\n+ std::vector<std::unique_ptr<FrameBuffer>> *buffers)\n+{\n+ unsigned int count = stream->configuration().bufferCount;\n+ ISPCameraData *data = cameraData(camera);\n+ \n+ for (unsigned int i = 0; i < count; i++) {\n+ std::string name = \"frame-\" + std::to_string(i);\n+ const int isp_fd = memfd_create(name.c_str(), 0);\n+ int ret = ftruncate(isp_fd, data->width * data->height * 3);\n+ if (ret < 0) {\n+ LOG(ISP, Error)\n+ << \"truncate: \"\n+ << strerror(-ret);\n+ return ret;\n+ }\n+ FileDescriptor temp = FileDescriptor(std::move(isp_fd));\n+\n+ FrameBuffer::Plane rgbPlane;\n+ rgbPlane.fd = std::move(temp);\n+ rgbPlane.length = data->width * data->height * 3;\n+\n+ std::vector<FrameBuffer::Plane> planes;\n+ planes.push_back(std::move(rgbPlane));\n+ std::unique_ptr<FrameBuffer> buffer = std::make_unique<FrameBuffer>(std::move(planes));\n+ buffers->push_back(std::move(buffer));\n+ }\n+\n+ data->tempCopyExportBuffers = buffers;\n+\n+ return count; \n+\n+}\n+\n+int PipelineHandlerISP::start(Camera *camera, [[maybe_unused]] const ControlList *controls)\n+{\n+ ISPCameraData *data = cameraData(camera);\n+ unsigned int count = data->stream_.configuration().bufferCount;\n+\n+ data->rawBuffers = new std::vector<std::unique_ptr<FrameBuffer>>;\n+ int ret = data->video_->allocateBuffers(count, data->rawBuffers);\n+ if (ret < 0) {\n+ LOG(ISP, Error) << strerror(-ret);\n+ return ret;\n+ }\n+\n+ for (unsigned int i = 0; i < count; i++)\n+ data->bufferPair[data->tempCopyExportBuffers->at(i).get()] = data->rawBuffers->at(i).get();\n+\n+ ret = data->video_->streamOn();\n+ if (ret < 0) {\n+ data->video_->releaseBuffers();\n+ return ret;\n+ }\n+\n+ data->threadISP_.start();\n+\n+ return 0;\n+}\n+\n+void PipelineHandlerISP::stop(Camera *camera)\n+{\n+ ISPCameraData *data = cameraData(camera);\n+\n+ data->threadISP_.exit();\n+ data->threadISP_.wait();\n+\n+ data->video_->streamOff();\n+ data->video_->releaseBuffers();\n+}\n+\n+int PipelineHandlerISP::queueRequestDevice(Camera *camera, Request *request)\n+{\n+ ISPCameraData *data = cameraData(camera);\n+ FrameBuffer *rgbBuffer = request->findBuffer(&data->stream_);\n+ \n+ if (!rgbBuffer) {\n+ LOG(ISP, Error) << \"Attempt to queue request with invalid stream\";\n+ return -ENOENT;\n+ }\n+\n+ int ret = data->video_->queueBuffer(data->bufferPair[rgbBuffer]);\n+ if (ret < 0)\n+ return ret;\n+\n+ return 0;\n+}\n+\n+bool PipelineHandlerISP::match(DeviceEnumerator *enumerator)\n+{\n+ DeviceMatch unicam(\"unicam\");\n+\n+ unicam.add(\"unicam-embedded\");\n+ unicam.add(\"unicam-image\");\n+\n+ MediaDevice *unicam_ = acquireMediaDevice(enumerator, unicam);\n+ if (!unicam_) {\n+ LOG(ISP, Debug) << \"unicam Device not found\";\n+ return false;\n+ }\n+\n+ LOG(ISP, Debug) << \"unicam Device Identified\";\n+\n+ std::unique_ptr<ISPCameraData> data = std::make_unique<ISPCameraData>(this, unicam_);\n+\n+ if(data->init()) return false;\n+\n+ std::set<Stream *> streams{&data->stream_};\n+ std::shared_ptr<Camera> camera = Camera::create(this, data->video_->deviceName(), streams);\n+ registerCamera(std::move(camera), std::move(data));\n+\n+ return true;\n+}\n+\n+\n+void ISPCameraData::ISPCompleted(FrameBuffer *buffer)\n+{\n+ Request *request = buffer->request();\n+\n+ pipe_->completeBuffer(request, buffer);\n+ pipe_->completeRequest(request);\n+\n+}\n+\n+void ISPCameraData::bufferReady(FrameBuffer *buffer)\n+{\n+ for (std::map<FrameBuffer*, FrameBuffer*>::iterator it = bufferPair.begin(); it != bufferPair.end(); it++) {\n+ if (it->second == buffer)\n+ isp_.invokeMethod(&ISP::processing, ConnectionTypeQueued, buffer, it->first, width, height);\n+ }\n+ \n+}\n+\n+int ISPCameraData::init()\n+{\n+ video_ = new V4L2VideoDevice(media_->getEntityByName(\"unicam-image\"));\n+ if (video_->open())\n+ return -ENODEV;\n+\n+ video_->bufferReady.connect(this, &ISPCameraData::bufferReady);\n+\n+ isp_.moveToThread(&threadISP_);\n+ isp_.ispCompleted.connect(this, &ISPCameraData::ISPCompleted);\n+\n+ return 0;\n+}\n+\n+REGISTER_PIPELINE_HANDLER(PipelineHandlerISP)\n+\n+} /* namespace libcamera */\n", "prefixes": [ "libcamera-devel", "RFC", "v1", "1/3" ] }