From patchwork Wed Aug 4 07:33:45 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Siyuan Fan X-Patchwork-Id: 13189 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 657EEC3236 for ; Wed, 4 Aug 2021 07:34:06 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 7BF8468822; Wed, 4 Aug 2021 09:34:05 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=foxmail.com header.i=@foxmail.com header.b="ADVV/K1S"; dkim-atps=neutral Received: from out162-62-58-211.mail.qq.com (out162-62-58-211.mail.qq.com [162.62.58.211]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id EF45668810 for ; Wed, 4 Aug 2021 09:33:58 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=foxmail.com; s=s201512; t=1628062434; bh=bjp+htc5ywvGucIThHbIlSty4pziM5OG38ZKNJgVSDs=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ADVV/K1SL0MK94+nrRvwsW5UL0g6HRnprYPnJhOgMXKzJ/uyyvLwAJSQrx7Tx1nQC lJzBqq0lFGkwbc+9vzdp7s9KPW+rIddxidAWCq+Mj/Gcpk/xac+w3kXsOq5aPYF132 95YJ28LwYV2y2DZcB82xYHyw5tvJc2NrSJUuqLJ8= Received: from localhost.localdomain ([123.126.82.9]) by newxmesmtplogicsvrsza5.qq.com (NewEsmtp) with SMTP id 8722689A; Wed, 04 Aug 2021 15:33:50 +0800 X-QQ-mid: xmsmtpt1628062431tgzczbxa0 Message-ID: X-QQ-XMAILINFO: ONalwM1aM/hi0gK1Uu1MxmqXNSAy/G+LXtilntyUerbhZ40hqZbN/ozEWm2PI6 EelJosgv80hynUarzVmM7B4NtoswZoWxVwZMWguTs2lf6L9dUa39jZaMJL1h0m1buS/r9qsxq9ql IJUe//39PcBB7K3zrOpnj8IUk5JnzJaIRhNEQgIYPp/BQ73zNrVp1ngj+gc6dGlVcHUzE4y6icle bjsSiPnrOVOimbqvN3IgdYr9eCybeekfrp4msW+G2muEfcFR6yJt6SPCezRNcIDuaAAlrh91hsEF 4h/BbQzMFbfWNMR3qWWIaCpMNCgo7pDYlHGlk2jhgDArnU3PhSFhmqgRFziQlQKaPsqJ0qdY3IlC PL1Lzyx9v76NTmigxKCQ12jIsbY3heSms0EufSLyhNNnv0JbDfYr073Jk2bVNEoENo0Gp0GNwl15 BUCbQDd8Re54eIfkm62uRot8ivSODsTSbX+8zDPE2X6JZM1YD5mvJXScyFHMI7DQym1OsMa6BA7g 5/J3WTqP+jqu+H0v+ECVYg8co4Ck0f7srHGSK7xPGikqzsVakg3CtG4isEFE0IqJTZb8Zyx7+nAi Fm9sIPVPZWsTdczCT4IfHKvH8VL8qNqQr9wIBARQD3/OVNRybb3RmMFOIpxj2C86q++Sc0MbZjYx is/WDikg1gSvfNcZUsmvLOnoZZ96gajzH3JJgrsTQoTPW5wZWg/5Q3sOJy7LWI0iz7It7AknvRrG GCaVrue4TwlTASxh46KbTGQkjFnABzSR2+ZfnA498rW6xfrCoTtw2QCg4OPuEgfus7zGbx3BFVij jAh3zp8O3OS2CSHwTbqtxRcXRf2hTeRFRbsqzJqiw0hsTj+K+MSrROxzaCFxQWRaIUkEAk6fWX1J Pn/LXnqRQSnpD1v/du/Vmj4CmwviQI0sV1QzwuHgzZSSusZDF0/5nN8o56ej0t0PV9F/uIjSaFE4 c6FR0ZVLZOTprnNGlphseT6DsD2nMvodnauGG35rUC5KdKg4dZjw== From: Siyuan Fan To: libcamera-devel@lists.libcamera.org Date: Wed, 4 Aug 2021 08:33:45 +0100 X-OQ-MSGID: <20210804073347.1368-2-siyuan.fan@foxmail.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210804073347.1368-1-siyuan.fan@foxmail.com> References: <20210804073347.1368-1-siyuan.fan@foxmail.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v1 1/3] pipeline: isp: The software ISP-based pipeline handler X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Fan Siyuan Add class PipelineHandlerISP to connect three parts, including application, Camera Sensor and ISP processing. The process is roughly as follows. The application requests the format and size of rgb image(e.x. RGB888 640x480), then pipeline handler configures the format and size of raw image (e.x. BGGR10 640X480) and calls V4L2VideoDevice::allocateBuffers and V4L2VideoDevice::queueBuffer to queue raw buffer. When bufferReady signal is emitted, which means that the raw buffer is filled with data captured by camera sensor. Later, pipeline handler calls the bufferReady function to pass the raw buffer to class isp::processing and the raw buffer is processed by isp pipeline. When processing done, the rgb buffer is exported to application by PipelineHandlerISP::exportFrameBuffers. Signed-off-by: Fan Siyuan --- src/libcamera/pipeline/isp/isp.cpp | 323 +++++++++++++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 src/libcamera/pipeline/isp/isp.cpp diff --git a/src/libcamera/pipeline/isp/isp.cpp b/src/libcamera/pipeline/isp/isp.cpp new file mode 100644 index 00000000..e5fbd536 --- /dev/null +++ b/src/libcamera/pipeline/isp/isp.cpp @@ -0,0 +1,323 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Siyuan Fan + * + * isp.cpp - The software ISP-based pipeline handler + */ + +#include "isp_processing.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "libcamera/base/thread.h" +#include "libcamera/internal/device_enumerator.h" +#include "libcamera/internal/media_device.h" +#include "libcamera/internal/pipeline_handler.h" +#include "libcamera/internal/v4l2_videodevice.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(ISP) + +class ISPCameraData : public CameraData +{ +public: + ISPCameraData(PipelineHandler *pipe, MediaDevice *media) + : CameraData(pipe), media_(media), video_(nullptr) + { + } + + ~ISPCameraData() + { + delete video_; + delete rawBuffers; + } + + int init(); + void bufferReady(FrameBuffer *buffer); + void ISPCompleted(FrameBuffer *buffer); + + Stream stream_; + ISP isp_; + Thread threadISP_; + int width; + int height; + + MediaDevice *media_; + V4L2VideoDevice *video_; + std::vector> *tempCopyExportBuffers; + std::vector> *rawBuffers; + std::map bufferPair; +}; + + +class ISPCameraConfiguration : public CameraConfiguration +{ +public: + ISPCameraConfiguration(); + + Status validate() override; +}; + +class PipelineHandlerISP : public PipelineHandler +{ +public: + PipelineHandlerISP(CameraManager *manager); + + CameraConfiguration *generateConfiguration(Camera *camera, + const StreamRoles &roles) override; + int configure(Camera *camera, CameraConfiguration *config) override; + + int exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) override; + + int start(Camera *camera, const ControlList *controls) override; + void stop(Camera *camera) override; + + int queueRequestDevice(Camera *camera, Request *request) override; + + bool match(DeviceEnumerator *enumerator) override; + +private: + ISPCameraData *cameraData(const Camera *camera) + { + return static_cast( + PipelineHandler::cameraData(camera)); + } +}; + +ISPCameraConfiguration::ISPCameraConfiguration() + : CameraConfiguration() +{ +} + +CameraConfiguration::Status ISPCameraConfiguration::validate() +{ + Status status = Valid; + + return status; +} + +PipelineHandlerISP::PipelineHandlerISP(CameraManager *manager) + : PipelineHandler(manager) +{ +} + +CameraConfiguration *PipelineHandlerISP::generateConfiguration(Camera *camera, + const StreamRoles &roles) +{ + ISPCameraData *data = cameraData(camera); + CameraConfiguration *config = new ISPCameraConfiguration(); + + if (roles.empty()) + return config; + + std::map> v4l2Formats = + data->video_->formats(); + std::map> deviceFormats; + std::transform(v4l2Formats.begin(), v4l2Formats.end(), + std::inserter(deviceFormats, deviceFormats.begin()), + [&](const decltype(v4l2Formats)::value_type &format) { + return decltype(deviceFormats)::value_type{ + format.first.toPixelFormat(), + format.second + }; + }); + + StreamFormats formats(deviceFormats); + StreamConfiguration cfg(formats); + + cfg.pixelFormat = formats::RGB888; + cfg.size = { 640, 480 }; + cfg.bufferCount = 4; + + config->addConfiguration(cfg); + + config->validate(); + + return config; +} + +int PipelineHandlerISP::configure(Camera *camera, CameraConfiguration *config) +{ + ISPCameraData *data = cameraData(camera); + StreamConfiguration &cfg = config->at(0); + + V4L2VideoDevice::Formats fmts = data->video_->formats(); + V4L2PixelFormat v4l2Format = fmts.begin()->first; + + V4L2DeviceFormat format = {}; + format.fourcc = v4l2Format; + format.size = cfg.size; + + data->width = format.size.width; + data->height = format.size.height; + + int ret = data->video_->setFormat(&format); + if (ret) + return ret; + + cfg.setStream(&data->stream_); + cfg.stride = format.planes[0].bpl; + + return 0; +} + +int PipelineHandlerISP::exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) +{ + unsigned int count = stream->configuration().bufferCount; + ISPCameraData *data = cameraData(camera); + + for (unsigned int i = 0; i < count; i++) { + std::string name = "frame-" + std::to_string(i); + const int isp_fd = memfd_create(name.c_str(), 0); + int ret = ftruncate(isp_fd, data->width * data->height * 3); + if (ret < 0) { + LOG(ISP, Error) + << "truncate: " + << strerror(-ret); + return ret; + } + FileDescriptor temp = FileDescriptor(std::move(isp_fd)); + + FrameBuffer::Plane rgbPlane; + rgbPlane.fd = std::move(temp); + rgbPlane.length = data->width * data->height * 3; + + std::vector planes; + planes.push_back(std::move(rgbPlane)); + std::unique_ptr buffer = std::make_unique(std::move(planes)); + buffers->push_back(std::move(buffer)); + } + + data->tempCopyExportBuffers = buffers; + + return count; + +} + +int PipelineHandlerISP::start(Camera *camera, [[maybe_unused]] const ControlList *controls) +{ + ISPCameraData *data = cameraData(camera); + unsigned int count = data->stream_.configuration().bufferCount; + + data->rawBuffers = new std::vector>; + int ret = data->video_->allocateBuffers(count, data->rawBuffers); + if (ret < 0) { + LOG(ISP, Error) << strerror(-ret); + return ret; + } + + for (unsigned int i = 0; i < count; i++) + data->bufferPair[data->tempCopyExportBuffers->at(i).get()] = data->rawBuffers->at(i).get(); + + ret = data->video_->streamOn(); + if (ret < 0) { + data->video_->releaseBuffers(); + return ret; + } + + data->threadISP_.start(); + + return 0; +} + +void PipelineHandlerISP::stop(Camera *camera) +{ + ISPCameraData *data = cameraData(camera); + + data->threadISP_.exit(); + data->threadISP_.wait(); + + data->video_->streamOff(); + data->video_->releaseBuffers(); +} + +int PipelineHandlerISP::queueRequestDevice(Camera *camera, Request *request) +{ + ISPCameraData *data = cameraData(camera); + FrameBuffer *rgbBuffer = request->findBuffer(&data->stream_); + + if (!rgbBuffer) { + LOG(ISP, Error) << "Attempt to queue request with invalid stream"; + return -ENOENT; + } + + int ret = data->video_->queueBuffer(data->bufferPair[rgbBuffer]); + if (ret < 0) + return ret; + + return 0; +} + +bool PipelineHandlerISP::match(DeviceEnumerator *enumerator) +{ + DeviceMatch unicam("unicam"); + + unicam.add("unicam-embedded"); + unicam.add("unicam-image"); + + MediaDevice *unicam_ = acquireMediaDevice(enumerator, unicam); + if (!unicam_) { + LOG(ISP, Debug) << "unicam Device not found"; + return false; + } + + LOG(ISP, Debug) << "unicam Device Identified"; + + std::unique_ptr data = std::make_unique(this, unicam_); + + if(data->init()) return false; + + std::set streams{&data->stream_}; + std::shared_ptr camera = Camera::create(this, data->video_->deviceName(), streams); + registerCamera(std::move(camera), std::move(data)); + + return true; +} + + +void ISPCameraData::ISPCompleted(FrameBuffer *buffer) +{ + Request *request = buffer->request(); + + pipe_->completeBuffer(request, buffer); + pipe_->completeRequest(request); + +} + +void ISPCameraData::bufferReady(FrameBuffer *buffer) +{ + for (std::map::iterator it = bufferPair.begin(); it != bufferPair.end(); it++) { + if (it->second == buffer) + isp_.invokeMethod(&ISP::processing, ConnectionTypeQueued, buffer, it->first, width, height); + } + +} + +int ISPCameraData::init() +{ + video_ = new V4L2VideoDevice(media_->getEntityByName("unicam-image")); + if (video_->open()) + return -ENODEV; + + video_->bufferReady.connect(this, &ISPCameraData::bufferReady); + + isp_.moveToThread(&threadISP_); + isp_.ispCompleted.connect(this, &ISPCameraData::ISPCompleted); + + return 0; +} + +REGISTER_PIPELINE_HANDLER(PipelineHandlerISP) + +} /* namespace libcamera */