From patchwork Wed Aug 11 06:12:55 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Siyuan Fan X-Patchwork-Id: 13295 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id EA3ACC3241 for ; Wed, 11 Aug 2021 06:13:36 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 16B6B68888; Wed, 11 Aug 2021 08:13:36 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=foxmail.com header.i=@foxmail.com header.b="e+7iYt7R"; dkim-atps=neutral Received: from out203-205-221-173.mail.qq.com (out203-205-221-173.mail.qq.com [203.205.221.173]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id D68D660261 for ; Wed, 11 Aug 2021 08:13:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=foxmail.com; s=s201512; t=1628662405; bh=bKjfddsYLO/vZgypLTtEZ9UOa5oMmmI3W8CdpeQXtD4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=e+7iYt7R2nTM7PybMi1Rwm//paCSPdeeJLb/C1Xr6XKVj4rRt7EWka9bOmuBWQGL8 eapo4zRXvyYFNOXlQouUbsswrDDJ9JbavfoGzdOsc9f/IWOk+q/zMDLtL84LAiFjLI S8Oi/+o7lGxCue0JVofOolrRaB69L2aZQzCMbAxg= Received: from localhost.localdomain ([123.126.82.8]) by newxmesmtplogicsvrszc9.qq.com (NewEsmtp) with SMTP id 3569928F; Wed, 11 Aug 2021 14:13:22 +0800 X-QQ-mid: xmsmtpt1628662404tiw8dybl6 Message-ID: X-QQ-XMAILINFO: OYpbVsTx4C81v8OvB++dVdoZgfaYp5yEeKh9IJvO7g3AYn7XuArdghl0aFh7uo CQb78NSt7Ofkji0GDVUvgXugAGw5m0ip477dAWz0ZS7WgSZ7OQcgyKtbcCNiy6k1YNqZ5QXD0LfQ utPdAggbiqnqFpo2yozuMx+Z9dDdPSZBB0HNC+PbDQ7oqrT7LR+y1hp7mxqCO8Dqt9y7rXJ5ST/2 1N7UUguqDX+gwTXvWj0aPQuHuZbab3Dl2u7SMFUThKZLqY56X108LTyJ/yxBBUISiMy29RFpqEzV lZ9TmW4sQev+sBxelE6QMy6kupHUrvBANzflnPs1Cbg084CAesBGnswj2lUZHcdMLk76uEm+Bai+ H7LMKqaPI9tTAKJ20j0AiVXZTgq1Qet0p7XLz3okQ4s+WPIsLp0/swBKsQ4PMH4LCGNLkLttrMPj aVyIHJaXk7/QF0zYbVBQGZUQF4BIQS1RnApI1BK2Z9gOETkwdjh4RGDc0jiiqFk8ajL1tUGAy8Ht ZYTArmrMu6FLZ4TwA2zDX8wbcYja0dKOHJ1TGefkz1+VBMFAL655HX1XaRMSU9t/G7m0heb4Y0T3 oS6KSADp1ZE1O6EJea6YeqNPpGAhx0ZmQ9+0tr82m+19hW1F5AzH5Yx1WqS6ghjDitOVa04if+Pd Do4Xd1TQXyKxdBuFqeU/O6Cutl4BTOlAHghwbNsbjNt43CCBdkbplurR0scet6s2Rf74TeYs40qE tvvS+76FIV+I0bpLL0HK4wSI+kueQ6AQsobvgpV1zJqrRlMRU9WeN/R/74jsW+kydXKPNZlWEdSQ Y6yS+dHjXQSIbJ7I4uEE16bz626CAjzXit7xyw2V09mC6HrKMaedpGBKPIpsvRwdYOws79HdkKUo Vs7LTq6Nsyqp9ZZ0beWSmZiS9w9HNdwcOdVIR/Wc6bNpa6Cfon1UlXNrejEi3gyecZJIdmhAzliV huWhAlKPB46f7tsWK2jVfNREpa8zREm7ex3vQDenU= From: Siyuan Fan To: libcamera-devel@lists.libcamera.org Date: Wed, 11 Aug 2021 07:12:55 +0100 X-OQ-MSGID: <20210811061258.7421-2-siyuan.fan@foxmail.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210811061258.7421-1-siyuan.fan@foxmail.com> References: <20210811061258.7421-1-siyuan.fan@foxmail.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v2 1/4] pipeline: isp: The software ISP-based pipeline handler X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Fan Siyuan Changes in V2: -fix the raw and rgb data flow based queue model in pipeline handler -move buffer alloc and thread to ISP class -Fill metadata information in dstBuffer Signed-off-by: Fan Siyuan --- src/libcamera/pipeline/isp/isp.cpp | 306 +++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 src/libcamera/pipeline/isp/isp.cpp diff --git a/src/libcamera/pipeline/isp/isp.cpp b/src/libcamera/pipeline/isp/isp.cpp new file mode 100644 index 00000000..c6b7808c --- /dev/null +++ b/src/libcamera/pipeline/isp/isp.cpp @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2021, Siyuan Fan + * + * isp.cpp - The software ISP-based pipeline handler + */ + +#include "../../swisp/isp.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "libcamera/internal/device_enumerator.h" +#include "libcamera/internal/media_device.h" +#include "libcamera/internal/pipeline_handler.h" +#include "libcamera/internal/v4l2_videodevice.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(ISP) + +class ISPCameraData : public CameraData +{ +public: + ISPCameraData(PipelineHandler *pipe, MediaDevice *media) + : CameraData(pipe), media_(media), video_(nullptr) + { + } + + ~ISPCameraData() + { + delete video_; + } + + int init(); + void bufferReady(FrameBuffer *buffer); + void ISPCompleted(FrameBuffer *buffer); + + Stream stream_; + CPU_ISP isp_; + int width_; + int height_; + + std::vector> rawBuffers_; + std::vector rawQueueBuffers_; + std::vector rgbQueueBuffers_; + + MediaDevice *media_; + V4L2VideoDevice *video_; +}; + + +class ISPCameraConfiguration : public CameraConfiguration +{ +public: + ISPCameraConfiguration(); + + Status validate() override; +}; + +class PipelineHandlerISP : public PipelineHandler +{ +public: + PipelineHandlerISP(CameraManager *manager); + + CameraConfiguration *generateConfiguration(Camera *camera, + const StreamRoles &roles) override; + int configure(Camera *camera, CameraConfiguration *config) override; + + int exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) override; + + int start(Camera *camera, const ControlList *controls) override; + void stop(Camera *camera) override; + + int queueRequestDevice(Camera *camera, Request *request) override; + + bool match(DeviceEnumerator *enumerator) override; + +private: + ISPCameraData *cameraData(const Camera *camera) + { + return static_cast( + PipelineHandler::cameraData(camera)); + } +}; + +ISPCameraConfiguration::ISPCameraConfiguration() + : CameraConfiguration() +{ +} + +CameraConfiguration::Status ISPCameraConfiguration::validate() +{ + Status status = Valid; + + return status; +} + +PipelineHandlerISP::PipelineHandlerISP(CameraManager *manager) + : PipelineHandler(manager) +{ +} + +CameraConfiguration *PipelineHandlerISP::generateConfiguration(Camera *camera, + const StreamRoles &roles) +{ + ISPCameraData *data = cameraData(camera); + CameraConfiguration *config = new ISPCameraConfiguration(); + + if (roles.empty()) + return config; + + std::map> v4l2Formats = + data->video_->formats(); + std::map> deviceFormats; + std::transform(v4l2Formats.begin(), v4l2Formats.end(), + std::inserter(deviceFormats, deviceFormats.begin()), + [&](const decltype(v4l2Formats)::value_type &format) { + return decltype(deviceFormats)::value_type{ + format.first.toPixelFormat(), + format.second + }; + }); + + StreamFormats formats(deviceFormats); + StreamConfiguration cfg(formats); + + cfg.pixelFormat = formats::RGB888; + cfg.size = { 640, 480 }; + cfg.bufferCount = 4; + + config->addConfiguration(cfg); + + config->validate(); + + return config; +} + +int PipelineHandlerISP::configure(Camera *camera, CameraConfiguration *config) +{ + ISPCameraData *data = cameraData(camera); + StreamConfiguration &cfg = config->at(0); + + V4L2VideoDevice::Formats fmts = data->video_->formats(); + V4L2PixelFormat v4l2Format = fmts.begin()->first; + + V4L2DeviceFormat format = {}; + format.fourcc = v4l2Format; + format.size = cfg.size; + + data->width_ = format.size.width; + data->height_ = format.size.height; + + int ret = data->video_->setFormat(&format); + if (ret) + return ret; + + cfg.setStream(&data->stream_); + cfg.stride = format.planes[0].bpl; + + return 0; +} + +int PipelineHandlerISP::exportFrameBuffers(Camera *camera, Stream *stream, + std::vector> *buffers) +{ + unsigned int count = stream->configuration().bufferCount; + ISPCameraData *data = cameraData(camera); + + count = data->isp_.exportBuffers(buffers, count, data->width_, data->height_); + + return count; + +} + +int PipelineHandlerISP::start(Camera *camera, [[maybe_unused]] const ControlList *controls) +{ + ISPCameraData *data = cameraData(camera); + unsigned int count = data->stream_.configuration().bufferCount; + + int ret = data->video_->allocateBuffers(count, &data->rawBuffers_); + if (ret < 0) { + LOG(ISP, Error) << strerror(-ret); + return ret; + } + + for (unsigned int i = 0; i < count; i++) { + data->rawQueueBuffers_.push_back(data->rawBuffers_[i].get()); + } + + + ret = data->video_->streamOn(); + if (ret < 0) { + data->video_->releaseBuffers(); + return ret; + } + + data->isp_.startThreadISP(); + + return 0; +} + +void PipelineHandlerISP::stop(Camera *camera) +{ + ISPCameraData *data = cameraData(camera); + + if (!(data->rawBuffers_.empty())) { + data->rawBuffers_.clear(); + } + + data->isp_.stopThreadISP(); + + data->video_->streamOff(); + data->video_->releaseBuffers(); +} + +int PipelineHandlerISP::queueRequestDevice(Camera *camera, Request *request) +{ + ISPCameraData *data = cameraData(camera); + FrameBuffer *rgbBuffer = request->findBuffer(&data->stream_); + data->rgbQueueBuffers_.push_back(rgbBuffer); + + if (!rgbBuffer) { + LOG(ISP, Error) << "Attempt to queue request with invalid stream"; + return -ENOENT; + } + + int ret = data->video_->queueBuffer(data->rawQueueBuffers_[0]); + if (ret < 0) + return ret; + FrameBuffer *temp = data->rawQueueBuffers_[0]; + data->rawQueueBuffers_.erase(data->rawQueueBuffers_.begin()); + data->rawQueueBuffers_.push_back(std::move(temp)); + + return 0; +} + +bool PipelineHandlerISP::match(DeviceEnumerator *enumerator) +{ + DeviceMatch unicam("unicam"); + + unicam.add("unicam-embedded"); + unicam.add("unicam-image"); + + MediaDevice *unicam_ = acquireMediaDevice(enumerator, unicam); + if (!unicam_) { + LOG(ISP, Debug) << "unicam Device not found"; + return false; + } + + LOG(ISP, Debug) << "unicam Device Identified"; + + std::unique_ptr data = std::make_unique(this, unicam_); + + if(data->init()) return false; + + std::set streams{&data->stream_}; + std::shared_ptr camera = Camera::create(this, data->video_->deviceName(), streams); + registerCamera(std::move(camera), std::move(data)); + + return true; +} + + +void ISPCameraData::ISPCompleted(FrameBuffer *buffer) +{ + Request *request = buffer->request(); + + pipe_->completeBuffer(request, buffer); + pipe_->completeRequest(request); + +} + +void ISPCameraData::bufferReady(FrameBuffer *buffer) +{ + LOG(ISP, Debug) << rgbQueueBuffers_[0]->planes()[0].fd.fd(); + isp_.invokeMethod(&CPU_ISP::processing, ConnectionTypeQueued, buffer, rgbQueueBuffers_[0], width_, height_); + rgbQueueBuffers_.erase(rgbQueueBuffers_.begin()); + rgbQueueBuffers_.shrink_to_fit(); + +} + +int ISPCameraData::init() +{ + video_ = new V4L2VideoDevice(media_->getEntityByName("unicam-image")); + if (video_->open()) + return -ENODEV; + + video_->bufferReady.connect(this, &ISPCameraData::bufferReady); + isp_.ispCompleted.connect(this, &ISPCameraData::ISPCompleted); + + return 0; +} + +REGISTER_PIPELINE_HANDLER(PipelineHandlerISP) + +} /* namespace libcamera */