From patchwork Tue Aug 10 12:17:40 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Siyuan Fan X-Patchwork-Id: 13282 Return-Path: X-Original-To: parsemail@patchwork.libcamera.org Delivered-To: parsemail@patchwork.libcamera.org Received: from lancelot.ideasonboard.com (lancelot.ideasonboard.com [92.243.16.209]) by patchwork.libcamera.org (Postfix) with ESMTPS id 7EE4CC3241 for ; Tue, 10 Aug 2021 12:17:58 +0000 (UTC) Received: from lancelot.ideasonboard.com (localhost [IPv6:::1]) by lancelot.ideasonboard.com (Postfix) with ESMTP id 4EE786888B; Tue, 10 Aug 2021 14:17:57 +0200 (CEST) Authentication-Results: lancelot.ideasonboard.com; dkim=fail reason="signature verification failed" (1024-bit key; unprotected) header.d=foxmail.com header.i=@foxmail.com header.b="fF3RuIun"; dkim-atps=neutral Received: from out203-205-221-202.mail.qq.com (out203-205-221-202.mail.qq.com [203.205.221.202]) by lancelot.ideasonboard.com (Postfix) with ESMTPS id 32E34687F0 for ; Tue, 10 Aug 2021 14:17:53 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=foxmail.com; s=s201512; t=1628597870; bh=MpPBdLHjXJ3P7/mkUaYI/7v+TX5Yp4xG20oBoUfcXag=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=fF3RuIunO9WbaEiD94szOF0bfKRXXrtKJE00UHYDsmLqP4403jzDOgzXVr2WleVSD WREsdW2hJ9sLUzPJgbaqYd3Y8IwD4got/JY6mjHp0g6fXYK1VxZBTDlOwiw7uQW0Qu EAl5bC/U5/GBZMP+qrIAVagmy1VQnajiAQssauqE= Received: from localhost.localdomain ([123.126.82.8]) by newxmesmtplogicsvrsza5.qq.com (NewEsmtp) with SMTP id 46D0AC53; Tue, 10 Aug 2021 20:17:45 +0800 X-QQ-mid: xmsmtpt1628597868ts7fxlrfm Message-ID: X-QQ-XMAILINFO: N8r1CgRsoLeWHmeMH9snNJkGbAYLkgsf33FaKFQrjfxE98ni6AGtKCckvU8D31 we5wOK1LGXcKZ37JqIz/OdqiKds1XgkR37r7eBtucS9HRQqzB6mZFx7wtkhau+s7YeXUpA2BylZp C6yDwn7tCNvMwPewwG23uQLrMGETuM5lMwwoyH+7FKDUs5aIX94i77oiEEgfGp5BNSow/x3v/FhU hurZ1JrIgyeQk+haIEXyf4TawxY6orfRT6AynIwoQqUzog9kmzL7itx0d63ZO+MYg6523mXs+kRv zpgYupEAhfV1NWufWIW4ugPeS81ISNwgrFESFHwKpj7KjPkJ23YGmtEvFlsycN5VoO8TyNBZhlK/ EAtYQlH2b7QOoKVoRdMVp7dkqnmMtCVIGZDO3gb9qFkos9cNmUSzmUducu4FhLWSIe7kfku/m7mr tMi27W5KlcsxOYm1WKHAwO1fpyPxWwENMC5WDPAT6O9nTqFgafmm2QV4b7zFIEtK/UbwZQVp4Rh+ mp7nrNNum0GSekRGooWDCppdTil4v+tftWIuNHxnvtcA767b/grafAPUpc/bRT/q+hZ9GNXNZeMs qsQO+Y5BhjHZJrwdjx5m+WzrLxdQIDu5ZKeMwWMvBXtpx8PuR6XGuqFrcaksz7YNAADmmIgubBss xs61i+1FXJtILs7I4Du6cIerDH+NDFEczcyip0ETdPcWhJYhuiXmfANT/JLa/Pc1uAYkeIBwLmmL 2rQogckLks9RBscwsafBg42+MGhRFLPIV7sNDxwB4ytxFewgFKv8UYkxBhZFDOMxaTfn3CUMckDj A7DsHRP13oeoeyzxzetwJ89YoDV9Opz6evxDinvofEYjlfTwWd7Ch61sf7fPCpWRlcmjX0piOeZs BtwF6tOTKyyOE2e8IhkNZYhqt5Mv9dmmZDs1UojliEYO1ZAenrCEt3url9Bo3ogCTqLi7I7ALIaM hajhz/r++caEQyrMNJ0iW1L0J7UhwU8HFTtlAlIOlllKTvjht0REKFqs7wlD5/jPaxz5GtQgQ= From: Siyuan Fan To: libcamera-devel@lists.libcamera.org Date: Tue, 10 Aug 2021 13:17:40 +0100 X-OQ-MSGID: <20210810121742.11012-3-siyuan.fan@foxmail.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210810121742.11012-1-siyuan.fan@foxmail.com> References: <20210810121742.11012-1-siyuan.fan@foxmail.com> MIME-Version: 1.0 Subject: [libcamera-devel] [RFC PATCH v2 2/4] libcamera: swisp: Move the softawre ISP process to swisp X-BeenThere: libcamera-devel@lists.libcamera.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: libcamera-devel-bounces@lists.libcamera.org Sender: "libcamera-devel" From: Fan Siyuan Changes in V2: -Design the abstract class ISP, CPU_ISP and future GPU_ISP inherit from it. -Add the rgb buf alloc and thread control member function Now the ISP must implement the below process: BLC-->demosaic-->AWB-->tone mapping-->gamma-->Noise reduction Plan to add: lens shading correct-->Color correct-->color space transform Signed-off-by: Fan Siyuan --- .../isp/isp_processing.cpp => swisp/isp.cpp} | 90 ++++++++++++++----- .../isp/isp_processing.h => swisp/isp.h} | 52 ++++++++--- 2 files changed, 111 insertions(+), 31 deletions(-) rename src/libcamera/{pipeline/isp/isp_processing.cpp => swisp/isp.cpp} (87%) rename src/libcamera/{pipeline/isp/isp_processing.h => swisp/isp.h} (54%) diff --git a/src/libcamera/pipeline/isp/isp_processing.cpp b/src/libcamera/swisp/isp.cpp similarity index 87% rename from src/libcamera/pipeline/isp/isp_processing.cpp rename to src/libcamera/swisp/isp.cpp index 94e8081d..6d073040 100644 --- a/src/libcamera/pipeline/isp/isp_processing.cpp +++ b/src/libcamera/swisp/isp.cpp @@ -2,10 +2,10 @@ /* * Copyright (C) 2021, Siyuan Fan * - * isp_processing.cpp - The software ISP class + * isp.cpp - The software ISP class */ -#include "isp_processing.h" +#include "isp.h" #include #include @@ -14,6 +14,7 @@ #include #include +#include #include "libcamera/base/log.h" @@ -21,7 +22,7 @@ namespace libcamera{ LOG_DECLARE_CATEGORY(ISP) -void ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height) +void CPU_ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height) { int blue, gr, gb, red; int histBlue[1024] = {0}, histGb[1024] = {0}, histGr[1024] = {0}, histRed[1024] = {0}; @@ -211,7 +212,7 @@ void ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, i } } -void ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) +void CPU_ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) { int len = width * height; for(int i = 0; i < len; i++) { @@ -224,7 +225,7 @@ void ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int heig } } -void ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, +void CPU_ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) { int index; @@ -245,7 +246,7 @@ void ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, } } -void ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) +void CPU_ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) { int index; for (int i = 0; i < height; i++) { @@ -295,7 +296,7 @@ void ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) } } -void ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) +void CPU_ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) { int index; for (int i = 0; i < height; i++) { @@ -340,7 +341,7 @@ void ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) } } -void ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) +void CPU_ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) { int index; for (int i = 0; i < height; i++) { @@ -390,15 +391,15 @@ void ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height) } } -void ISP::demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, - int width, int height) +void CPU_ISP::demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, + int width, int height) { firstPixelInsert(data, B, width, height); twoPixelInsert(data, G, width, height); lastPixelInsert(data, R, width, height); } -void ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) +void CPU_ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) { float aveB = 0, aveG = 0, aveR = 0; float Kb, Kg, Kr; @@ -428,8 +429,7 @@ void ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int } } -void ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, - int width, int height) +void CPU_ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height) { float nor = 1.0 / 1023.0; float gamma = 1.0 / val; @@ -444,24 +444,24 @@ void ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, } } -void ISP::compress_10bit_to_8bit (uint16_t *src, uint8_t *dst, int width, int height) +void CPU_ISP::compress_10bit_to_8bit (uint16_t *src, uint8_t *dst, int width, int height) { for (int i = 0; i < width * height; i++) { dst[i] = src[i] >> 2 & 0xff; } } -float ISP::distance(int x, int y, int i, int j) +float CPU_ISP::distance(int x, int y, int i, int j) { return float(sqrt(pow(x - i, 2) + pow(y - j, 2))); } -double ISP::gaussian(float x, double sigma) +double CPU_ISP::gaussian(float x, double sigma) { return exp(-(pow(x, 2)) / (2 * pow(sigma, 2))) / (2 * 3.1415926 * pow(sigma, 2)); } -void ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B, +void CPU_ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B, int diameter, double sigmaI, double sigmaS, int width, int height) { @@ -541,7 +541,12 @@ void ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B, } } -void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height) +void CPU_ISP::noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) +{ + bilateralFilter(R, G, B, 5, 24.0, 32.0, width, height); +} + +void CPU_ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height) { uint8_t *rgb_buf; uint16_t *rawData; @@ -554,6 +559,7 @@ void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, const FrameBuffer::Plane &plane = srcBuffer->planes()[0]; rawData = (uint16_t *)mmap(NULL, plane.length, PROT_READ|PROT_WRITE, MAP_SHARED, plane.fd.fd(), 0); + LOG(ISP, Debug) << plane.length << " " << plane.fd.fd(); if (rawData == MAP_FAILED) { LOG(ISP, Error) << "Read raw data failed"; ispCompleted.emit(dstBuffer); @@ -567,7 +573,7 @@ void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, autoContrast(gData, 0.01, 0.01, width, height); autoContrast(bData, 0.01, 0.01, width, height); gammaCorrect(rData, gData, bData, 2.2, width, height); - //bilateralFilter(rData, gData, bData, 5, 24.0, 32.0); + //bilateralFilter(rData, gData, bData, 5, 24.0, 32.0, width, height); const FrameBuffer::Plane &rgbPlane = dstBuffer->planes()[0]; rgb_buf = (uint8_t *)mmap(NULL, rgbPlane.length, PROT_READ|PROT_WRITE, MAP_SHARED, rgbPlane.fd.fd(), 0); @@ -584,10 +590,54 @@ void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, compress_10bit_to_8bit(gData, gData_8, width, height); compress_10bit_to_8bit(bData, bData_8, width, height); + dstBuffer->metadata_.status = srcBuffer->metadata().status; + dstBuffer->metadata_.sequence = srcBuffer->metadata().sequence; + dstBuffer->metadata_.timestamp = srcBuffer->metadata().timestamp; + + dstBuffer->metadata_.planes.clear(); + dstBuffer->metadata_.planes.push_back({rgbPlane.length}); + delete[] rgbData; ispCompleted.emit(dstBuffer); } -} /* namespace libcamera */ +int CPU_ISP::exportBuffers(std::vector> *buffers, + unsigned int count, int width, int height) +{ + int bufferByte = width * height * 3; + + for (unsigned int i = 0; i < count; i++) { + std::string name = "frame-" + std::to_string(i); + + const int isp_fd = memfd_create(name.c_str(), 0); + int ret = ftruncate(isp_fd, bufferByte); + if (ret < 0) { + LOG(ISP, Error) << "Failed to resize memfd" << strerror(-ret); + return ret; + } + + FrameBuffer::Plane rgbPlane; + rgbPlane.fd = FileDescriptor(std::move(isp_fd)); + rgbPlane.length = bufferByte; + + std::vector planes{rgbPlane}; + buffers->emplace_back(std::make_unique(std::move(planes))); + } + + return count; +} + +void CPU_ISP::startThreadISP() +{ + moveToThread(&thread_); + thread_.start(); +} + +void CPU_ISP::stopThreadISP() +{ + thread_.exit(); + thread_.wait(); +} +} /* namespace libcamera */ \ No newline at end of file diff --git a/src/libcamera/pipeline/isp/isp_processing.h b/src/libcamera/swisp/isp.h similarity index 54% rename from src/libcamera/pipeline/isp/isp_processing.h rename to src/libcamera/swisp/isp.h index b80b4218..4099e2c6 100644 --- a/src/libcamera/pipeline/isp/isp_processing.h +++ b/src/libcamera/swisp/isp.h @@ -2,15 +2,16 @@ /* * Copyright (C) 2021, Siyuan Fan * - * isp_processing.h - The software ISP class + * isp.h - The software ISP class */ -#ifndef __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__ -#define __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__ +#ifndef __LIBCAMERA_SWISP_ISP_H__ +#define __LIBCAMERA_SWISP_ISP_H__ #include -#include "libcamera/base/signal.h" #include "libcamera/base/object.h" +#include "libcamera/base/signal.h" +#include "libcamera/base/thread.h" namespace libcamera{ @@ -19,10 +20,29 @@ using std::uint8_t; class ISP : public Object { +public: + ISP() {} + + virtual void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) = 0; + + virtual void demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, + int width, int height) = 0; + + virtual void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) = 0; + + virtual void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height) = 0; + + virtual void noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) = 0; + + virtual ~ISP() {} +}; + +class CPU_ISP : public ISP +{ public: void autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height); - void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height); + void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) override; void readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, int width, int height); @@ -34,12 +54,11 @@ public: void lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height); void demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B, - int width, int height); + int width, int height) override; - void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height); + void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) override; - void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, - int width, int height); + void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height) override; float distance(int x, int y, int i, int j); @@ -49,14 +68,25 @@ public: int diameter, double sigmaI, double sigmaS, int width, int height); + void noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) override; + void compress_10bit_to_8bit(uint16_t *src, uint8_t *dst, int width, int height); void processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height); + int exportBuffers(std::vector> *buffers, + unsigned int count, int width, int height); + + void startThreadISP(); + + void stopThreadISP(); + Signal ispCompleted; +private: + Thread thread_; }; -} +} /* namespace libcamera */ -#endif /* __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__ */ +#endif /* __LIBCAMERA_SWISP_ISP_H__ */ \ No newline at end of file