[libcamera-devel,RFC,v2,2/4] libcamera: swisp: Move the softawre ISP process to swisp
diff mbox series

Message ID tencent_BF16B7D9BE760D0F69B2504CCE47A0EBAA08@qq.com
State New
Headers show
Series
  • [libcamera-devel,RFC,v2,1/4] pipeline: isp: The software ISP-based pipeline handler
Related show

Commit Message

Siyuan Fan Aug. 10, 2021, 12:17 p.m. UTC 
From: Fan Siyuan <siyuan.fan@foxmail.com>

 Changes in V2:
 -Design the abstract class ISP, CPU_ISP and future GPU_ISP inherit from it.
 -Add the rgb buf alloc and thread control member function

 Now the ISP must implement the below process:
 BLC-->demosaic-->AWB-->tone mapping-->gamma-->Noise reduction

 Plan to add:
 lens shading correct-->Color correct-->color space transform

Signed-off-by: Fan Siyuan <siyuan.fan@foxmail.com>
---
 .../isp/isp_processing.cpp => swisp/isp.cpp}  | 90 ++++++++++++++-----
 .../isp/isp_processing.h => swisp/isp.h}      | 52 ++++++++---
 2 files changed, 111 insertions(+), 31 deletions(-)
 rename src/libcamera/{pipeline/isp/isp_processing.cpp => swisp/isp.cpp} (87%)
 rename src/libcamera/{pipeline/isp/isp_processing.h => swisp/isp.h} (54%)

Patch
diff mbox series 

diff --git a/src/libcamera/pipeline/isp/isp_processing.cpp b/src/libcamera/swisp/isp.cpp
similarity index 87%
rename from src/libcamera/pipeline/isp/isp_processing.cpp
rename to src/libcamera/swisp/isp.cpp
index 94e8081d..6d073040 100644
--- a/src/libcamera/pipeline/isp/isp_processing.cpp
+++ b/src/libcamera/swisp/isp.cpp
@@ -2,10 +2,10 @@ 
 /*
  * Copyright (C) 2021, Siyuan Fan <siyuan.fan@foxmail.com>
  *
- * isp_processing.cpp - The software ISP class
+ * isp.cpp - The software ISP class
  */
 
-#include "isp_processing.h"
+#include "isp.h"
 
 #include <math.h>
 #include <stdlib.h>
@@ -14,6 +14,7 @@ 
 #include <unistd.h>
 
 #include <libcamera/request.h>
+#include <libcamera/file_descriptor.h>
 
 #include "libcamera/base/log.h"
 
@@ -21,7 +22,7 @@  namespace libcamera{
 
 LOG_DECLARE_CATEGORY(ISP)
 
-void ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height)
+void CPU_ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height)
 {
         int blue, gr, gb, red;
         int histBlue[1024] = {0}, histGb[1024] = {0}, histGr[1024] = {0}, histRed[1024] = {0};
@@ -211,7 +212,7 @@  void ISP::autoContrast(uint16_t *data, float lowCut, float highCut, int width, i
         }
 }
 
-void ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height)
+void CPU_ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height)
 {
         int len = width * height;
         for(int i = 0; i < len; i++) {
@@ -224,7 +225,7 @@  void ISP::blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int heig
         }
 }
 
-void ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
+void CPU_ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
                        int width, int height)
 {
         int index;
@@ -245,7 +246,7 @@  void ISP::readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
         }
 }
 
-void ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
+void CPU_ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
 {
         int index;
         for (int i = 0; i < height; i++) {
@@ -295,7 +296,7 @@  void ISP::firstPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
         }
 }
 
-void ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
+void CPU_ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
 {
         int index;
         for (int i = 0; i < height; i++) {
@@ -340,7 +341,7 @@  void ISP::twoPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
         }
 }
 
-void ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
+void CPU_ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
 {
         int index;
         for (int i = 0; i < height; i++) {
@@ -390,15 +391,15 @@  void ISP::lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height)
         }
 }
 
-void ISP::demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
-                   int width, int height)
+void CPU_ISP::demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
+                       int width, int height)
 {
         firstPixelInsert(data, B, width, height);
         twoPixelInsert(data, G, width, height);
         lastPixelInsert(data, R, width, height);
 }
 
-void ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height)
+void CPU_ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height)
 {
         float aveB = 0, aveG = 0, aveR = 0;
         float Kb, Kg, Kr;
@@ -428,8 +429,7 @@  void ISP::autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int
         }
 }
 
-void ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val,
-                       int width, int height)
+void CPU_ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height)
 {
         float nor = 1.0 / 1023.0;
         float gamma = 1.0 / val;
@@ -444,24 +444,24 @@  void ISP::gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val,
         }
 }
 
-void ISP::compress_10bit_to_8bit (uint16_t *src, uint8_t *dst, int width, int height)
+void CPU_ISP::compress_10bit_to_8bit (uint16_t *src, uint8_t *dst, int width, int height)
 {
         for (int i = 0; i < width * height; i++) {
                 dst[i] = src[i] >> 2 & 0xff;
         }
 }
 
-float ISP::distance(int x, int y, int i, int j)
+float CPU_ISP::distance(int x, int y, int i, int j)
 {
     return float(sqrt(pow(x - i, 2) + pow(y - j, 2)));
 }
 
-double ISP::gaussian(float x, double sigma)
+double CPU_ISP::gaussian(float x, double sigma)
 {
     return exp(-(pow(x, 2)) / (2 * pow(sigma, 2))) / (2 * 3.1415926 * pow(sigma, 2));
 }
 
-void ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B,
+void CPU_ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B,
                           int diameter, double sigmaI,
                           double sigmaS, int width, int height)
 {
@@ -541,7 +541,12 @@  void ISP::bilateralFilter(uint16_t *R, uint16_t *G, uint16_t *B,
         }
 }
 
-void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height)
+void CPU_ISP::noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height)
+{
+        bilateralFilter(R, G, B, 5, 24.0, 32.0, width, height);
+}
+
+void CPU_ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height)
 {
         uint8_t *rgb_buf;
         uint16_t *rawData;
@@ -554,6 +559,7 @@  void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width,
 
         const FrameBuffer::Plane &plane =  srcBuffer->planes()[0]; 
         rawData = (uint16_t *)mmap(NULL, plane.length, PROT_READ|PROT_WRITE, MAP_SHARED, plane.fd.fd(), 0);
+        LOG(ISP, Debug) << plane.length << " " << plane.fd.fd();
         if (rawData == MAP_FAILED) {
             LOG(ISP, Error) << "Read raw data failed";
             ispCompleted.emit(dstBuffer);
@@ -567,7 +573,7 @@  void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width,
         autoContrast(gData, 0.01, 0.01, width, height);
         autoContrast(bData, 0.01, 0.01, width, height);
         gammaCorrect(rData, gData, bData, 2.2, width, height);
-        //bilateralFilter(rData, gData, bData, 5, 24.0, 32.0);
+        //bilateralFilter(rData, gData, bData, 5, 24.0, 32.0, width, height);
     
         const FrameBuffer::Plane &rgbPlane = dstBuffer->planes()[0];
         rgb_buf = (uint8_t *)mmap(NULL, rgbPlane.length, PROT_READ|PROT_WRITE, MAP_SHARED, rgbPlane.fd.fd(), 0);
@@ -584,10 +590,54 @@  void ISP::processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width,
         compress_10bit_to_8bit(gData, gData_8, width, height);
         compress_10bit_to_8bit(bData, bData_8, width, height);
 
+        dstBuffer->metadata_.status = srcBuffer->metadata().status;
+        dstBuffer->metadata_.sequence = srcBuffer->metadata().sequence;
+        dstBuffer->metadata_.timestamp = srcBuffer->metadata().timestamp;
+
+        dstBuffer->metadata_.planes.clear();
+        dstBuffer->metadata_.planes.push_back({rgbPlane.length});
+
         delete[] rgbData;
 
         ispCompleted.emit(dstBuffer);
 }
 
-} /* namespace libcamera */
+int CPU_ISP::exportBuffers(std::vector<std::unique_ptr<FrameBuffer>> *buffers,
+                       unsigned int count, int width, int height)
+{
+        int bufferByte = width * height * 3;
+
+        for (unsigned int i = 0; i < count; i++) {
+                std::string name = "frame-" + std::to_string(i);
+
+                const int isp_fd = memfd_create(name.c_str(), 0);
+                int ret = ftruncate(isp_fd, bufferByte);
+                if (ret < 0) {
+                        LOG(ISP, Error) << "Failed to resize memfd" << strerror(-ret);
+                        return ret;
+                }
+
+                FrameBuffer::Plane rgbPlane;
+                rgbPlane.fd = FileDescriptor(std::move(isp_fd));
+                rgbPlane.length = bufferByte;
+
+                std::vector<FrameBuffer::Plane> planes{rgbPlane};
+                buffers->emplace_back(std::make_unique<FrameBuffer>(std::move(planes)));
+        }
+
+        return count;
+}
+
+void CPU_ISP::startThreadISP()
+{
+        moveToThread(&thread_);
+        thread_.start();
+}
+
+void CPU_ISP::stopThreadISP()
+{
+        thread_.exit();
+        thread_.wait();
+}
 
+} /* namespace libcamera */
\ No newline at end of file
diff --git a/src/libcamera/pipeline/isp/isp_processing.h b/src/libcamera/swisp/isp.h
similarity index 54%
rename from src/libcamera/pipeline/isp/isp_processing.h
rename to src/libcamera/swisp/isp.h
index b80b4218..4099e2c6 100644
--- a/src/libcamera/pipeline/isp/isp_processing.h
+++ b/src/libcamera/swisp/isp.h
@@ -2,15 +2,16 @@ 
 /*
  * Copyright (C) 2021, Siyuan Fan <siyuan.fan@foxmail.com>
  *
- * isp_processing.h - The software ISP class
+ * isp.h - The software ISP class
  */
-#ifndef __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__
-#define __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__
+#ifndef __LIBCAMERA_SWISP_ISP_H__
+#define __LIBCAMERA_SWISP_ISP_H__
 
 #include <libcamera/framebuffer.h>
 
-#include "libcamera/base/signal.h"
 #include "libcamera/base/object.h"
+#include "libcamera/base/signal.h"
+#include "libcamera/base/thread.h"
 
 namespace libcamera{
 
@@ -19,10 +20,29 @@  using std::uint8_t;
 
 class ISP : public Object
 {
+public:
+        ISP() {}
+
+        virtual void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) = 0;
+
+        virtual void demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
+                              int width, int height) = 0;
+
+        virtual void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) = 0;
+
+        virtual void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height)  = 0;
+
+        virtual void noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) = 0;
+
+        virtual ~ISP() {}
+};
+
+class CPU_ISP : public ISP
+{
 public:
         void autoContrast(uint16_t *data, float lowCut, float highCut, int width, int height);
 
-        void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height);
+        void blackLevelCorrect(uint16_t *data, uint16_t offset, int width, int height) override;
 
         void readChannels(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
                           int width, int height);
@@ -34,12 +54,11 @@  public:
         void lastPixelInsert(uint16_t *src, uint16_t *dst, int width, int height);
 
         void demosaic(uint16_t *data, uint16_t *R, uint16_t *G, uint16_t *B,
-                      int width, int height);
+                      int width, int height) override;
 
-        void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height);
+        void autoWhiteBalance(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) override;
 
-        void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val,
-                          int width, int height);
+        void gammaCorrect(uint16_t *R, uint16_t *G, uint16_t *B, float val, int width, int height) override;
 
         float distance(int x, int y, int i, int j);
 
@@ -49,14 +68,25 @@  public:
                              int diameter, double sigmaI, double sigmaS,
                              int width, int height);
 
+        void noiseReduction(uint16_t *R, uint16_t *G, uint16_t *B, int width, int height) override;
+
         void compress_10bit_to_8bit(uint16_t *src, uint8_t *dst, int width, int height);    
 
         void processing(FrameBuffer *srcBuffer, FrameBuffer *dstBuffer, int width, int height);
 
+        int exportBuffers(std::vector<std::unique_ptr<FrameBuffer>> *buffers,
+                           unsigned int count, int width, int height);
+
+        void startThreadISP();
+
+        void stopThreadISP();
+
         Signal<FrameBuffer *> ispCompleted;
 
+private:
+        Thread thread_;
 };
 
-}
+} /* namespace libcamera */
 
-#endif /* __LIBCAMERA_PIPELINE_ISP_PROCESSING_H__ */
+#endif /* __LIBCAMERA_SWISP_ISP_H__ */
\ No newline at end of file