Patch Detail
Show a patch.
GET /api/1.1/patches/20809/?format=api
{ "id": 20809, "url": "https://patchwork.libcamera.org/api/1.1/patches/20809/?format=api", "web_url": "https://patchwork.libcamera.org/patch/20809/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20240806144517.1903397-2-stefan.klug@ideasonboard.com>", "date": "2024-08-06T14:44:41", "name": "[v1,1/1] Documentation: Add first version of the tuning-guide", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": false, "hash": "2c60a502f69e0af20979040890947bc45990740d", "submitter": { "id": 184, "url": "https://patchwork.libcamera.org/api/1.1/people/184/?format=api", "name": "Stefan Klug", "email": "stefan.klug@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/20809/mbox/", "series": [ { "id": 4491, "url": "https://patchwork.libcamera.org/api/1.1/series/4491/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4491", "date": "2024-08-06T14:44:40", "name": "First version of the tuning guide", "version": 1, "mbox": "https://patchwork.libcamera.org/series/4491/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/20809/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/20809/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id EC565C323E\n\tfor <parsemail@patchwork.libcamera.org>;\n\tTue, 6 Aug 2024 14:45:59 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 4F5D663380;\n\tTue, 6 Aug 2024 16:45:59 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[213.167.242.64])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 2757D6337C\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tTue, 6 Aug 2024 16:45:57 +0200 (CEST)", "from ideasonboard.com (unknown\n\t[IPv6:2a00:6020:448c:6c00:c4f5:45bb:f0b0:892])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id CF234B2A;\n\tTue, 6 Aug 2024 16:45:04 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"BGIdOpVN\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1722955504;\n\tbh=hRAv7Gzr8bL/xZOn7K7je5MBVUGewZ+BKFEO8TnBivw=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=BGIdOpVN4bUyklyIvLRNgOGbnBVBlEBKL+3grK7a11vcvr4suXIWfCBCYP3uSWsNs\n\tvYwTW+47n9tKKQRkOBzyzuiaHBDVTYdn+yZDTFt9cKRELOXZLHXiBlD1R1fdYPvy71\n\tMOU2KgcIP78BQ/TpB6a7JRG5RbHsSUyZKBGklXww=", "From": "Stefan Klug <stefan.klug@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Stefan Klug <stefan.klug@ideasonboard.com>", "Subject": "[PATCH v1 1/1] Documentation: Add first version of the tuning-guide", "Date": "Tue, 6 Aug 2024 16:44:41 +0200", "Message-ID": "<20240806144517.1903397-2-stefan.klug@ideasonboard.com>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20240806144517.1903397-1-stefan.klug@ideasonboard.com>", "References": "<20240806144517.1903397-1-stefan.klug@ideasonboard.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "This patch adds a initial version of the tuning guide for libcamera. The\nprocess is established based on the imx8mp and will be improved when we\ndo more tuning for different ISPs with our own tooling.\n\nSigned-off-by: Stefan Klug <stefan.klug@ideasonboard.com>\n---\n Documentation/conf.py | 4 +-\n Documentation/guides/tuning/tuning.rst | 276 +++++++++++++++++++++++++\n Documentation/index.rst | 1 +\n Documentation/meson.build | 1 +\n 4 files changed, 280 insertions(+), 2 deletions(-)\n create mode 100644 Documentation/guides/tuning/tuning.rst", "diff": "diff --git a/Documentation/conf.py b/Documentation/conf.py\nindex 7eeea7f3865b..5387942b9af5 100644\n--- a/Documentation/conf.py\n+++ b/Documentation/conf.py\n@@ -21,8 +21,8 @@\n # -- Project information -----------------------------------------------------\n \n project = 'libcamera'\n-copyright = '2018-2019, The libcamera documentation authors'\n-author = u'Kieran Bingham, Jacopo Mondi, Laurent Pinchart, Niklas Söderlund'\n+copyright = '2018-2024, The libcamera documentation authors'\n+author = u'Kieran Bingham, Jacopo Mondi, Laurent Pinchart, Niklas Söderlund, Stefan Klug'\n \n # Version information is provided by the build environment, through the\n # sphinx command line.\ndiff --git a/Documentation/guides/tuning/tuning.rst b/Documentation/guides/tuning/tuning.rst\nnew file mode 100644\nindex 000000000000..1c30f3f20b4e\n--- /dev/null\n+++ b/Documentation/guides/tuning/tuning.rst\n@@ -0,0 +1,276 @@\n+.. SPDX-License-Identifier: CC-BY-SA-4.0\n+\n+Sensor Tuning Guide\n+===================\n+\n+To create visually good images out of the raw data provided by the camera\n+sensor, a lot of image processing takes place. This is usually done inside an\n+ISP (Image Signal Processor). To be able to do the necessary processing, the\n+corresponding algorithms need to be parameterized according to the used hardware\n+(typically sensor, light and lens). Creating these parameters is called tuning.\n+The tuning process results in a tuning file which is then used by libcamera to\n+parameterize the algorithms at runtime.\n+\n+The processing block of an ISP vary from vendor to vendor and can be\n+arbitrarily complex. Never the less a diagram of the most common and important\n+blocks is shown below:\n+\n+ ::\n+\n+ +--------+ \n+ | Light |\n+ +--------+\n+ |\n+ v\n+ +--------+ +-----+ +-----+ +-----+ +-----+ +-------+\n+ | Sensor | -> | BLC | -> | AWB | -> | LSC | -> | CCM | -> | Gamma |\n+ +--------+ +-----+ +-----+ +-----+ +-----+ +-------+\n+\n+**Light** The light used to light the scene has a crucial influence on the\n+resulting image. The human eye and brain are specialized in automatically\n+adapting to different lights. In a camera this has to be done by an algorithm.\n+Light is a complex topic and to correctly describe light sources you need to\n+describe the spectrum of the light. To simplify things, lights are categorized\n+according to their `color temperature (K)\n+<https://en.wikipedia.org/wiki/Color_temperature>`_. This is important to\n+keep in mind as it means that calibrations may differ between light sources even\n+though they have the same nominal color temperature. For best results the tuning\n+images need to be taken for a complete range of light sources.\n+\n+**Sensor** The sensor captures the incoming light and produces the raw images.\n+Data is commonly filtered by a color filter array into red, green and blue\n+channels. As these filters are not perfect some postprocessing needs to be done\n+to recreate the correct color.\n+\n+**BLC** Black level correction. Even without incoming light a sensor produces\n+pixel values above zero. This is due to a artificially added pedestal value\n+(Which is added to get a evenly distributed noise level around zero instead of\n+only the upper half) and other effects in the electronics light dark currents.\n+To get good images with real black, that black level needs to be subtracted. As\n+that level is typically known for a sensor it is hardcoded in libcamera and does\n+not need any calibration at the moment.\n+\n+**AWB** Auto white balance. For a proper image the color channels need to be\n+adjusted, to get correct white balance. This means that monochrome objects in\n+the scene appear monochrome in the output image (white is white and gray is\n+gray). In libcamera this is done based on a gray world model. No tuning is\n+necessary for this step.\n+\n+**LSC** Lens shading correction. The lens in use has a big influence on the\n+resulting image. The typical effects on the image are lens-shading (also called\n+vignetting). This means that due to the physical properties of a lens the\n+transmission of light falls of to the corners of the lens. To make things even\n+harder, this falloff can be different for different colors/wave lengths. LSC is\n+therefore tuned for a set of light sources and for each color channel\n+individually.\n+\n+**CCM** Color correction matrix. After the previous processing blocks the grays\n+are preserved, but colors are still not correct. This is mostly due to the\n+color temperature of the light source and the imperfections in the color filter\n+array of the sensor. To correct for this a so called color correction matrix is\n+calculated. This is a 3x3 matrix, that is used to optimize the captured colors\n+regarding the perceived color error. To do this a chart of known and precisely\n+measured colors (macbeth chart) is captured. Then the matrix is calculated\n+using linear optimization to best map each measured colour of the chart to it's\n+known value. The color error is measured in `deltaE\n+<https://en.wikipedia.org/wiki/Color_difference>`_.\n+\n+**Gamma** Gamma correction. Today's display usually apply a gamma of 2.2 for\n+display. For images to be perceived correctly by the human eye, they need to be\n+encoded with the corresponding inverse gamma. See also\n+<https://en.wikipedia.org/wiki/Gamma_correction>. This block doesn't need\n+tuning, but is crucial for correct visual display.\n+\n+Materials needed for the tuning process\n+---------------------------------------\n+\n+Precisely calibrated optical equipment is very expensive and out of the scope of\n+this document. Still it is possible to get reasonably good calibration results\n+at little costs. The most important devices needed are: \n+\n+ - A light box with the ability to produce defined light of different color\n+ temperatures. Typical temperatures used for calibration are 2400K\n+ (incandescent), 2700K (fluorescent), 5000K (daylight fluorescent), 6500K\n+ (daylight). As a first test, keylights for webcam streaming can be used.\n+ These are available with support for color temperatures ranging from 2500K\n+ to 9000K. For better results professional light boxes are needed.\n+ - A ColorChecker chart. These are sold from calibrite and it makes sense to\n+ get the original one.\n+ - A integration sphere. This is used to create completely homogenious light\n+ for the lens shading calibration. We had good results with the use of\n+ a homogeneous area light (the above mentioned keylight).\n+ - An environment without external light sources. Ideally calibration is done\n+ without any external lights in a fully dark room. A black curtain is one\n+ solution, working by night is the cheap alternative.\n+\n+\n+Overview over the process \n+-------------------------\n+\n+The tuning process of libcamera consists of the following steps:\n+\n+ 1. Take images for the tuning steps with different color temperatures\n+ 2. Configure the tuning process\n+ 3. Run the tuning script to create a corresponding tuning file\n+\n+\n+Taking raw images with defined exposure/gain\n+--------------------------------------------\n+\n+For all the tuning files you need to capture a raw image with a defined exposure\n+time and gain. It is crucial that no pixels are saturated on any channel. At the\n+same time the images should not be too dark. Strive for a exposure time, so that\n+the brightest pixel is roughly 90% saturated. Finding the correct exposure time\n+is currently a manual process. We are working on solutions to aid in that\n+process. After finding the correct exposure time settings, the easiest way to\n+capture a raw image is with the ``cam`` application that gets installed with the\n+regular libcamera install.\n+\n+Create a ``constant-exposure.yaml`` file with the following content:\n+\n+.. code:: yaml\n+\n+ frames:\n+ - 0:\n+ AeEnable: false\n+ ExposureTime: 1000 \n+\n+Then capture one or more images using the following command:\n+ ``cam -c 1 --script constant-exposure.yaml -C -s role=raw --file=image_#.dng``\n+ \n+\n+Capture images for lens shading correction\n+------------------------------------------\n+\n+To be able to correct lens shading artifacts, images of a homogeneously lit\n+neutral gray background are needed. Ideally a integration sphere is used for\n+that task.\n+\n+As a fallback images of a area light or a white wall can be used. If there are\n+multiple images for the same color temperature, the tuning scripts will average\n+the tuning results which can further improve the tuning quality.\n+\n+.. Note:: Currently lsc is ignored in the ccm calculation. This can lead to\n+ slight color deviations and will be improved in the future.\n+\n+Images shall be taken for multiple color temperatures and named\n+``alsc_<temperature>k_#.dng``. ``#`` can be any number, if multiple images are\n+taken for the same color temperature.\n+\n+.. figure:: img/setup_lens_shading.jpg\n+ :width: 50%\n+\n+ Sample setup using an area light. Due to the wide angle lens the camera has\n+ to moved closely in front of the light. This is not a perfect solution as\n+ angle to the light might get way too large.\n+\n+.. figure:: img/camshark-alsc.png\n+ :width: 100%\n+\n+ A calibration image for lens shading correction.\n+ \n+- Ensure the full sensor is lit. Especially with wide angle lenses it may be\n+ difficult to get the full field of view lit homogeneously.\n+- Ensure no color channel is saturated (can be checked with camshark)\n+\n+Take a raw dng image for multiple color temperatures:\n+ ``cam -c 1 --script constant-exposure.yaml -C -s role=raw --file=alsc_3200k_#.dng``\n+\n+\n+\n+Capture images for color calibration\n+------------------------------------\n+\n+To do the color calibration, raw images of the color checker need to be taken\n+for different light sources. These need to be named\n+``<sensor_name>_<lux-level>l_<temperature>k_0.dng``.\n+\n+For best results the following hints should be taken into account:\n+ - Ensure that the 18% gray patch (third from the right in bottom line) is\n+ roughly at 18% saturation for all channels (a mean of 16-20% should be fine)\n+ - Ensure the color checker is homogeneously lit (ideally from 45 degree above)\n+ - No straylight from other light sources is present\n+ - The color checker is not too small and not too big (so that neither lens\n+ shading artifacts nor lens distortions are prevailing in the area of the\n+ color chart)\n+\n+If no lux meter is at hand to precisely measure the lux level, a lux meter app\n+on a mobile phone can provide a sufficient estimation.\n+\n+.. figure:: img/setup_calibration2.jpg\n+ :width: 50%\n+\n+\n+Run the tuning scripts\n+----------------------\n+\n+After taking the calibration images, you should have a directory with all the\n+tuning files. It should look something like this:\n+\n+ ::\n+\n+ ../tuning-data/\n+ ├── alsc_2500k_0.dng\n+ ├── alsc_2500k_1.dng\n+ ├── alsc_2500k_2.dng\n+ ├── alsc_6500k_0.dng\n+ ├── imx335_1000l_2500k_0.dng\n+ ├── imx335_1200l_4000k_0.dng\n+ ├── imx335_1600l_6000k_0.dng\n+\n+\n+The tuning scripts are part of the libcamera source tree. After cloning the\n+libcamera sources the necessary steps to create a tuning file are:\n+\n+ ::\n+ \n+ # install the necessary python packages\n+ cd libcamera\n+ python -m venv venv\n+ source ./venv/bin/activate\n+ pip install -r utils/tuning/requirements.txt\n+\n+ # run the tuning script\n+ utils/tuning/rkisp1.py -c config.yaml -i ../tuning-data/ -o tuning-file.yaml\n+\n+After the tuning script has run, the tuning file can be tested with any\n+libcamera based application like `qcam`. To quickly switch to a specific tuning\n+file, the environment variable ``LIBCAMERA_<pipeline>_TUNING_FILE`` is helpful.\n+E.g.:\n+\n+ ::\n+\n+ LIBCAMERA_RKISP1_TUNING_FILE=/path/to/tuning-file.yaml qcam -c1\n+\n+\n+Sample images\n+-------------\n+\n+\n+.. figure:: img/image-no-blc.png\n+ :width: 800\n+\n+ Image without black level correction (and completely invalid color estimation).\n+\n+.. figure:: img/image-no-lsc-3000.png\n+ :width: 800\n+\n+ Image without lens shading correction @ 3000k. The vignetting artifacts can\n+ be clearly seen \n+\n+.. figure:: img/image-no-lsc-6000.png\n+ :width: 800\n+\n+ Image without lens shading correction @ 6000k. The vignetting artifacts can\n+ be clearly seen\n+\n+.. figure:: img/image-fully-tuned-3000.png\n+ :width: 800\n+\n+ Fully tuned image @ 3000k\n+\n+.. figure:: img/image-fully-tuned-6000.png\n+ :width: 800\n+\n+ Fully tuned image @ 6000k\n+ \ndiff --git a/Documentation/index.rst b/Documentation/index.rst\nindex 5442ae75dde7..991dcf2b66fb 100644\n--- a/Documentation/index.rst\n+++ b/Documentation/index.rst\n@@ -16,6 +16,7 @@\n \n Developer Guide <guides/introduction>\n Application Writer's Guide <guides/application-developer>\n+ Sensor Tuning Guide <guides/tuning/tuning>\n Pipeline Handler Writer's Guide <guides/pipeline-handler>\n IPA Writer's guide <guides/ipa>\n Tracing guide <guides/tracing>\ndiff --git a/Documentation/meson.build b/Documentation/meson.build\nindex 30d395234952..471eabcac344 100644\n--- a/Documentation/meson.build\n+++ b/Documentation/meson.build\n@@ -77,6 +77,7 @@ if sphinx.found()\n 'guides/ipa.rst',\n 'guides/pipeline-handler.rst',\n 'guides/tracing.rst',\n+ 'guides/tuning/tuning.rst',\n 'index.rst',\n 'lens_driver_requirements.rst',\n 'python-bindings.rst',\n", "prefixes": [ "v1", "1/1" ] }