Patch Detail
Show a patch.
GET /api/1.1/patches/20906/?format=api
{ "id": 20906, "url": "https://patchwork.libcamera.org/api/1.1/patches/20906/?format=api", "web_url": "https://patchwork.libcamera.org/patch/20906/", "project": { "id": 1, "url": "https://patchwork.libcamera.org/api/1.1/projects/1/?format=api", "name": "libcamera", "link_name": "libcamera", "list_id": "libcamera_core", "list_email": "libcamera-devel@lists.libcamera.org", "web_url": "", "scm_url": "", "webscm_url": "" }, "msgid": "<20240814074013.52693-2-stefan.klug@ideasonboard.com>", "date": "2024-08-14T07:40:04", "name": "[v2,1/1] Documentation: Add first version of the tuning-guide", "commit_ref": null, "pull_url": null, "state": "new", "archived": false, "hash": "89ac43f1a35a0cda078d37139309cc6d8c34e5e0", "submitter": { "id": 184, "url": "https://patchwork.libcamera.org/api/1.1/people/184/?format=api", "name": "Stefan Klug", "email": "stefan.klug@ideasonboard.com" }, "delegate": null, "mbox": "https://patchwork.libcamera.org/patch/20906/mbox/", "series": [ { "id": 4517, "url": "https://patchwork.libcamera.org/api/1.1/series/4517/?format=api", "web_url": "https://patchwork.libcamera.org/project/libcamera/list/?series=4517", "date": "2024-08-14T07:40:03", "name": "First version of the tuning guide", "version": 2, "mbox": "https://patchwork.libcamera.org/series/4517/mbox/" } ], "comments": "https://patchwork.libcamera.org/api/patches/20906/comments/", "check": "pending", "checks": "https://patchwork.libcamera.org/api/patches/20906/checks/", "tags": {}, "headers": { "Return-Path": "<libcamera-devel-bounces@lists.libcamera.org>", "X-Original-To": "parsemail@patchwork.libcamera.org", "Delivered-To": "parsemail@patchwork.libcamera.org", "Received": [ "from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id A761FBDB13\n\tfor <parsemail@patchwork.libcamera.org>;\n\tWed, 14 Aug 2024 07:40:24 +0000 (UTC)", "from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 37835633BD;\n\tWed, 14 Aug 2024 09:40:24 +0200 (CEST)", "from perceval.ideasonboard.com (perceval.ideasonboard.com\n\t[IPv6:2001:4b98:dc2:55:216:3eff:fef7:d647])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 3EB6C63398\n\tfor <libcamera-devel@lists.libcamera.org>;\n\tWed, 14 Aug 2024 09:40:22 +0200 (CEST)", "from ideasonboard.com (unknown\n\t[IPv6:2a00:6020:448c:6c00:8a6:aa2:ebee:5ae5])\n\tby perceval.ideasonboard.com (Postfix) with ESMTPSA id 876C16B5;\n\tWed, 14 Aug 2024 09:39:24 +0200 (CEST)" ], "Authentication-Results": "lancelot.ideasonboard.com; dkim=pass (1024-bit key;\n\tunprotected) header.d=ideasonboard.com header.i=@ideasonboard.com\n\theader.b=\"ey8tFd0e\"; dkim-atps=neutral", "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/simple; d=ideasonboard.com;\n\ts=mail; t=1723621164;\n\tbh=DsfzGOGRUlvxaK3vwO41F06C+JbZN+x3scwjgCuTgPQ=;\n\th=From:To:Cc:Subject:Date:In-Reply-To:References:From;\n\tb=ey8tFd0eYycMcNNmRNx6Xc5NU/LqcOxhWme3a+zkx4EQ518Q30RahDda/X4E8ZU+N\n\tNFprhFXdmF9fVJOAgnwjWwGkxOIRvRxlzaa3o852UYiRF44mksabg7Rthu8wcpEebx\n\tbPe3hybvKcdKiLaMJAMX5S+sHHpZZEjVcnPfxUC0=", "From": "Stefan Klug <stefan.klug@ideasonboard.com>", "To": "libcamera-devel@lists.libcamera.org", "Cc": "Stefan Klug <stefan.klug@ideasonboard.com>", "Subject": "[PATCH v2 1/1] Documentation: Add first version of the tuning-guide", "Date": "Wed, 14 Aug 2024 09:40:04 +0200", "Message-ID": "<20240814074013.52693-2-stefan.klug@ideasonboard.com>", "X-Mailer": "git-send-email 2.43.0", "In-Reply-To": "<20240814074013.52693-1-stefan.klug@ideasonboard.com>", "References": "<20240814074013.52693-1-stefan.klug@ideasonboard.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "libcamera-devel@lists.libcamera.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "<libcamera-devel.lists.libcamera.org>", "List-Unsubscribe": "<https://lists.libcamera.org/options/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=unsubscribe>", "List-Archive": "<https://lists.libcamera.org/pipermail/libcamera-devel/>", "List-Post": "<mailto:libcamera-devel@lists.libcamera.org>", "List-Help": "<mailto:libcamera-devel-request@lists.libcamera.org?subject=help>", "List-Subscribe": "<https://lists.libcamera.org/listinfo/libcamera-devel>,\n\t<mailto:libcamera-devel-request@lists.libcamera.org?subject=subscribe>", "Errors-To": "libcamera-devel-bounces@lists.libcamera.org", "Sender": "\"libcamera-devel\" <libcamera-devel-bounces@lists.libcamera.org>" }, "content": "This patch adds a initial version of the tuning guide for libcamera. The\nprocess is established based on the imx8mp and will be improved when we\ndo more tuning for different ISPs with our own tooling.\n\nSigned-off-by: Stefan Klug <stefan.klug@ideasonboard.com>\n---\n Documentation/conf.py | 4 +-\n Documentation/guides/tuning/tuning.rst | 291 +++++++++++++++++++++++++\n Documentation/index.rst | 1 +\n Documentation/meson.build | 1 +\n 4 files changed, 295 insertions(+), 2 deletions(-)\n create mode 100644 Documentation/guides/tuning/tuning.rst", "diff": "diff --git a/Documentation/conf.py b/Documentation/conf.py\nindex 7eeea7f3865b..5387942b9af5 100644\n--- a/Documentation/conf.py\n+++ b/Documentation/conf.py\n@@ -21,8 +21,8 @@\n # -- Project information -----------------------------------------------------\n \n project = 'libcamera'\n-copyright = '2018-2019, The libcamera documentation authors'\n-author = u'Kieran Bingham, Jacopo Mondi, Laurent Pinchart, Niklas Söderlund'\n+copyright = '2018-2024, The libcamera documentation authors'\n+author = u'Kieran Bingham, Jacopo Mondi, Laurent Pinchart, Niklas Söderlund, Stefan Klug'\n \n # Version information is provided by the build environment, through the\n # sphinx command line.\ndiff --git a/Documentation/guides/tuning/tuning.rst b/Documentation/guides/tuning/tuning.rst\nnew file mode 100644\nindex 000000000000..a58dda350556\n--- /dev/null\n+++ b/Documentation/guides/tuning/tuning.rst\n@@ -0,0 +1,291 @@\n+.. SPDX-License-Identifier: CC-BY-SA-4.0\n+\n+Sensor Tuning Guide\n+===================\n+\n+To create visually good images from the raw data provided by the camera sensor,\n+a lot of image processing takes place. This is usually done inside an ISP (Image\n+Signal Processor). To be able to do the necessary processing, the corresponding\n+algorithms need to be parameterized according to the hardware in use (typically\n+sensor, light and lens). Calculating these parameters is a process called\n+tuning. The tuning process results in a tuning file which is then used by\n+libcamera to provide calibrated parameters to the algorithms at runtime.\n+\n+The processing blocks of an ISP vary from vendor to vendor and can be\n+arbitrarily complex. Nevertheless a diagram of common blocks frequently found in\n+an ISP design is shown below:\n+\n+ ::\n+\n+ +--------+ \n+ | Light |\n+ +--------+\n+ |\n+ v\n+ +--------+ +-----+ +-----+ +-----+ +-----+ +-------+\n+ | Sensor | -> | BLC | -> | AWB | -> | LSC | -> | CCM | -> | Gamma |\n+ +--------+ +-----+ +-----+ +-----+ +-----+ +-------+\n+\n+**Light** The light used to light the scene has a crucial influence on the\n+resulting image. The human eye and brain are specialized in automatically\n+adapting to different lights. In a camera this has to be done by an algorithm.\n+Light is a complex topic and to correctly describe light sources you need to\n+describe the spectrum of the light. To simplify things, lights are categorized\n+according to their `color temperature (K)\n+<https://en.wikipedia.org/wiki/Color_temperature>`_. This is important to\n+keep in mind as it means that calibrations may differ between light sources even\n+though they have the same nominal color temperature.\n+\n+For best results the tuning images need to be taken for a complete range of\n+light sources.\n+\n+**Sensor** The sensor captures the incoming light and converts a measured\n+analogue signal to a digital representation which conveys the raw images. The\n+light is commonly filtered by a color filter array into red, green and blue\n+channels. As these filters are not perfect some postprocessing needs to be done\n+to recreate the correct color.\n+\n+**BLC** Black level correction. Even without incoming light a sensor produces\n+pixel values above zero. This is due to two system artifacts that impact the\n+measured light levels on the sensor. Firstly, a deliberate and artificially\n+added pedestal value is added to get an evenly distributed noise level around\n+zero to avoid negative values and clipping the data. \n+\n+Secondly, additional underlying electrical noise can be caused by various\n+external factors including thermal noise and electrical interferences. \n+\n+To get good images with real black, that black level needs to be subtracted. As\n+that level is typically known for a sensor it is hardcoded in libcamera and does\n+not need any calibration at the moment. If needed, that value can be manually\n+overwritten in the tuning configuration.\n+\n+**AWB** Auto white balance. For a proper image the color channels need to be\n+adjusted, to get correct white balance. This means that monochrome objects in\n+the scene appear monochrome in the output image (white is white and gray is\n+gray). Presently in the libipa implementation of libcamera, this is managed by a\n+grey world model. No tuning is necessary for this step.\n+\n+**LSC** Lens shading correction. The lens in use has a big influence on the\n+resulting image. The typical effects on the image are lens-shading (also called\n+vignetting). This means that due to the physical properties of a lens the\n+transmission of light falls of towards the corners of the lens. To make things\n+even harder, this falloff can be different for different colors/wavelengths. LSC\n+is therefore tuned for a set of light sources and for each color channel\n+individually.\n+\n+**CCM** Color correction matrix. After the previous processing blocks the grays\n+are preserved, but colors are still not correct. This is mostly due to the color\n+temperature of the light source and the imperfections in the color filter array\n+of the sensor. To correct for this a 'color correction matrix' is calculated.\n+This is a 3x3 matrix, that is used to optimize the captured colors regarding the\n+perceived color error. To do this a chart of known and precisely measured colors\n+commonly called a macbeth chart is captured. Then the matrix is calculated using\n+linear optimization to best map each measured colour of the chart to it's known\n+value. The color error is measured in `deltaE\n+<https://en.wikipedia.org/wiki/Color_difference>`_.\n+\n+**Gamma** Gamma correction. Today's display usually apply a gamma of 2.2 for\n+display. For images to be perceived correctly by the human eye, they need to be\n+encoded with the corresponding inverse gamma. See also\n+<https://en.wikipedia.org/wiki/Gamma_correction>. This block doesn't need\n+tuning, but is crucial for correct visual display.\n+\n+Materials needed for the tuning process\n+---------------------------------------\n+\n+Precisely calibrated optical equipment is very expensive and out of the scope of\n+this document. Still it is possible to get reasonably good calibration results\n+at little costs. The most important devices needed are: \n+\n+ - A light box with the ability to produce defined light of different color\n+ temperatures. Typical temperatures used for calibration are 2400K\n+ (incandescent), 2700K (fluorescent), 5000K (daylight fluorescent), 6500K\n+ (daylight). As a first test, keylights for webcam streaming can be used.\n+ These are available with support for color temperatures ranging from 2500K\n+ to 9000K. For better results professional light boxes are needed.\n+ - A ColorChecker chart. These are sold from calibrite and it makes sense to\n+ get the original one.\n+ - A integration sphere. This is used to create completely homogenious light\n+ for the lens shading calibration. We had good results with the use of a\n+ large light panel with sufficient diffusion to get an even distribution of\n+ light (the above mentioned keylight).\n+ - An environment without external light sources. Ideally calibration is done\n+ without any external lights in a fully dark room. A black curtain is one\n+ solution, working by night is the cheap alternative.\n+\n+\n+Overview over the process \n+-------------------------\n+\n+The tuning process of libcamera consists of the following steps:\n+\n+ 1. Take images for the tuning steps with different color temperatures\n+ 2. Configure the tuning process\n+ 3. Run the tuning script to create a corresponding tuning file\n+\n+\n+Taking raw images with defined exposure/gain\n+--------------------------------------------\n+\n+For all the tuning files you need to capture a raw image with a defined exposure\n+time and gain. It is crucial that no pixels are saturated on any channel. At the\n+same time the images should not be too dark. Strive for a exposure time, so that\n+the brightest pixel is roughly 90% saturated. Finding the correct exposure time\n+is currently a manual process. We are working on solutions to aid in that\n+process. After finding the correct exposure time settings, the easiest way to\n+capture a raw image is with the ``cam`` application that gets installed with the\n+regular libcamera install.\n+\n+Create a ``constant-exposure-<temperature>.yaml`` file with the following content:\n+\n+.. code:: yaml\n+\n+ frames:\n+ - 0:\n+ AeEnable: false\n+ AnalogueGain: 1.0\n+ ExposureTime: 1000\n+\n+Then capture one or more images using the following command:\n+ ``cam -c 1 --script constant-exposure-<temperature>.yaml -C -s role=raw --file=image_#.dng``\n+\n+.. Note:: Typically the brightness changes for different colour temperatures. So \n+ this has to be readjusted for every colour temperature.\n+ \n+\n+Capture images for lens shading correction\n+------------------------------------------\n+\n+To be able to correct lens shading artifacts, images of a homogeneously lit\n+neutral gray background are needed. Ideally a integration sphere is used for\n+that task.\n+\n+As a fallback images of a flat panel light can be used. If there are multiple\n+images for the same color temperature, the tuning scripts will average the\n+tuning results which can further improve the tuning quality.\n+\n+.. Note:: Currently lsc is ignored in the ccm calculation. This can lead to\n+ slight color deviations and will be improved in the future.\n+\n+Images shall be taken for multiple color temperatures and named\n+``alsc_<temperature>k_#.dng``. ``#`` can be any number, if multiple images are\n+taken for the same color temperature.\n+\n+.. figure:: img/setup_lens_shading.jpg\n+ :width: 50%\n+\n+ Sample setup using a flat panel light. In this specific setup, the camera\n+ has to be moved close to the light due to the wide angle lens. This has the\n+ downside that the angle to the light might get way too steep.\n+\n+.. figure:: img/camshark-alsc.png\n+ :width: 100%\n+\n+ A calibration image for lens shading correction.\n+ \n+- Ensure the full sensor is lit. Especially with wide angle lenses it may be\n+ difficult to get the full field of view lit homogeneously.\n+- Ensure that the brightest area of the image does not contain any pixels that\n+ reach more than 95% on any of the colors (can be checked with camshark).\n+\n+Take a raw dng image for multiple color temperatures:\n+ ``cam -c 1 --script constant-exposure-<temperature>.yaml -C -s role=raw --file=alsc_3200k_#.dng``\n+\n+\n+\n+Capture images for color calibration\n+------------------------------------\n+\n+To do the color calibration, raw images of the color checker need to be taken\n+for different light sources. These need to be named\n+``<sensor_name>_<lux-level>l_<temperature>k_0.dng``.\n+\n+For best results the following hints should be taken into account:\n+ - Ensure that the 18% gray patch (third from the right in bottom line) is\n+ roughly at 18% saturation for all channels (a mean of 16-20% should be fine)\n+ - Ensure the color checker is homogeneously lit (ideally from 45 degree above)\n+ - No straylight from other light sources is present\n+ - The color checker is not too small and not too big (so that neither lens\n+ shading artifacts nor lens distortions are prevailing in the area of the\n+ color chart)\n+\n+If no lux meter is at hand to precisely measure the lux level, a lux meter app\n+on a mobile phone can provide a sufficient estimation.\n+\n+.. figure:: img/setup_calibration2.jpg\n+ :width: 50%\n+\n+\n+Run the tuning scripts\n+----------------------\n+\n+After taking the calibration images, you should have a directory with all the\n+tuning files. It should look something like this:\n+\n+ ::\n+\n+ ../tuning-data/\n+ ├── alsc_2500k_0.dng\n+ ├── alsc_2500k_1.dng\n+ ├── alsc_2500k_2.dng\n+ ├── alsc_6500k_0.dng\n+ ├── imx335_1000l_2500k_0.dng\n+ ├── imx335_1200l_4000k_0.dng\n+ ├── imx335_1600l_6000k_0.dng\n+\n+\n+The tuning scripts are part of the libcamera source tree. After cloning the\n+libcamera sources the necessary steps to create a tuning file are:\n+\n+ ::\n+ \n+ # install the necessary python packages\n+ cd libcamera\n+ python -m venv venv\n+ source ./venv/bin/activate\n+ pip install -r utils/tuning/requirements.txt\n+\n+ # run the tuning script\n+ utils/tuning/rkisp1.py -c config.yaml -i ../tuning-data/ -o tuning-file.yaml\n+\n+After the tuning script has run, the tuning file can be tested with any\n+libcamera based application like `qcam`. To quickly switch to a specific tuning\n+file, the environment variable ``LIBCAMERA_<pipeline>_TUNING_FILE`` is helpful.\n+E.g.:\n+\n+ ::\n+\n+ LIBCAMERA_RKISP1_TUNING_FILE=/path/to/tuning-file.yaml qcam -c1\n+\n+\n+Sample images\n+-------------\n+\n+\n+.. figure:: img/image-no-blc.png\n+ :width: 800\n+\n+ Image without black level correction (and completely invalid color estimation).\n+\n+.. figure:: img/image-no-lsc-3000.png\n+ :width: 800\n+\n+ Image without lens shading correction @ 3000k. The vignetting artifacts can\n+ be clearly seen \n+\n+.. figure:: img/image-no-lsc-6000.png\n+ :width: 800\n+\n+ Image without lens shading correction @ 6000k. The vignetting artifacts can\n+ be clearly seen\n+\n+.. figure:: img/image-fully-tuned-3000.png\n+ :width: 800\n+\n+ Fully tuned image @ 3000k\n+\n+.. figure:: img/image-fully-tuned-6000.png\n+ :width: 800\n+\n+ Fully tuned image @ 6000k\n+ \ndiff --git a/Documentation/index.rst b/Documentation/index.rst\nindex 5442ae75dde7..991dcf2b66fb 100644\n--- a/Documentation/index.rst\n+++ b/Documentation/index.rst\n@@ -16,6 +16,7 @@\n \n Developer Guide <guides/introduction>\n Application Writer's Guide <guides/application-developer>\n+ Sensor Tuning Guide <guides/tuning/tuning>\n Pipeline Handler Writer's Guide <guides/pipeline-handler>\n IPA Writer's guide <guides/ipa>\n Tracing guide <guides/tracing>\ndiff --git a/Documentation/meson.build b/Documentation/meson.build\nindex 1ba40fdf67ac..8bf09f31afa0 100644\n--- a/Documentation/meson.build\n+++ b/Documentation/meson.build\n@@ -135,6 +135,7 @@ if sphinx.found()\n 'guides/ipa.rst',\n 'guides/pipeline-handler.rst',\n 'guides/tracing.rst',\n+ 'guides/tuning/tuning.rst',\n 'index.rst',\n 'lens_driver_requirements.rst',\n 'python-bindings.rst',\n", "prefixes": [ "v2", "1/1" ] }