From c0886381a2bbe494b900d699a3858573316059b2 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Mon, 11 Dec 2023 23:47:47 +0300 Subject: [PATCH 12/25] libcamera: ipa: Soft IPA: add a Simple Soft IPA implementation Auto exposure/gain and AWB implementation by Dennis, Toon and Martti. Co-authored-by: Dennis Bonke Signed-off-by: Dennis Bonke Co-authored-by: Marttico Signed-off-by: Marttico Co-authored-by: Toon Langendam Signed-off-by: Toon Langendam Signed-off-by: Andrey Konovalov Signed-off-by: Hans de Goede Tested-by: Bryan O'Donoghue # sc8280xp Lenovo x13s Tested-by: Pavel Machek --- meson_options.txt | 3 +- src/ipa/simple/meson.build | 9 + src/ipa/simple/simple/data/meson.build | 9 + src/ipa/simple/simple/data/soft.conf | 3 + src/ipa/simple/simple/meson.build | 26 +++ src/ipa/simple/simple/soft_simple.cpp | 273 +++++++++++++++++++++++++ 6 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 src/ipa/simple/simple/data/meson.build create mode 100644 src/ipa/simple/simple/data/soft.conf create mode 100644 src/ipa/simple/simple/meson.build create mode 100644 src/ipa/simple/simple/soft_simple.cpp diff --git a/meson_options.txt b/meson_options.txt index 5fdc7be8..8ec08658 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -27,7 +27,7 @@ option('gstreamer', option('ipas', type : 'array', - choices : ['ipu3', 'rkisp1', 'rpi/vc4', 'vimc'], + choices : ['ipu3', 'rkisp1', 'rpi/vc4', 'simple/simple', 'vimc'], description : 'Select which IPA modules to build') option('lc-compliance', @@ -46,6 +46,7 @@ option('pipelines', 'rkisp1', 'rpi/vc4', 'simple', + 'simple/simple', 'uvcvideo', 'vimc' ], diff --git a/src/ipa/simple/meson.build b/src/ipa/simple/meson.build index 9688bbdb..14be5dc2 100644 --- a/src/ipa/simple/meson.build +++ b/src/ipa/simple/meson.build @@ -1,3 +1,12 @@ # SPDX-License-Identifier: CC0-1.0 subdir('common') + +foreach pipeline : pipelines + pipeline = pipeline.split('/') + if pipeline.length() < 2 or pipeline[0] != 'simple' + continue + endif + + subdir(pipeline[1]) +endforeach diff --git a/src/ipa/simple/simple/data/meson.build b/src/ipa/simple/simple/data/meson.build new file mode 100644 index 00000000..33548cc6 --- /dev/null +++ b/src/ipa/simple/simple/data/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: CC0-1.0 + +conf_files = files([ + 'soft.conf', +]) + +install_data(conf_files, + install_dir : ipa_data_dir / 'soft', + install_tag : 'runtime') diff --git a/src/ipa/simple/simple/data/soft.conf b/src/ipa/simple/simple/data/soft.conf new file mode 100644 index 00000000..0c70e7c0 --- /dev/null +++ b/src/ipa/simple/simple/data/soft.conf @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Dummy configuration file for the soft IPA. diff --git a/src/ipa/simple/simple/meson.build b/src/ipa/simple/simple/meson.build new file mode 100644 index 00000000..8b5d76b5 --- /dev/null +++ b/src/ipa/simple/simple/meson.build @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: CC0-1.0 + +ipa_name = 'ipa_soft_simple' + +mod = shared_module(ipa_name, + ['soft_simple.cpp', libcamera_generated_ipa_headers], + name_prefix : '', + include_directories : [ipa_includes, libipa_includes, '..'], + dependencies : libcamera_private, + link_with : libipa, + link_whole : soft_ipa_common_lib, + install : true, + install_dir : ipa_install_dir) + +if ipa_sign_module + custom_target(ipa_name + '.so.sign', + input : mod, + output : ipa_name + '.so.sign', + command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'], + install : false, + build_by_default : true) +endif + +subdir('data') + +ipa_names += ipa_name diff --git a/src/ipa/simple/simple/soft_simple.cpp b/src/ipa/simple/simple/soft_simple.cpp new file mode 100644 index 00000000..93fc1545 --- /dev/null +++ b/src/ipa/simple/simple/soft_simple.cpp @@ -0,0 +1,273 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2023, Linaro Ltd + * + * soft_simple.cpp - Simple Software Image Processing Algorithm module + */ + +#include + +#include +#include + +#include + +#include +#include + +#include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/software_isp/debayer_params.h" +#include "libcamera/internal/software_isp/swisp_stats.h" + +#include "common/soft_base.h" + +#define EXPOSURE_OPTIMAL_VALUE 2.5 +#define EXPOSURE_SATISFACTORY_OFFSET 0.2 + +namespace libcamera { + +LOG_DECLARE_CATEGORY(IPASoft) + +namespace ipa::soft { + +class IPASoftSimple final : public IPASoftBase +{ +public: + IPASoftSimple() + : IPASoftBase(), ignore_updates_(0) + { + } + + ~IPASoftSimple() + { + if (stats_) + munmap(stats_, sizeof(SwIspStats)); + if (params_) + munmap(params_, sizeof(DebayerParams)); + } + + int platformInit(const ControlInfoMap &sensorInfoMap) override; + int platformConfigure(const ControlInfoMap &sensorInfoMap) override; + int platformStart() override; + void platformStop() override; + void platformProcessStats(const ControlList &sensorControls) override; + +private: + void update_exposure(double exposuremsv); + + DebayerParams *params_; + SwIspStats *stats_; + int exposure_min_, exposure_max_; + int again_min_, again_max_; + int again_, exposure_; + int ignore_updates_; +}; + +int IPASoftSimple::platformInit(const ControlInfoMap &sensorInfoMap) +{ + params_ = static_cast(mmap(nullptr, sizeof(DebayerParams), + PROT_WRITE, MAP_SHARED, + fdParams_.get(), 0)); + if (!params_) { + LOG(IPASoft, Error) << "Unable to map Parameters"; + return -ENODEV; + } + + stats_ = static_cast(mmap(nullptr, sizeof(SwIspStats), + PROT_READ, MAP_SHARED, + fdStats_.get(), 0)); + if (!stats_) { + LOG(IPASoft, Error) << "Unable to map Statistics"; + return -ENODEV; + } + + if (sensorInfoMap.find(V4L2_CID_EXPOSURE) == sensorInfoMap.end()) { + LOG(IPASoft, Error) << "Don't have exposure control"; + return -EINVAL; + } + + if (sensorInfoMap.find(V4L2_CID_ANALOGUE_GAIN) == sensorInfoMap.end()) { + LOG(IPASoft, Error) << "Don't have gain control"; + return -EINVAL; + } + + return 0; +} + +int IPASoftSimple::platformConfigure(const ControlInfoMap &sensorInfoMap) +{ + const ControlInfo &exposure_info = sensorInfoMap.find(V4L2_CID_EXPOSURE)->second; + const ControlInfo &gain_info = sensorInfoMap.find(V4L2_CID_ANALOGUE_GAIN)->second; + + exposure_min_ = exposure_info.min().get(); + if (!exposure_min_) { + LOG(IPASoft, Warning) << "Minimum exposure is zero, that can't be linear"; + exposure_min_ = 1; + } + exposure_max_ = exposure_info.max().get(); + again_min_ = gain_info.min().get(); + if (!again_min_) { + LOG(IPASoft, Warning) << "Minimum gain is zero, that can't be linear"; + again_min_ = 100; + } + again_max_ = gain_info.max().get(); + + LOG(IPASoft, Info) << "Exposure " << exposure_min_ << "-" << exposure_max_ + << ", gain " << again_min_ << "-" << again_max_; + + return 0; +} + +int IPASoftSimple::platformStart() +{ + return 0; +} + +void IPASoftSimple::platformStop() +{ +} + +void IPASoftSimple::platformProcessStats(const ControlList &sensorControls) +{ + /* + * Calculate red and blue gains for AWB. + * Clamp max gain at 4.0, this also avoids 0 division. + */ + if (stats_->sumR_ <= stats_->sumG_ / 4) + params_->gainR = 1024; + else + params_->gainR = 256 * stats_->sumG_ / stats_->sumR_; + + if (stats_->sumB_ <= stats_->sumG_ / 4) + params_->gainB = 1024; + else + params_->gainB = 256 * stats_->sumG_ / stats_->sumB_; + + /* Green gain and gamma values are fixed */ + params_->gainG = 256; + params_->gamma = 0.5; + + setIspParams.emit(0); + + /* + * AE / AGC, use 2 frames delay to make sure that the exposure and + * the gain set have applied to the camera sensor. + */ + if (ignore_updates_ > 0) { + --ignore_updates_; + return; + } + + unsigned int denom = 0; + unsigned int num = 0; + unsigned int y_histogramSmall[5] = {}; + + for (int i = 0; i < 16; i++) + y_histogramSmall[(i - i / 8) / 3] += stats_->y_histogram[i]; + + for (int i = 0; i < 5; i++) { + LOG(IPASoft, Debug) << i << ": " << y_histogramSmall[i]; + denom += y_histogramSmall[i]; + num += y_histogramSmall[i] * (i + 1); + } + + float exposuremsv = (float)num / denom; + + /* sanity check */ + if (!sensorControls.contains(V4L2_CID_EXPOSURE) || + !sensorControls.contains(V4L2_CID_ANALOGUE_GAIN)) { + LOG(IPASoft, Error) << "Control(s) missing"; + return; + } + + ControlList ctrls(sensorControls); + + exposure_ = ctrls.get(V4L2_CID_EXPOSURE).get(); + again_ = ctrls.get(V4L2_CID_ANALOGUE_GAIN).get(); + + update_exposure(exposuremsv); + + ctrls.set(V4L2_CID_EXPOSURE, exposure_); + ctrls.set(V4L2_CID_ANALOGUE_GAIN, again_); + + ignore_updates_ = 2; + + setSensorControls.emit(ctrls); + + LOG(IPASoft, Debug) << "exposuremsv " << exposuremsv + << " exp " << exposure_ << " again " << again_ + << " gain R/B " << params_->gainR << "/" << params_->gainB; +} + +/* DENOMINATOR of 10 gives ~10% increment/decrement; DENOMINATOR of 5 - about ~20% */ +#define DENOMINATOR 10 +#define UP_NUMERATOR (DENOMINATOR + 1) +#define DOWN_NUMERATOR (DENOMINATOR - 1) + +void IPASoftSimple::update_exposure(double exposuremsv) +{ + int next; + + if (exposuremsv < EXPOSURE_OPTIMAL_VALUE - EXPOSURE_SATISFACTORY_OFFSET) { + next = exposure_ * UP_NUMERATOR / DENOMINATOR; + if (next - exposure_ < 1) + exposure_ += 1; + else + exposure_ = next; + if (exposure_ >= exposure_max_) { + next = again_ * UP_NUMERATOR / DENOMINATOR; + if (next - again_ < 1) + again_ += 1; + else + again_ = next; + } + } + + if (exposuremsv > EXPOSURE_OPTIMAL_VALUE + EXPOSURE_SATISFACTORY_OFFSET) { + if (exposure_ == exposure_max_ && again_ != again_min_) { + next = again_ * DOWN_NUMERATOR / DENOMINATOR; + if (again_ - next < 1) + again_ -= 1; + else + again_ = next; + } else { + next = exposure_ * DOWN_NUMERATOR / DENOMINATOR; + if (exposure_ - next < 1) + exposure_ -= 1; + else + exposure_ = next; + } + } + + if (exposure_ > exposure_max_) + exposure_ = exposure_max_; + else if (exposure_ < exposure_min_) + exposure_ = exposure_min_; + + if (again_ > again_max_) + again_ = again_max_; + else if (again_ < again_min_) + again_ = again_min_; +} + +} /* namespace ipa::soft */ + +/* + * External IPA module interface + */ +extern "C" { +const struct IPAModuleInfo ipaModuleInfo = { + IPA_MODULE_API_VERSION, + 0, + "SimplePipelineHandler", + "soft/simple", +}; + +IPAInterface *ipaCreate() +{ + return new ipa::soft::IPASoftSimple(); +} + +} /* extern "C" */ + +} /* namespace libcamera */ -- 2.43.0