@@ -3,74 +3,76 @@
* Copyright (C) 2020, Laurent Pinchart
* Copyright (C) 2019, Martijn Braam
*
* Pipeline handler for simple pipelines
*/
#include <algorithm>
#include <iterator>
#include <list>
#include <map>
#include <memory>
#include <optional>
#include <queue>
#include <set>
#include <stdint.h>
#include <string.h>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <linux/media-bus-format.h>
#include <libcamera/base/log.h>
#include <libcamera/camera.h>
#include <libcamera/color_space.h>
#include <libcamera/control_ids.h>
#include <libcamera/geometry.h>
#include <libcamera/pixel_format.h>
+#include <libcamera/property_ids.h>
#include <libcamera/stream.h>
#include "libcamera/internal/camera.h"
#include "libcamera/internal/camera_manager.h"
#include "libcamera/internal/camera_sensor.h"
#include "libcamera/internal/camera_sensor_properties.h"
#include "libcamera/internal/converter.h"
#include "libcamera/internal/delayed_controls.h"
#include "libcamera/internal/device_enumerator.h"
#include "libcamera/internal/formats.h"
#include "libcamera/internal/global_configuration.h"
#include "libcamera/internal/media_device.h"
#include "libcamera/internal/pipeline_handler.h"
#include "libcamera/internal/request.h"
+#include "libcamera/internal/software_isp/debayer_geometry.h"
#include "libcamera/internal/software_isp/software_isp.h"
#include "libcamera/internal/v4l2_subdevice.h"
#include "libcamera/internal/v4l2_videodevice.h"
namespace libcamera {
LOG_DEFINE_CATEGORY(SimplePipeline)
/* -----------------------------------------------------------------------------
*
* Overview
* --------
*
* The SimplePipelineHandler relies on generic kernel APIs to control a camera
* device, without any device-specific code and with limited device-specific
* static data.
*
* To qualify for support by the simple pipeline handler, a device shall
*
* - be supported by V4L2 drivers, exposing the Media Controller API, the V4L2
* subdev APIs and the media bus format-based enumeration extension for the
* VIDIOC_ENUM_FMT ioctl ;
* - not expose any device-specific API from drivers to userspace ;
* - include one or more camera sensor media entities and one or more video
* capture devices ;
* - have a capture pipeline with linear paths from the camera sensors to the
* video capture devices ; and
* - have an optional memory-to-memory device to perform format conversion
* and/or scaling, exposed as a V4L2 M2M device.
*
@@ -265,60 +267,62 @@ static const SimplePipelineInfo supportedDevices[] = {
{ "mxc-isi", {}, false },
{ "qcom-camss", {}, true },
{ "sun6i-csi", {}, false },
};
bool isRaw(const StreamConfiguration &cfg)
{
return libcamera::PixelFormatInfo::info(cfg.pixelFormat).colourEncoding ==
libcamera::PixelFormatInfo::ColourEncodingRAW;
}
} /* namespace */
class SimpleCameraData : public Camera::Private
{
public:
SimpleCameraData(SimplePipelineHandler *pipe,
unsigned int numStreams,
MediaEntity *sensor);
bool isValid() const { return sensor_ != nullptr; }
SimplePipelineHandler *pipe();
int init();
int setupLinks();
int setupFormats(V4L2SubdeviceFormat *format,
V4L2Subdevice::Whence whence,
Transform transform = Transform::Identity);
void imageBufferReady(FrameBuffer *buffer);
void clearIncompleteRequests();
+ std::optional<Rectangle> computeScalerCrop(const Size &inputSize, Transform transform);
+ void updateScalerCrop(std::optional<Rectangle> crop);
unsigned int streamIndex(const Stream *stream) const
{
return stream - &streams_.front();
}
struct Entity {
/* The media entity, always valid. */
MediaEntity *entity;
/*
* Whether or not the entity is a subdev that supports the
* routing API.
*/
bool supportsRouting;
/*
* The local sink pad connected to the upstream entity, null for
* the camera sensor at the beginning of the pipeline.
*/
const MediaPad *sink;
/*
* The local source pad connected to the downstream entity, null
* for the video node at the end of the pipeline.
*/
const MediaPad *source;
/*
* The link on the source pad, to the downstream entity, null
* for the video node at the end of the pipeline.
*/
MediaLink *sourceLink;
};
@@ -333,60 +337,61 @@ public:
};
std::vector<Stream> streams_;
Stream *rawStream_;
/*
* All entities in the pipeline, from the camera sensor to the video
* node.
*/
std::list<Entity> entities_;
std::unique_ptr<CameraSensor> sensor_;
V4L2VideoDevice *video_;
V4L2Subdevice *frameStartEmitter_;
std::vector<Configuration> configs_;
std::map<PixelFormat, std::vector<const Configuration *>> formats_;
std::unique_ptr<DelayedControls> delayedCtrls_;
std::vector<std::unique_ptr<FrameBuffer>> conversionBuffers_;
struct RequestOutputs {
Request *request;
std::map<const Stream *, FrameBuffer *> outputs;
};
std::queue<RequestOutputs> conversionQueue_;
bool useConversion_;
std::unique_ptr<Converter> converter_;
std::unique_ptr<SoftwareIsp> swIsp_;
SimpleFrames frameInfo_;
+ std::optional<Rectangle> scalerCrop_;
private:
void tryPipeline(unsigned int code, const Size &size);
static std::vector<const MediaPad *> routedSourcePads(MediaPad *sink);
void tryCompleteRequest(Request *request);
void conversionInputDone(FrameBuffer *buffer);
void conversionOutputDone(FrameBuffer *buffer);
void ispStatsReady(uint32_t frame, uint32_t bufferId);
void metadataReady(uint32_t frame, const ControlList &metadata);
void setSensorControls(const ControlList &sensorControls);
};
class SimpleCameraConfiguration : public CameraConfiguration
{
public:
SimpleCameraConfiguration(Camera *camera, SimpleCameraData *data);
Status validate() override;
const SimpleCameraData::Configuration *pipeConfig() const
{
return pipeConfig_;
}
bool needConversion() const { return needConversion_; }
const Transform &combinedTransform() const { return combinedTransform_; }
private:
@@ -956,60 +961,130 @@ void SimpleCameraData::imageBufferReady(FrameBuffer *buffer)
}
if (converter_)
converter_->queueBuffers(buffer, conversionQueue_.front().outputs);
else
/*
* request->sequence() cannot be retrieved from `buffer' inside
* queueBuffers because unique_ptr's make buffer->request() invalid
* already here.
*/
swIsp_->queueBuffers(request->sequence(), buffer,
conversionQueue_.front().outputs);
conversionQueue_.pop();
return;
}
/* Otherwise simply complete the request. */
pipe->completeBuffer(request, buffer);
tryCompleteRequest(request);
}
void SimpleCameraData::clearIncompleteRequests()
{
while (!conversionQueue_.empty()) {
pipe()->cancelRequest(conversionQueue_.front().request);
conversionQueue_.pop();
}
}
+std::optional<Rectangle> SimpleCameraData::computeScalerCrop(const Size &inputSize,
+ Transform transform)
+{
+ IPACameraSensorInfo sensorInfo;
+ if (sensor_->sensorInfo(&sensorInfo))
+ return std::nullopt;
+
+ /*
+ * The buffer-to-sensor mapping requires a capture path that preserves
+ * the sensor output. Intermediate cropping or scaling needs a separate
+ * mapping; do not report the analog crop as if it included that work.
+ * Unavailable selection stages preserve the preceding rectangle on the
+ * supported pass-through paths, provided the pad formats match.
+ */
+ if (sensorInfo.outputSize != inputSize)
+ return std::nullopt;
+
+ for (const Entity &entity : entities_) {
+ if (!entity.sink || !entity.source)
+ continue;
+
+ V4L2Subdevice *subdev = pipe()->subdev(entity.entity);
+ for (const MediaPad *pad : { entity.sink, entity.source }) {
+ V4L2SubdeviceFormat format{};
+ if (subdev->getFormat(pad->index(), &format) ||
+ format.size != inputSize)
+ return std::nullopt;
+
+ for (unsigned int target : { V4L2_SEL_TGT_CROP, V4L2_SEL_TGT_COMPOSE }) {
+ /* Composition is applied on the sink pad. */
+ if (pad == entity.source && target == V4L2_SEL_TGT_COMPOSE)
+ continue;
+
+ std::optional<Rectangle> selection;
+ int ret = subdev->getSelection(pad->index(), target, &selection);
+ if (ret)
+ return std::nullopt;
+ if (!selection)
+ continue;
+
+ if (*selection != Rectangle(inputSize))
+ return std::nullopt;
+ }
+ }
+ }
+
+ return debayerCropToSensor(swIsp_->inputCrop(), inputSize,
+ sensorInfo.analogCrop, transform);
+}
+
+void SimpleCameraData::updateScalerCrop(std::optional<Rectangle> crop)
+{
+ scalerCrop_ = crop;
+ if (crop) {
+ properties_.set(properties::ScalerCropMaximum, *crop);
+ return;
+ }
+
+ if (!properties_.contains(properties::ScalerCropMaximum.id()))
+ return;
+
+ ControlList updatedProperties = properties_;
+ updatedProperties.clear();
+ for (const auto &[id, value] : properties_) {
+ if (id != properties::ScalerCropMaximum.id())
+ updatedProperties.set(id, value);
+ }
+ properties_ = std::move(updatedProperties);
+}
+
void SimpleCameraData::tryCompleteRequest(Request *request)
{
if (request->hasPendingBuffers())
return;
SimpleFrameInfo *info = frameInfo_.find(request->sequence());
if (!info) {
/* Something is really wrong, let's return. */
return;
}
if (info->metadataRequired && !info->metadataProcessed)
return;
frameInfo_.destroy(info->frame);
pipe()->completeRequest(request);
}
void SimpleCameraData::conversionInputDone(FrameBuffer *buffer)
{
if (rawStream_) {
/* Complete the input buffer as with raw-only processing. */
Request *request = buffer->request();
if (pipe()->completeBuffer(request, buffer))
tryCompleteRequest(request);
} else {
/* Queue the input buffer back for capture. */
video_->queueBuffer(buffer);
}
}
@@ -1576,76 +1651,87 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c)
<< pipeConfig->captureSize << "-" << videoFormat
<< " (got " << captureFormat << ")";
return -EINVAL;
}
if (requested_bpl && captureFormat.planes[0].bpl != requested_bpl) {
LOG(SimplePipeline, Info)
<< "Input buffer stride ignored by the driver. "
<< "Requested " << requested_bpl
<< ", got " << captureFormat.planes[0].bpl;
}
/* Configure the converter if needed. */
std::vector<std::reference_wrapper<const StreamConfiguration>> outputCfgs;
data->useConversion_ = config->needConversion();
data->rawStream_ = nullptr;
for (unsigned int i = 0; i < config->size(); ++i) {
StreamConfiguration &cfg = config->at(i);
bool rawStream = isRaw(cfg);
cfg.setStream(&data->streams_[i]);
if (data->useConversion_ && !rawStream)
outputCfgs.push_back(cfg);
if (rawStream)
data->rawStream_ = &data->streams_[i];
}
- if (outputCfgs.empty())
+ if (outputCfgs.empty()) {
+ data->updateScalerCrop(std::nullopt);
return 0;
+ }
StreamConfiguration inputCfg;
inputCfg.pixelFormat = videoFormat.toPixelFormat();
inputCfg.size = pipeConfig->captureSize;
inputCfg.stride = captureFormat.planes[0].bpl;
inputCfg.bufferCount = kNumInternalBuffers;
if (data->converter_) {
- return data->converter_->configure(inputCfg, outputCfgs);
+ ret = data->converter_->configure(inputCfg, outputCfgs);
} else {
ipa::softisp::IPAConfigInfo configInfo;
configInfo.sensorControls = data->sensor_->controls();
- return data->swIsp_->configure(inputCfg, outputCfgs, configInfo, &data->controlInfo_);
+ ret = data->swIsp_->configure(inputCfg, outputCfgs, configInfo, &data->controlInfo_);
}
+ if (ret)
+ return ret;
+
+ std::optional<Rectangle> crop;
+ if (!data->converter_)
+ crop = data->computeScalerCrop(inputCfg.size, config->combinedTransform());
+ data->updateScalerCrop(crop);
+
+ return 0;
}
int SimplePipelineHandler::exportFrameBuffers(Camera *camera, Stream *stream,
std::vector<std::unique_ptr<FrameBuffer>> *buffers)
{
SimpleCameraData *data = cameraData(camera);
unsigned int count = stream->configuration().bufferCount;
/*
* Export buffers on the converter or capture video node, depending on
* whether the converter is used or not.
*/
if (data->useConversion_ && stream != data->rawStream_)
return data->converter_
? data->converter_->exportBuffers(stream, count, buffers)
: data->swIsp_->exportBuffers(stream, count, buffers);
else
return data->video_->exportBuffers(count, buffers);
}
int SimplePipelineHandler::start(Camera *camera, [[maybe_unused]] const ControlList *controls)
{
SimpleCameraData *data = cameraData(camera);
V4L2VideoDevice *video = data->video_;
V4L2Subdevice *frameStartEmitter = data->frameStartEmitter_;
int ret;
const MediaPad *pad = acquirePipeline(data);
if (pad) {
LOG(SimplePipeline, Info)