[{"id":11360,"web_url":"https://patchwork.libcamera.org/comment/11360/","msgid":"<20200713113037.GD2866302@oden.dyn.berto.se>","date":"2020-07-13T11:30:37","subject":"Re: [libcamera-devel] [PATCH 2/9] libcamera: pipeline: ipa:\n\traspberrypi: Rework drop frame signalling","submitter":{"id":5,"url":"https://patchwork.libcamera.org/api/people/5/","name":"Niklas Söderlund","email":"niklas.soderlund@ragnatech.se"},"content":"Hi Naushir,\n\nThanks for your work.\n\nOn 2020-07-13 09:47:21 +0100, Naushir Patuck wrote:\n> The IPA now signals up front how many frames it wants the pipeline\n> handler to drop. This makes it easier to handle up-coming changes to the\n> buffer handling for import/export buffers.\n\nI like this patch. I think it makes the IPA interface easier :-) As you \nsay however it depends on pending work and may need to be refreshed to \nfit that.\n\n> \n> Signed-off-by: Naushir Patuck \n> ---\n> include/libcamera/ipa/raspberrypi.h | 2 +-\n> src/ipa/raspberrypi/raspberrypi.cpp | 21 ++++++------\n> .../pipeline/raspberrypi/raspberrypi.cpp | 33 +++++++++++--------\n> 3 files changed, 32 insertions(+), 24 deletions(-)\n> \n> diff --git a/include/libcamera/ipa/raspberrypi.h b/include/libcamera/ipa/raspberrypi.h\n> index a18ce9a8..2dab2b00 100644\n> --- a/include/libcamera/ipa/raspberrypi.h\n> +++ b/include/libcamera/ipa/raspberrypi.h\n> @@ -15,8 +15,8 @@ enum RPiOperations {\n> \tRPI_IPA_ACTION_V4L2_SET_ISP,\n> \tRPI_IPA_ACTION_STATS_METADATA_COMPLETE,\n> \tRPI_IPA_ACTION_RUN_ISP,\n> -\tRPI_IPA_ACTION_RUN_ISP_AND_DROP_FRAME,\n> \tRPI_IPA_ACTION_SET_SENSOR_CONFIG,\n> +\tRPI_IPA_ACTION_SET_DROP_FRAME_COUNT,\n> \tRPI_IPA_ACTION_EMBEDDED_COMPLETE,\n> \tRPI_IPA_EVENT_SIGNAL_STAT_READY,\n> \tRPI_IPA_EVENT_SIGNAL_ISP_PREPARE,\n> diff --git a/src/ipa/raspberrypi/raspberrypi.cpp b/src/ipa/raspberrypi/raspberrypi.cpp\n> index b1f27861..f3568882 100644\n> --- a/src/ipa/raspberrypi/raspberrypi.cpp\n> +++ b/src/ipa/raspberrypi/raspberrypi.cpp\n> @@ -64,8 +64,8 @@ class IPARPi : public IPAInterface\n> public:\n> \tIPARPi()\n> \t\t: lastMode_({}), controller_(), controllerInit_(false),\n> -\t\t frame_count_(0), check_count_(0), hide_count_(0),\n> -\t\t mistrust_count_(0), lsTableHandle_(0), lsTable_(nullptr)\n> +\t\t frame_count_(0), check_count_(0), mistrust_count_(0),\n> +\t\t lsTableHandle_(0), lsTable_(nullptr)\n> \t{\n> \t}\n> \n> @@ -132,8 +132,6 @@ private:\n> \tuint64_t frame_count_;\n> \t/* For checking the sequencing of Prepare/Process calls. */\n> \tuint64_t check_count_;\n> -\t/* How many frames the pipeline handler should hide, or \"drop\". */\n> -\tunsigned int hide_count_;\n> \t/* How many frames we should avoid running control algos on. */\n> \tunsigned int mistrust_count_;\n> \t/* LS table allocation passed in from the pipeline handler. */\n> @@ -241,14 +239,19 @@ void IPARPi::configure(const CameraSensorInfo &sensorInfo,\n> \t */\n> \tframe_count_ = 0;\n> \tcheck_count_ = 0;\n> +\tint drop_frame = 0;\n> \tif (controllerInit_) {\n> -\t\thide_count_ = helper_->HideFramesModeSwitch();\n> +\t\tdrop_frame = helper_->HideFramesModeSwitch();\n> \t\tmistrust_count_ = helper_->MistrustFramesModeSwitch();\n> \t} else {\n> -\t\thide_count_ = helper_->HideFramesStartup();\n> +\t\tdrop_frame = helper_->HideFramesStartup();\n> \t\tmistrust_count_ = helper_->MistrustFramesStartup();\n> \t}\n> \n> +\tIPAOperationData op;\n> +\top.operation = RPI_IPA_ACTION_SET_DROP_FRAME_COUNT;\n> +\top.data.push_back(drop_frame);\n> +\n> \tstruct AgcStatus agcStatus;\n> \t/* These zero values mean not program anything (unless overwritten). */\n> \tagcStatus.shutter_time = 0.0;\n> @@ -339,13 +342,11 @@ void IPARPi::processEvent(const IPAOperationData &event)\n> \t\t * they are \"unreliable\".\n> \t\t */\n> \t\tprepareISP(embeddedbufferId);\n> +\t\tframe_count_++;\n> \n> \t\t/* Ready to push the input buffer into the ISP. */\n> \t\tIPAOperationData op;\n> -\t\tif (++frame_count_ > hide_count_)\n> -\t\t\top.operation = RPI_IPA_ACTION_RUN_ISP;\n> -\t\telse\n> -\t\t\top.operation = RPI_IPA_ACTION_RUN_ISP_AND_DROP_FRAME;\n> +\t\top.operation = RPI_IPA_ACTION_RUN_ISP;\n> \t\top.data = { bayerbufferId & RPiIpaMask::ID };\n> \t\tqueueFrameAction.emit(0, op);\n> \t\tbreak;\n> diff --git a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> index 7bae72f9..8b6f578f 100644\n> --- a/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> +++ b/src/libcamera/pipeline/raspberrypi/raspberrypi.cpp\n> @@ -131,7 +131,7 @@ class RPiCameraData : public CameraData\n> public:\n> \tRPiCameraData(PipelineHandler *pipe)\n> \t\t: CameraData(pipe), sensor_(nullptr), lsTable_(nullptr),\n> -\t\t state_(State::Stopped), dropFrame_(false), ispOutputCount_(0)\n> +\t\t state_(State::Stopped), dropFrameCount_(0), ispOutputCount_(0)\n> \t{\n> \t}\n> \n> @@ -168,8 +168,8 @@ public:\n> \n> \tCameraSensor *sensor_;\n> \t/* Array of Unicam and ISP device streams and associated buffers/streams. */\n> -\tRPiDevice unicam_;\n> -\tRPiDevice isp_;\n> +\tRPi::RPiDevice unicam_;\n> +\tRPi::RPiDevice isp_;\n> \t/* The vector below is just for convenience when iterating over all streams. */\n> \tstd::vector streams_;\n> \t/* Buffers passed to the IPA. */\n> @@ -194,14 +194,15 @@ public:\n> \tstd::queue embeddedQueue_;\n> \tstd::deque requestQueue_;\n> \n> +\tunsigned int dropFrameCount_;\n> +\n> private:\n> \tvoid checkRequestCompleted();\n> \tvoid tryRunPipeline();\n> \tvoid tryFlushQueues();\n> \tFrameBuffer *updateQueue(std::queue &q, uint64_t timestamp, V4L2VideoDevice *dev);\n> \n> -\tbool dropFrame_;\n> -\tint ispOutputCount_;\n> +\tunsigned int ispOutputCount_;\n> };\n> \n> class RPiCameraConfiguration : public CameraConfiguration\n> @@ -1058,6 +1059,11 @@ void RPiCameraData::queueFrameAction(unsigned int frame, const IPAOperationData\n> \t\tgoto done;\n> \t}\n> \n> +\tcase RPI_IPA_ACTION_SET_DROP_FRAME_COUNT: {\n> +\t\tdropFrameCount_ = action.data[0];\n> +\t\tgoto done;\n> +\t}\n> +\n> \tcase RPI_IPA_ACTION_V4L2_SET_ISP: {\n> \t\tControlList controls = action.controls[0];\n> \t\tisp_[Isp::Input].dev()->setControls(&controls);\n> @@ -1091,7 +1097,6 @@ void RPiCameraData::queueFrameAction(unsigned int frame, const IPAOperationData\n> \t\tbreak;\n> \t}\n> \n> -\tcase RPI_IPA_ACTION_RUN_ISP_AND_DROP_FRAME:\n> \tcase RPI_IPA_ACTION_RUN_ISP: {\n> \t\tunsigned int bufferId = action.data[0];\n> \t\tFrameBuffer *buffer = unicam_[Unicam::Image].getBuffers()->at(bufferId).get();\n> @@ -1100,7 +1105,6 @@ void RPiCameraData::queueFrameAction(unsigned int frame, const IPAOperationData\n> \t\t\t\t<< \", timestamp: \" << buffer->metadata().timestamp;\n> \n> \t\tisp_[Isp::Input].dev()->queueBuffer(buffer);\n> -\t\tdropFrame_ = (action.operation == RPI_IPA_ACTION_RUN_ISP_AND_DROP_FRAME) ? true : false;\n> \t\tispOutputCount_ = 0;\n> \t\tbreak;\n> \t}\n> @@ -1266,7 +1270,7 @@ void RPiCameraData::clearIncompleteRequests()\n> void RPiCameraData::handleStreamBuffer(FrameBuffer *buffer, const RPi::RPiStream *stream)\n> {\n> \tif (stream->isExternal()) {\n> -\t\tif (!dropFrame_) {\n> +\t\tif (!dropFrameCount_) {\n> \t\t\tRequest *request = buffer->request();\n> \t\t\tpipe_->completeBuffer(camera_, request, buffer);\n> \t\t}\n> @@ -1278,7 +1282,7 @@ void RPiCameraData::handleStreamBuffer(FrameBuffer *buffer, const RPi::RPiStream\n> \t\t * simply memcpy to the Request buffer and requeue back to the\n> \t\t * device.\n> \t\t */\n> -\t\tif (stream == &unicam_[Unicam::Image] && !dropFrame_) {\n> +\t\tif (stream == &unicam_[Unicam::Image] && !dropFrameCount_) {\n> \t\t\tconst Stream *rawStream = static_cast(&isp_[Isp::Input]);\n> \t\t\tRequest *request = requestQueue_.front();\n> \t\t\tFrameBuffer *raw = request->findBuffer(const_cast(rawStream));\n> @@ -1323,7 +1327,7 @@ void RPiCameraData::checkRequestCompleted()\n> \t * If we are dropping this frame, do not touch the request, simply\n> \t * change the state to IDLE when ready.\n> \t */\n> -\tif (!dropFrame_) {\n> +\tif (!dropFrameCount_) {\n> \t\tRequest *request = requestQueue_.front();\n> \t\tif (request->hasPendingBuffers())\n> \t\t\treturn;\n> @@ -1342,10 +1346,13 @@ void RPiCameraData::checkRequestCompleted()\n> \t * frame.\n> \t */\n> \tif (state_ == State::IpaComplete &&\n> -\t ((ispOutputCount_ == 3 && dropFrame_) || requestCompleted)) {\n> +\t ((ispOutputCount_ == 3 && dropFrameCount_) || requestCompleted)) {\n> \t\tstate_ = State::Idle;\n> -\t\tif (dropFrame_)\n> -\t\t\tLOG(RPI, Info) << \"Dropping frame at the request of the IPA\";\n> +\t\tif (dropFrameCount_) {\n> +\t\t\tdropFrameCount_--;\n> +\t\t\tLOG(RPI, Info) << \"Dropping frame at the request of the IPA ( \"\n> +\t\t\t\t << dropFrameCount_ << \" left)\";\n> +\t\t}\n> \t}\n> }\n> \n> -- \n> 2.25.1\n> \n> _______________________________________________\n> libcamera-devel mailing list\n> libcamera-devel@lists.libcamera.org\n> https://lists.libcamera.org/listinfo/libcamera-devel","headers":{"Return-Path":"","X-Original-To":"parsemail@patchwork.libcamera.org","Delivered-To":"parsemail@patchwork.libcamera.org","Received":["from lancelot.ideasonboard.com (lancelot.ideasonboard.com\n\t[92.243.16.209])\n\tby patchwork.libcamera.org (Postfix) with ESMTPS id F14CCBD792\n\tfor ;\n\tMon, 13 Jul 2020 11:30:40 +0000 (UTC)","from lancelot.ideasonboard.com (localhost [IPv6:::1])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTP id 845AA605D1;\n\tMon, 13 Jul 2020 13:30:40 +0200 (CEST)","from mail-lj1-x233.google.com (mail-lj1-x233.google.com\n\t[IPv6:2a00:1450:4864:20::233])\n\tby lancelot.ideasonboard.com (Postfix) with ESMTPS id 74488603A4\n\tfor ;\n\tMon, 13 Jul 2020 13:30:39 +0200 (CEST)","by mail-lj1-x233.google.com with SMTP id q7so17277711ljm.1\n\tfor ;\n\tMon, 13 Jul 2020 04:30:39 -0700 (PDT)","from localhost (h-209-203.A463.priv.bahnhof.se. [155.4.209.203])\n\tby smtp.gmail.com with ESMTPSA id\n\tb2sm1921175lji.63.2020.07.13.04.30.37\n\t(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n\tMon, 13 Jul 2020 04:30:37 -0700 (PDT)"],"Authentication-Results":"lancelot.ideasonboard.com;\n\tdkim=fail reason=\"signature verification failed\" (2048-bit key;\n\tunprotected) header.d=ragnatech-se.20150623.gappssmtp.com\n\theader.i=@ragnatech-se.20150623.gappssmtp.com\n\theader.b=\"Mus/fKrB\"; dkim-atps=neutral","DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=ragnatech-se.20150623.gappssmtp.com; s=20150623;\n\th=date:from:to:cc:subject:message-id:references:mime-version\n\t:content-disposition:content-transfer-encoding:in-reply-to;\n\tbh=dDPYPjdUuDbPaMQKYSOSVWarNw3IV3YMGAunyLv+KfQ=;\n\tb=Mus/fKrBlvR+xAwhhiRnbvdx7TRlaMCBXOaFLaxLcVB9epO94kPDD12gJtm/lPEqk+\n\ty+fyuunU1mygIwXKiNTciCtNwcJ/HvlBCNZsVGLVAU8/BKGHIR09K9s4tFzEtfChqWk2\n\t+1AfnJjauZJELO0avreehalrhxI5clkiwcr9iTy/VbQWpRKk4X/jCfe8ZzBJA/++Gmo0\n\tD6AA3qZiivZ9WFsOhOhvMioNdZaf6BGP+cld8+KDtgc4FmAQn4M0ytgEj/QauUHeaXbH\n\t6K2F7of+GLp/OPQ487yHUbXpxsWN/bGapMOrdrZxBVgmf/4yfFQJ7mmxS/4IIkhnTTP2\n\tjsUA==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:date:from:to:cc:subject:message-id:references\n\t:mime-version:content-disposition:content-transfer-encoding\n\t:in-reply-to;\n\tbh=dDPYPjdUuDbPaMQKYSOSVWarNw3IV3YMGAunyLv+KfQ=;\n\tb=cCCAuVfhLDTZiVBNSVt8/3kw8toUwwMSD8YnG+fR8X28g4w05klZVTu5u8VMK8gjBc\n\t3e6lmGaTkruRbW1yBflLZJ86waEbv1L9h8EiC+xBEjVDmasElaog9i7Wo925QXXPb+/U\n\tpOD4zbxT5fi6eoy+J0knKZ/fPxYycKQ/QUBJE8PGGqNdDkVIcmeYhytXjWgye2GaCxvM\n\tprfQDGV2bT3BzT2WoAll5tzHxgmHnqpwTnSW18Rw6H3llIMYrwPZSKaPE5F9yzx5hoP4\n\tM5d/rxq3iNqPCt8zcgiCrr9edpJUUqPInt/wD1b2ct03fuv7uUDaolFuzP+IJSi/AnMK\n\t0RCg==","X-Gm-Message-State":"AOAM53366hwcNDLkNG4jDsiP5sohxJzqNo0KxTLbQeah7MRuqBSQGHKc\n\tvSfbF75sUNKkodIVtb5WLqfFyA==","X-Google-Smtp-Source":"ABdhPJy7HHK1NUReUeV1IZrRGgpccQ/vpAcC2+To+ZhZYpKf303usgopyaYYDHjfsTE+0/tLd04/MA==","X-Received":"by 2002:a05:651c:10f:: with SMTP id\n\ta15mr46404646ljb.192.1594639838786; \n\tMon, 13 Jul 2020 04:30:38 -0700 (PDT)","Date":"Mon, 13 Jul 2020 13:30:37 +0200","From":"Niklas =?iso-8859-1?q?S=F6derlund?= ","To":"Naushir Patuck ","Message-ID":"<20200713113037.GD2866302@oden.dyn.berto.se>","References":"<20200713084727.232422-1-naush@raspberrypi.com>\n\t<20200713084727.232422-3-naush@raspberrypi.com>","MIME-Version":"1.0","Content-Disposition":"inline","In-Reply-To":"<20200713084727.232422-3-naush@raspberrypi.com>","Subject":"Re: [libcamera-devel] [PATCH 2/9] libcamera: pipeline: ipa:\n\traspberrypi: Rework drop frame signalling","X-BeenThere":"libcamera-devel@lists.libcamera.org","X-Mailman-Version":"2.1.29","Precedence":"list","List-Id":"","List-Unsubscribe":",\n\t","List-Archive":"","List-Post":"","List-Help":"","List-Subscribe":",\n\t","Cc":"libcamera-devel@lists.libcamera.org","Content-Type":"text/plain; charset=\"iso-8859-1\"","Content-Transfer-Encoding":"quoted-printable","Errors-To":"libcamera-devel-bounces@lists.libcamera.org","Sender":"\"libcamera-devel\" "}}]