Commit a4c00309 authored by Andreas Pehrson's avatar Andreas Pehrson
Browse files

Bug 1817724 - Clean up and tighten VideoCaptureAvFoundation somewhat. r=webrtc-reviewers,ng 

This patch mainly changes VideoCaptureAdapter to use an instance method to set
the capturer member under a mutex. It also moves the setting of this capturer
to start rather than init (and unsets it on stop). With this we shouldn't need
the prior fixes to RTCCameraVideoCapturer.m.

Some other cleanup happens as well:
- Constification of members
- Moving the RTCVideoFrame to webrtc::VideoFrame conversion to OnFrame()
- Optimizing away StartFrameRecording()
- Only calculating image type once

Differential Revision: https://phabricator.services.mozilla.com/D171960
parent 9bc760fc

dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h

+14 −11
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ 


#import "components/capturer/RTCCameraVideoCapturer.h"



#include "api/scoped_refptr.h"

#include "api/sequence_checker.h"

#include "modules/video_capture/video_capture_impl.h"

#include "mozilla/Maybe.h"
@@ -40,28 +41,27 @@ class VideoCaptureAvFoundation : public VideoCaptureImpl { 
  int32_t CaptureSettings(VideoCaptureCapability& aSettings) override;



  // Callback. This can be called on any thread.

  int32_t OnFrame(webrtc::VideoFrame& aFrame) MOZ_EXCLUDES(api_lock_);

  int32_t OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) MOZ_EXCLUDES(api_lock_);



  void SetTrackingId(uint32_t aTrackingIdProcId) MOZ_EXCLUDES(api_lock_) override;



  // Allows the capturer to start the recording before calling OnFrame, to cover more operations

  // under the same measurement.

  void StartFrameRecording(int32_t aWidth, int32_t aHeight) MOZ_EXCLUDES(api_lock_);



  // Registers the current thread with the profiler if not already registered.

  void MaybeRegisterCallbackThread();



 private:

  // Control thread checker.

  SequenceChecker mChecker;

  AVCaptureDevice* _Nonnull mDevice RTC_GUARDED_BY(mChecker);

  VideoCaptureAdapter* _Nonnull mAdapter RTC_GUARDED_BY(mChecker);

  RTC_OBJC_TYPE(RTCCameraVideoCapturer) * _Nullable mCapturer RTC_GUARDED_BY(mChecker);

  AVCaptureDevice* _Nonnull const mDevice RTC_GUARDED_BY(mChecker);

  VideoCaptureAdapter* _Nonnull const mAdapter RTC_GUARDED_BY(mChecker);

  RTCCameraVideoCapturer* _Nonnull const mCapturer RTC_GUARDED_BY(mChecker);

  // If capture has started, this is the capability it was started for. Written on the mChecker

  // thread only.

  mozilla::Maybe<VideoCaptureCapability> mCapability MOZ_GUARDED_BY(api_lock_);

  // The image type that mCapability maps to. Set in lockstep with mCapability.

  mozilla::Maybe<mozilla::CaptureStage::ImageType> mImageType MOZ_GUARDED_BY(api_lock_);

  // Id string uniquely identifying this capture source. Written on the mChecker thread only.

  mozilla::Maybe<mozilla::TrackingId> mTrackingId MOZ_GUARDED_BY(api_lock_);

  // Adds frame specific markers to the profiler while mTrackingId is set.

  // Adds frame specific markers to the profiler while mTrackingId is set. Callback thread only.

  mozilla::PerformanceRecorderMulti<mozilla::CaptureStage> mCaptureRecorder;

  mozilla::PerformanceRecorderMulti<mozilla::CopyVideoStage> mConversionRecorder;

  std::atomic<ProfilerThreadId> mCallbackThreadId;
@@ -69,8 +69,11 @@ class VideoCaptureAvFoundation : public VideoCaptureImpl { 


}  // namespace webrtc::videocapturemodule



@interface VideoCaptureAdapter : NSObject <RTC_OBJC_TYPE (RTCVideoCapturerDelegate)>

@property(nonatomic) webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable capturer;

@interface VideoCaptureAdapter : NSObject <RTCVideoCapturerDelegate> {

  webrtc::Mutex _mutex;

  webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable _capturer RTC_GUARDED_BY(_mutex);

}

- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer;

@end



#endif

dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm

+103 −86
Original line number Diff line number Diff line
@@ -73,33 +73,29 @@ AVCaptureDeviceFormat* _Nullable FindFormat(AVCaptureDevice* _Nonnull aDevice, 
}  // namespace



@implementation VideoCaptureAdapter

@synthesize capturer = _capturer;



- (void)capturer:(RTC_OBJC_TYPE(RTCVideoCapturer) * _Nonnull)capturer

    didCaptureVideoFrame:(RTC_OBJC_TYPE(RTCVideoFrame) * _Nonnull)frame {

  _capturer->StartFrameRecording(frame.width, frame.height);

  const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;

  RTC_OBJC_TYPE(RTCI420Buffer)* buffer = [[frame buffer] toI420];

  // Accessing the (intended-to-be-private) native buffer directly is hacky but lets us skip two

  // copies

  rtc::scoped_refptr<webrtc::I420BufferInterface> nativeBuffer = [buffer nativeI420Buffer];

  webrtc::VideoFrame nativeFrame = webrtc::VideoFrame::Builder()

                                       .set_video_frame_buffer(nativeBuffer)

                                       .set_rotation(ToNativeRotation(frame.rotation))

                                       .set_timestamp_us(timestamp_us)

                                       .build();

  _capturer->OnFrame(nativeFrame);

- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer {

  webrtc::MutexLock lock(&_mutex);

  _capturer = capturer;

}



- (void)capturer:(RTCVideoCapturer* _Nonnull)capturer

    didCaptureVideoFrame:(RTCVideoFrame* _Nonnull)frame {

  rtc::scoped_refptr<webrtc::videocapturemodule::VideoCaptureAvFoundation> cap;

  {

    webrtc::MutexLock lock(&_mutex);

    cap = rtc::scoped_refptr(_capturer);

  }

  if (!cap) return;

  cap->OnFrame(frame);

}

@end



namespace webrtc::videocapturemodule {

VideoCaptureAvFoundation::VideoCaptureAvFoundation(AVCaptureDevice* _Nonnull aDevice)

    : mDevice(aDevice),

      mAdapter([[VideoCaptureAdapter alloc] init]),

      mCapturer(nullptr),

      mCapturer([[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:mAdapter]),

      mCallbackThreadId() {

  {

  const char* uniqueId = [[aDevice uniqueID] UTF8String];

  size_t len = strlen(uniqueId);

  _deviceUniqueId = new (std::nothrow) char[len + 1];
@@ -108,10 +104,6 @@ VideoCaptureAvFoundation::VideoCaptureAvFoundation(AVCaptureDevice* _Nonnull aDe 
  }

}



  mAdapter.capturer = this;

  mCapturer = [[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:mAdapter];

}



VideoCaptureAvFoundation::~VideoCaptureAvFoundation() {

  // Must block until capture has fully stopped, including async operations.

  MOZ_RELEASE_ASSERT(mOkToDestroy);
@@ -158,6 +150,9 @@ int32_t VideoCaptureAvFoundation::StartCapture(const VideoCaptureCapability& aCa 
    }

  }



  [mAdapter setCapturer:this];



  {

    Monitor monitor("VideoCaptureAVFoundation::StartCapture");

    Monitor* copyableMonitor = &monitor;

    MonitorAutoLock lock(monitor);
@@ -176,12 +171,45 @@ int32_t VideoCaptureAvFoundation::StartCapture(const VideoCaptureCapability& aCa 
      monitor.Wait();

    }



  if (*rv == 0) {

    if (*rv != 0) {

      return *rv;

    }

  }



  MutexLock lock(&api_lock_);

  mCapability = Some(aCapability);

  mImageType = Some([type = aCapability.videoType] {

    switch (type) {

      case webrtc::VideoType::kI420:

        return CaptureStage::ImageType::I420;

      case webrtc::VideoType::kYUY2:

        return CaptureStage::ImageType::YUY2;

      case webrtc::VideoType::kYV12:

      case webrtc::VideoType::kIYUV:

        return CaptureStage::ImageType::YV12;

      case webrtc::VideoType::kUYVY:

        return CaptureStage::ImageType::UYVY;

      case webrtc::VideoType::kNV12:

        return CaptureStage::ImageType::NV12;

      case webrtc::VideoType::kNV21:

        return CaptureStage::ImageType::NV21;

      case webrtc::VideoType::kMJPEG:

        return CaptureStage::ImageType::MJPEG;

      case webrtc::VideoType::kRGB24:

      case webrtc::VideoType::kABGR:

      case webrtc::VideoType::kARGB:

      case webrtc::VideoType::kARGB4444:

      case webrtc::VideoType::kRGB565:

      case webrtc::VideoType::kARGB1555:

      case webrtc::VideoType::kBGRA:

      case webrtc::VideoType::kUnknown:

        // Unlikely, and not represented by CaptureStage::ImageType.

        return CaptureStage::ImageType::Unknown;

    }

    return CaptureStage::ImageType::Unknown;

  }());



  return *rv;

  return 0;

}



int32_t VideoCaptureAvFoundation::StopCapture() {
@@ -208,6 +236,9 @@ int32_t VideoCaptureAvFoundation::StopCapture() { 
  while (!done) {

    monitor.Wait();

  }



  [mAdapter setCapturer:nil];



  return 0;

}
@@ -222,10 +253,31 @@ int32_t VideoCaptureAvFoundation::CaptureSettings(VideoCaptureCapability& aSetti 
  return -1;

}



int32_t VideoCaptureAvFoundation::OnFrame(webrtc::VideoFrame& aFrame) {

  MutexLock lock(&api_lock_);

int32_t VideoCaptureAvFoundation::OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) {

  MaybeRegisterCallbackThread();

  if (MutexLock lock(&api_lock_); MOZ_LIKELY(mTrackingId)) {

    mCaptureRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aFrame.width,

                           aFrame.height, mImageType.valueOr(CaptureStage::ImageType::Unknown));

    if (mCapability && mCapability->videoType != webrtc::VideoType::kI420) {

      mConversionRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aFrame.width,

                                aFrame.height);

    }

  }



  const int64_t timestamp_us = aFrame.timeStampNs / rtc::kNumNanosecsPerMicrosec;

  RTCI420Buffer* buffer = [aFrame.buffer toI420];

  mConversionRecorder.Record(0);

  int32_t rv = DeliverCapturedFrame(aFrame);

  // Accessing the (intended-to-be-private) native buffer directly is hacky but lets us skip two

  // copies

  rtc::scoped_refptr<webrtc::I420BufferInterface> nativeBuffer = buffer.nativeI420Buffer;

  auto frame = webrtc::VideoFrame::Builder()

                   .set_video_frame_buffer(nativeBuffer)

                   .set_rotation(ToNativeRotation(aFrame.rotation))

                   .set_timestamp_us(timestamp_us)

                   .build();



  MutexLock lock(&api_lock_);

  int32_t rv = DeliverCapturedFrame(frame);

  mCaptureRecorder.Record(0);

  return rv;

}
@@ -241,41 +293,6 @@ void VideoCaptureAvFoundation::SetTrackingId(uint32_t aTrackingIdProcId) { 
  mTrackingId.emplace(TrackingId::Source::Camera, aTrackingIdProcId);

}



void VideoCaptureAvFoundation::StartFrameRecording(int32_t aWidth, int32_t aHeight) {

  MaybeRegisterCallbackThread();

  MutexLock lock(&api_lock_);

  if (MOZ_UNLIKELY(!mTrackingId)) {

    return;

  }

  auto fromWebrtcVideoType = [](webrtc::VideoType aType) -> CaptureStage::ImageType {

    switch (aType) {

      case webrtc::VideoType::kI420:

        return CaptureStage::ImageType::I420;

      case webrtc::VideoType::kYUY2:

        return CaptureStage::ImageType::YUY2;

      case webrtc::VideoType::kYV12:

        return CaptureStage::ImageType::YV12;

      case webrtc::VideoType::kUYVY:

        return CaptureStage::ImageType::UYVY;

      case webrtc::VideoType::kNV12:

        return CaptureStage::ImageType::NV12;

      case webrtc::VideoType::kNV21:

        return CaptureStage::ImageType::NV21;

      case webrtc::VideoType::kMJPEG:

        return CaptureStage::ImageType::MJPEG;

      default:

        return CaptureStage::ImageType::Unknown;

    }

  };

  mCaptureRecorder.Start(

      0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aWidth, aHeight,

      mCapability.map([&](const auto& aCap) { return fromWebrtcVideoType(aCap.videoType); })

          .valueOr(CaptureStage::ImageType::Unknown));

  if (mCapability && mCapability->videoType != webrtc::VideoType::kI420) {

    mConversionRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aWidth, aHeight);

  }

}



void VideoCaptureAvFoundation::MaybeRegisterCallbackThread() {

  ProfilerThreadId id = profiler_current_thread_id();

  if (MOZ_LIKELY(id == mCallbackThreadId)) {