feat: pause支持退后台后再播放

iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/Decoders/QGMP4FrameHWDecoder.m

@@ -86,6 +86,8 @@
     NSInteger _finishFrameIndex;
     NSError *_constructErr;
     QGMP4ParserProxy *_mp4Parser;
+    
+    int _invalidRetryCount;
 }
 
 @property (atomic, strong) dispatch_queue_t decodeQueue; //dispatch decode task
@@ -148,14 +150,11 @@ NSString *const QGMP4HWDErrorDomain = @"QGMP4HWDErrorDomain";
 }
 
 - (void)registerNotification {
-    
-    [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveEnterBackgroundNotification:) name:UIApplicationDidEnterBackgroundNotification object:nil];
-    [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveEnterBackgroundNotification:) name:UIApplicationWillResignActiveNotification object:nil];
+
 }
 
 - (void)hwd_didReceiveEnterBackgroundNotification:(NSNotification *)notification {
     
-    [self onInputEnd];
 }
 
 - (void)decodeFrame:(NSInteger)frameIndex buffers:(NSMutableArray *)buffers {
@@ -167,12 +166,11 @@ NSString *const QGMP4HWDErrorDomain = @"QGMP4HWDErrorDomain";
     self.currentDecodeFrame = frameIndex;
     _buffers = buffers;
     dispatch_async(self.decodeQueue, ^{
-        [self _decodeFrame:frameIndex];
+        [self _decodeFrame:frameIndex drop:NO];
     });
 }
 
-- (void)_decodeFrame:(NSInteger)frameIndex {
-    
+- (void)_decodeFrame:(NSInteger)frameIndex drop:(BOOL)dropFlag {
     if (_isFinish) {
         return ;
     }
@@ -219,74 +217,113 @@ NSString *const QGMP4HWDErrorDomain = @"QGMP4HWDErrorDomain";
     if (blockBuffer) {
         CFRelease(blockBuffer);
     }
+    
     // 7. use VTDecompressionSessionDecodeFrame
     if (@available(iOS 9.0, *)) {
         __typeof(self) __weak weakSelf = self;
         VTDecodeFrameFlags flags = 0;
         VTDecodeInfoFlags flagOut = 0;
-        VTDecompressionSessionDecodeFrameWithOutputHandler(_mDecodeSession, sampleBuffer, flags, &flagOut, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef  _Nullable imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
-            CFRelease(sampleBuffer);
+        OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(_mDecodeSession, sampleBuffer, flags, &flagOut, ^(OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef  _Nullable imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
             __typeof(self) strongSelf = weakSelf;
             if (strongSelf == nil) {
                 return;
             }
             
-            if(status == kVTInvalidSessionErr) {
-                VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ kVTInvalidSessionErr error:%@", @(frameIndex), @(status));
-            } else if(status == kVTVideoDecoderBadDataErr) {
-                VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ kVTVideoDecoderBadDataErr error:%@", @(frameIndex), @(status));
-            } else if(status != noErr) {
-                VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ error:%@", @(frameIndex), @(status));
+            [strongSelf handleDecodePixelBuffer:imageBuffer
+                                   sampleBuffer:sampleBuffer
+                                     frameIndex:frameIndex
+                                     currentPts:currentPts
+                                      startDate:startDate
+                                         status:status
+                                       needDrop:dropFlag];
+        });
+        
+        if (status == kVTInvalidSessionErr) {
+            // 防止陷入死循环
+            if (_invalidRetryCount >= 3) {
+                return;
             }
             
-            QGMP4AnimatedImageFrame *newFrame = [[QGMP4AnimatedImageFrame alloc] init];
-            // imagebuffer会在frame回收时释放
-            CVPixelBufferRetain(imageBuffer);
-            newFrame.pixelBuffer = imageBuffer;
-            newFrame.frameIndex = frameIndex; //dts顺序
-            NSTimeInterval decodeTime = [[NSDate date] timeIntervalSinceDate:startDate]*1000;
-            newFrame.decodeTime = decodeTime;
-            newFrame.defaultFps =(int) strongSelf->_mp4Parser.fps;
-            newFrame.pts = currentPts;
-            
-            // 8. insert into buffer
-            [strongSelf->_buffers addObject:newFrame];
-            
-            // 9. sort
-            [strongSelf->_buffers sortUsingComparator:^NSComparisonResult(QGMP4AnimatedImageFrame * _Nonnull obj1, QGMP4AnimatedImageFrame * _Nonnull obj2) {
-                return [@(obj1.pts) compare:@(obj2.pts)];
-            }];
-        });
+            [self resetDecoder];
+            // 从最近I帧一直解码到当前帧，中间帧丢弃
+            [self findKeyFrameAndDecodeToCurrent:frameIndex];
+        } else {
+            _invalidRetryCount = 0;
+        }
+        
     } else {
         // 7. use VTDecompressionSessionDecodeFrame
         VTDecodeFrameFlags flags = 0;
         VTDecodeInfoFlags flagOut = 0;
         _status = VTDecompressionSessionDecodeFrame(_mDecodeSession, sampleBuffer, flags, &outputPixelBuffer, &flagOut);
         
-        if(_status == kVTInvalidSessionErr) {
-        } else if(_status == kVTVideoDecoderBadDataErr) {
-        } else if(_status != noErr) {
+        if (_status == kVTInvalidSessionErr) {
+            // 防止陷入死循环
+            if (_invalidRetryCount >= 3) {
+                return;
+            }
+            
+            [self resetDecoder];
+            // 从最近I帧一直解码到当前帧，中间帧丢弃
+            [self findKeyFrameAndDecodeToCurrent:frameIndex];
+            
+            return;
+        } else {
+            _invalidRetryCount = 0;
         }
-        CFRelease(sampleBuffer);
-        
-        QGMP4AnimatedImageFrame *newFrame = [[QGMP4AnimatedImageFrame alloc] init];
-        // imagebuffer会在frame回收时释放
-        newFrame.pixelBuffer = outputPixelBuffer;
-        newFrame.frameIndex = frameIndex;
-        NSTimeInterval decodeTime = [[NSDate date] timeIntervalSinceDate:startDate]*1000;
-        newFrame.decodeTime = decodeTime;
-        newFrame.defaultFps = (int)_mp4Parser.fps;
         
-        // 8. insert into buffer
-        [_buffers addObject:newFrame];
+        [self handleDecodePixelBuffer:outputPixelBuffer
+                         sampleBuffer:sampleBuffer
+                           frameIndex:frameIndex
+                           currentPts:currentPts
+                            startDate:startDate
+                               status:_status
+                             needDrop:dropFlag];
         
-        // 9. sort
-        [_buffers sortUsingComparator:^NSComparisonResult(QGMP4AnimatedImageFrame * _Nonnull obj1, QGMP4AnimatedImageFrame * _Nonnull obj2) {
-            return [@(obj1.pts) compare:@(obj2.pts)];
-        }];
     }
 }
 
+- (void)handleDecodePixelBuffer:(CVPixelBufferRef)pixelBuffer
+                   sampleBuffer:(CMSampleBufferRef)sampleBuffer
+                     frameIndex:(NSInteger)frameIndex
+                     currentPts:(uint64_t)currentPts
+                      startDate:(NSDate *)startDate
+                         status:(OSStatus)status
+                       needDrop:(BOOL)dropFlag {
+    
+    CFRelease(sampleBuffer);
+    
+    if(status == kVTInvalidSessionErr) {
+        VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ kVTInvalidSessionErr error:%@", @(frameIndex), @(status));
+    } else if(status == kVTVideoDecoderBadDataErr) {
+        VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ kVTVideoDecoderBadDataErr error:%@", @(frameIndex), @(status));
+    } else if(status != noErr) {
+        VAP_Error(kQGVAPModuleCommon, @"decompress fail! frame:%@ error:%@", @(frameIndex), @(status));
+    }
+    
+    if (dropFlag) {
+        return;
+    }
+    
+    QGMP4AnimatedImageFrame *newFrame = [[QGMP4AnimatedImageFrame alloc] init];
+    // imagebuffer会在frame回收时释放
+    CVPixelBufferRetain(pixelBuffer);
+    newFrame.pixelBuffer = pixelBuffer;
+    newFrame.frameIndex = frameIndex; //dts顺序
+    NSTimeInterval decodeTime = [[NSDate date] timeIntervalSinceDate:startDate]*1000;
+    newFrame.decodeTime = decodeTime;
+    newFrame.defaultFps = (int)_mp4Parser.fps;
+    newFrame.pts = currentPts;
+    
+    // 8. insert into buffer
+    [_buffers addObject:newFrame];
+    
+    // 9. sort
+    [_buffers sortUsingComparator:^NSComparisonResult(QGMP4AnimatedImageFrame * _Nonnull obj1, QGMP4AnimatedImageFrame * _Nonnull obj2) {
+        return [@(obj1.pts) compare:@(obj2.pts)];
+    }];
+}
+
 #pragma mark - override
 
 - (BOOL)shouldStopDecode:(NSInteger)nextFrameIndex {
@@ -391,6 +428,10 @@ NSString *const QGMP4HWDErrorDomain = @"QGMP4HWDErrorDomain";
     }
     
     // 3. create VTDecompressionSession
+    return [self createDecompressionSession];;
+}
+
+- (BOOL)createDecompressionSession {
     CFDictionaryRef attrs = NULL;
     const void *keys[] = {kCVPixelBufferPixelFormatTypeKey};
     //      kCVPixelFormatType_420YpCbCr8Planar is YUV420
@@ -431,6 +472,39 @@ NSString *const QGMP4HWDErrorDomain = @"QGMP4HWDErrorDomain";
     return YES;
 }
 
+- (void)resetDecoder {
+    // delete
+    if (_mDecodeSession) {
+        VTDecompressionSessionWaitForAsynchronousFrames(_mDecodeSession);
+        VTDecompressionSessionInvalidate(_mDecodeSession);
+        CFRelease(_mDecodeSession);
+        _mDecodeSession = NULL;
+    }
+    
+    // recreate
+    [self createDecompressionSession];
+}
+
+- (void)findKeyFrameAndDecodeToCurrent:(NSInteger)frameIndex {
+    NSArray<NSNumber *> *keyframeIndexes = [_mp4Parser videoSyncSampleIndexes];
+    NSInteger index = [[keyframeIndexes firstObject] integerValue];
+    for(NSNumber *number in keyframeIndexes) {
+        if(number.integerValue < frameIndex) {
+            index = number.integerValue;
+            continue;
+        } else {
+            break;
+        }
+    }
+    
+    // seek to last key frame
+    while (index < frameIndex) {
+        [self _decodeFrame:index drop:YES];
+        index++;
+    }
+    [self _decodeFrame:frameIndex drop:NO];
+}
+
 - (void)_onInputEnd  {
     if (_isFinish) {
         return ;

iOS/QGVAPlayer/QGVAPlayer/Classes/Controllers/QGAnimatedImageDecodeManager.m

@@ -46,7 +46,7 @@
         _decoderDelegate = delegate;
         [self createDecodersByConfig:config];
         _bufferManager = [[QGAnimatedImageBufferManager alloc] initWithConfig:config];
-        [self initializeBuffers];
+        [self initializeBuffersFromIndex:0];
         [self setupAudioPlayerIfNeed];
     }
     return self;
@@ -65,13 +65,27 @@
         if (frameIndex == 0 && _bufferManager.buffers.count < _config.bufferCount) {
             return nil;
         }
-        [self checkIfDecodeFinish:frameIndex];
+        BOOL decodeFinish = [self checkIfDecodeFinish:frameIndex];
         QGBaseAnimatedImageFrame *frame = [_bufferManager popVideoFrame];
         if (frame) {
             // pts顺序
             frame.frameIndex = frameIndex;
             [self decodeFrame:frameIndex+_config.bufferCount];
         }
+        else if (!decodeFinish){
+            // buffer已经空了，但还没有结束（退后台时可能出现这种情况）
+            NSInteger decoderIndex = _decoders.count==1?0:frameIndex%_decoders.count;
+            QGBaseDecoder *decoder = _decoders[decoderIndex];
+            if ([decoder shouldStopDecode:frameIndex]) {
+                // 其实已经该结束了
+                if ([self.decoderDelegate respondsToSelector:@selector(decoderDidFinishDecode:)]) {
+                    [self.decoderDelegate decoderDidFinishDecode:decoder];
+                }
+                return nil;
+            }
+            
+            [self initializeBuffersFromIndex:frameIndex];
+        }
         return frame;
     }
 }
@@ -85,7 +99,7 @@
 
 #pragma mark - private methods
 
-- (void)checkIfDecodeFinish:(NSInteger)frameIndex {
+- (BOOL)checkIfDecodeFinish:(NSInteger)frameIndex {
     
     NSInteger decoderIndex = _decoders.count==1?0:frameIndex%_decoders.count;
     QGBaseDecoder *decoder = _decoders[decoderIndex];
@@ -93,7 +107,9 @@
         if ([self.decoderDelegate respondsToSelector:@selector(decoderDidFinishDecode:)]) {
             [self.decoderDelegate decoderDidFinishDecode:decoder];
         }
+        return YES;
     }
+    return NO;
 }
 
 - (void)decodeFrame:(NSInteger)frameIndex {
@@ -134,10 +150,10 @@
     }
 }
 
-- (void)initializeBuffers {
+- (void)initializeBuffersFromIndex:(NSInteger)start {
     
     for (int i = 0; i < _config.bufferCount; i++) {
-        [self decodeFrame:i];
+        [self decodeFrame:start+i];
     }
 }
 

iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.h

@@ -168,6 +168,12 @@ The table is compactly coded. Each entry gives the index of the first chunk of a
 
 @end
 
+@interface QGMP4StssBox : QGMP4Box
+
+@property(nonatomic, strong) NSMutableArray<NSNumber *> *syncSamples;
+
+@end
+
 /**
  * ctts
  */

iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Box.m

@@ -257,6 +257,23 @@ NSInteger const kQGBoxTypeLengthInBytes = 4;
 
 @end
 
+@implementation QGMP4StssBox
+
+- (void)boxDidParsed:(QGMp4BoxDataFetcher)datablock {
+    if (!_syncSamples) {
+        _syncSamples = [NSMutableArray new];
+    }
+    NSData *stssData = datablock(self);
+    const char *bytes = stssData.bytes;
+    uint32_t sample_count = READ32BIT(&bytes[12]);
+    for (int i = 0; i < sample_count; i++) {
+        NSInteger index = READ32BIT(&bytes[16 + 4 * i]) - 1;
+        [_syncSamples addObject:[NSNumber numberWithInteger:index]];
+    }
+}
+
+@end
+
 /**
  Decoding Time to Sample Box
  用来计算dts
@@ -341,7 +358,6 @@ stts记录了sample的时间信息，⾥⾯有多个entry，每个entry⾥⾯的
         case QGMP4BoxType_url:
         case QGMP4BoxType_stbl:
         case QGMP4BoxType_avc1:
-        case QGMP4BoxType_stss:
         case QGMP4BoxType_udta:
         case QGMP4BoxType_meta:
         case QGMP4BoxType_ilst:
@@ -351,6 +367,8 @@ stts记录了sample的时间信息，⾥⾯有多个entry，每个entry⾥⾯的
         case QGMP4BoxType_loci:
         case QGMP4BoxType_smhd:
             return [QGMP4Box class];
+        case QGMP4BoxType_stss:
+            return [QGMP4StssBox class];
         case QGMP4BoxType_mdat:
             return [QGMP4MdatBox class];
         case QGMP4BoxType_avcC:

iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Parser.h

@@ -49,6 +49,7 @@
 @property (nonatomic, strong) NSData *spsData;          //sps
 @property (nonatomic, strong) NSData *ppsData;          //pps
 @property (nonatomic, strong) NSArray *videoSamples;    //所有帧数据，包含了位置和大小等信息
+@property (nonatomic, strong) NSArray *videoSyncSampleIndexes;  // 所有关键帧的index
 @property (nonatomic, strong) QGMP4Box *rootBox;        //mp4文件根box
 @property (nonatomic, strong) QGMP4TrackBox *videoTrackBox;     //视频track
 @property (nonatomic, strong) QGMP4TrackBox *audioTrackBox;     //音频track

iOS/QGVAPlayer/QGVAPlayer/Classes/MP4Parser/QGMP4Parser.m

@@ -320,6 +320,11 @@ void readBoxTypeAndLength(NSFileHandle *fileHandle, unsigned long long offset, Q
     return _videoSamples;
 }
 
+- (NSArray *)videoSyncSampleIndexes {
+    QGMP4StssBox *stssBox = [self.videoTrackBox subBoxOfType:QGMP4BoxType_stss];
+    return stssBox.syncSamples;
+}
+
 /**
  调用该方法才会解析mp4文件并得到必要信息。
  */

iOS/QGVAPlayer/QGVAPlayer/Classes/UIView+VAP.m

@@ -65,13 +65,11 @@ NSInteger const VapMaxCompatibleVersion = 2;
 - (void)hwd_registerNotification {
     
     [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveEnterBackgroundNotification:) name:UIApplicationDidEnterBackgroundNotification object:nil];
-    [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveEnterBackgroundNotification:) name:UIApplicationWillResignActiveNotification object:nil];
     [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveWillEnterForegroundNotification:) name:UIApplicationWillEnterForegroundNotification object:nil];
-    [[NSNotificationCenter defaultCenter] hwd_addSafeObserver:self selector:@selector(hwd_didReceiveWillEnterForegroundNotification:) name:UIApplicationDidBecomeActiveNotification object:nil];
 }
 
 - (void)hwd_didReceiveEnterBackgroundNotification:(NSNotification *)notification {
-    [self hwd_stopHWDMP4];
+    [self pauseHWDMP4];
 }
 
 //结束播放