Add compression to CCT (#4718)

Also address a small bug in FLL whereby the compressed data was used no matter what (even if nil because of compression failure).

GoogleDataTransportCCTSupport/CHANGELOG.md

@@ -1,4 +1,6 @@
 # v1.3.1
+- Adds compression to requests to CCT.
+- Requests going to the FLL backend will only use compressed data when smaller.
 - Added extensive debug logging that can be turned on by changing
 GDT_VERBOSE_LOGGING to 1 in GDTCORConsoleLogger.h.
 

GoogleDataTransportCCTSupport/GDTCCTLibrary/GDTCCTCompressionHelper.m

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2020 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDTCCTLibrary/Private/GDTCCTCompressionHelper.h"
+
+#import <zlib.h>
+
+@implementation GDTCCTCompressionHelper
+
++ (nullable NSData *)gzippedData:(NSData *)data {
+#if defined(__LP64__) && __LP64__
+  // Don't support > 32bit length for 64 bit, see note in header.
+  if (data.length > UINT_MAX) {
+    return nil;
+  }
+#endif
+
+  const uint kChunkSize = 1024;
+
+  const void *bytes = [data bytes];
+  NSUInteger length = [data length];
+
+  int level = Z_DEFAULT_COMPRESSION;
+  if (!bytes || !length) {
+    return nil;
+  }
+
+  z_stream strm;
+  bzero(&strm, sizeof(z_stream));
+
+  int memLevel = 8;          // Default.
+  int windowBits = 15 + 16;  // Enable gzip header instead of zlib header.
+
+  int retCode;
+  if ((retCode = deflateInit2(&strm, level, Z_DEFLATED, windowBits, memLevel,
+                              Z_DEFAULT_STRATEGY)) != Z_OK) {
+    return nil;
+  }
+
+  // Hint the size at 1/4 the input size.
+  NSMutableData *result = [NSMutableData dataWithCapacity:(length / 4)];
+  unsigned char output[kChunkSize];
+
+  // Setup the input.
+  strm.avail_in = (unsigned int)length;
+  strm.next_in = (unsigned char *)bytes;
+
+  // Collect the data.
+  do {
+    // update what we're passing in
+    strm.avail_out = kChunkSize;
+    strm.next_out = output;
+    retCode = deflate(&strm, Z_FINISH);
+    if ((retCode != Z_OK) && (retCode != Z_STREAM_END)) {
+      deflateEnd(&strm);
+      return nil;
+    }
+    // Collect what we got.
+    unsigned gotBack = kChunkSize - strm.avail_out;
+    if (gotBack > 0) {
+      [result appendBytes:output length:gotBack];
+    }
+
+  } while (retCode == Z_OK);
+
+  // If the loop exits, it used all input and the stream ended.
+  NSAssert(strm.avail_in == 0,
+           @"Should have finished deflating without using all input, %u bytes left", strm.avail_in);
+  NSAssert(retCode == Z_STREAM_END,
+           @"thought we finished deflate w/o getting a result of stream end, code %d", retCode);
+
+  // Clean up.
+  deflateEnd(&strm);
+
+  return result;
+}
+
++ (BOOL)isGzipped:(NSData *)data {
+  const UInt8 *bytes = (const UInt8 *)data.bytes;
+  return (data.length >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b);
+}
+
+@end

GoogleDataTransportCCTSupport/GDTCCTLibrary/GDTCCTUploader.m

@@ -24,6 +24,7 @@
 #import <nanopb/pb_decode.h>
 #import <nanopb/pb_encode.h>
 
+#import "GDTCCTLibrary/Private/GDTCCTCompressionHelper.h"
 #import "GDTCCTLibrary/Private/GDTCCTNanopbHelpers.h"
 #import "GDTCCTLibrary/Private/GDTCCTPrioritizer.h"
 
@@ -115,12 +116,6 @@ NSNotificationName const GDTCCTUploadCompleteNotification = @"com.GDTCCTUploader
       return;
     }
     NSURL *serverURL = self.serverURL ? self.serverURL : [self defaultServerURL];
-    NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:serverURL];
-    NSString *userAgent = [NSString stringWithFormat:@"datatransport/%@ fllsupport/%@ apple/",
-                                                     kGDTCORVersion, kGDTCCTSupportSDKVersion];
-    [request setValue:userAgent forHTTPHeaderField:@"User-Agent"];
-    request.HTTPMethod = @"POST";
-    GDTCORLogDebug("CCT: request created: %@", request);
 
     id completionHandler = ^(NSData *_Nullable data, NSURLResponse *_Nullable response,
                              NSError *_Nullable error) {
@@ -165,8 +160,13 @@ NSNotificationName const GDTCCTUploadCompleteNotification = @"com.GDTCCTUploader
     self->_currentUploadPackage = package;
     NSData *requestProtoData =
         [self constructRequestProtoFromPackage:(GDTCORUploadPackage *)package];
+    NSData *gzippedData = [GDTCCTCompressionHelper gzippedData:requestProtoData];
+    BOOL usingGzipData = gzippedData != nil && gzippedData.length < requestProtoData.length;
+    NSData *dataToSend = usingGzipData ? gzippedData : requestProtoData;
+    NSURLRequest *request = [self constructRequestWithURL:serverURL data:dataToSend];
+    GDTCORLogDebug("CCT: request created: %@", request);
     self.currentTask = [self.uploaderSession uploadTaskWithRequest:request
-                                                          fromData:requestProtoData
+                                                          fromData:dataToSend
                                                  completionHandler:completionHandler];
     GDTCORLogDebug("%@", @"CCT: The upload task is about to begin.");
     [self.currentTask resume];
@@ -234,6 +234,28 @@ NSNotificationName const GDTCCTUploadCompleteNotification = @"com.GDTCCTUploader
   return data ? data : [[NSData alloc] init];
 }
 
+/** Constructs a request to CCT given a URL and request body data.
+ *
+ * @param URL The URL to send the request to.
+ * @param data The request body data.
+ * @return A new NSURLRequest ready to be sent to CCT.
+ */
+- (NSURLRequest *)constructRequestWithURL:(NSURL *)URL data:(NSData *)data {
+  BOOL isGzipped = [GDTCCTCompressionHelper isGzipped:data];
+  NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:URL];
+  [request setValue:@"application/x-protobuf" forHTTPHeaderField:@"Content-Type"];
+  if (isGzipped) {
+    [request setValue:@"gzip" forHTTPHeaderField:@"Content-Encoding"];
+  }
+  [request setValue:@"gzip" forHTTPHeaderField:@"Accept-Encoding"];
+  NSString *userAgent = [NSString stringWithFormat:@"datatransport/%@ cctsupport/%@ apple/",
+                                                   kGDTCORVersion, kGDTCCTSupportSDKVersion];
+  [request setValue:userAgent forHTTPHeaderField:@"User-Agent"];
+  request.HTTPMethod = @"POST";
+  [request setHTTPBody:data];
+  return request;
+}
+
 #pragma mark - GDTCORUploadPackageProtocol
 
 - (void)packageExpired:(GDTCORUploadPackage *)package {

GoogleDataTransportCCTSupport/GDTCCTLibrary/GDTFLLUploader.m

@@ -16,8 +16,6 @@
 
 #import "GDTCCTLibrary/Private/GDTFLLUploader.h"
 
-#import <zlib.h>
-
 #import <GoogleDataTransport/GDTCORConsoleLogger.h>
 #import <GoogleDataTransport/GDTCORPlatform.h>
 #import <GoogleDataTransport/GDTCORRegistrar.h>
@@ -26,6 +24,7 @@
 #import <nanopb/pb_decode.h>
 #import <nanopb/pb_encode.h>
 
+#import "GDTCCTLibrary/Private/GDTCCTCompressionHelper.h"
 #import "GDTCCTLibrary/Private/GDTCCTNanopbHelpers.h"
 #import "GDTCCTLibrary/Private/GDTFLLPrioritizer.h"
 
@@ -189,13 +188,13 @@ NSNotificationName const GDTFLLUploadCompleteNotification = @"com.GDTFLLUploader
     self->_currentUploadPackage = package;
     NSData *requestProtoData =
         [self constructRequestProtoFromPackage:(GDTCORUploadPackage *)package];
-    NSData *gzippedData = [GDTFLLUploader gzippedData:requestProtoData];
+    NSData *gzippedData = [GDTCCTCompressionHelper gzippedData:requestProtoData];
     BOOL usingGzipData = gzippedData != nil && gzippedData.length < requestProtoData.length;
     NSData *dataToSend = usingGzipData ? gzippedData : requestProtoData;
     NSURLRequest *request = [self constructRequestWithURL:serverURL data:dataToSend];
     GDTCORLogDebug("FLL: request created: %@", request);
     self.currentTask = [self.uploaderSession uploadTaskWithRequest:request
-                                                          fromData:gzippedData
+                                                          fromData:dataToSend
                                                  completionHandler:completionHandler];
     GDTCORLogDebug("%@", @"FLL: The upload task is about to begin.");
     [self.currentTask resume];
@@ -238,79 +237,6 @@ NSNotificationName const GDTFLLUploadCompleteNotification = @"com.GDTFLLUploader
 
 #pragma mark - Private helper methods
 
-/** Compresses the given data and returns a new data object.
- *
- * @note Reduced version from GULNSData+zlib.m of GoogleUtilities.
- * @return Compressed data, or nil if there was an error.
- */
-+ (nullable NSData *)gzippedData:(NSData *)data {
-#if defined(__LP64__) && __LP64__
-  // Don't support > 32bit length for 64 bit, see note in header.
-  if (data.length > UINT_MAX) {
-    return nil;
-  }
-#endif
-
-  const uint kChunkSize = 1024;
-
-  const void *bytes = [data bytes];
-  NSUInteger length = [data length];
-
-  int level = Z_DEFAULT_COMPRESSION;
-  if (!bytes || !length) {
-    return nil;
-  }
-
-  z_stream strm;
-  bzero(&strm, sizeof(z_stream));
-
-  int memLevel = 8;          // Default.
-  int windowBits = 15 + 16;  // Enable gzip header instead of zlib header.
-
-  int retCode;
-  if ((retCode = deflateInit2(&strm, level, Z_DEFLATED, windowBits, memLevel,
-                              Z_DEFAULT_STRATEGY)) != Z_OK) {
-    return nil;
-  }
-
-  // Hint the size at 1/4 the input size.
-  NSMutableData *result = [NSMutableData dataWithCapacity:(length / 4)];
-  unsigned char output[kChunkSize];
-
-  // Setup the input.
-  strm.avail_in = (unsigned int)length;
-  strm.next_in = (unsigned char *)bytes;
-
-  // Collect the data.
-  do {
-    // update what we're passing in
-    strm.avail_out = kChunkSize;
-    strm.next_out = output;
-    retCode = deflate(&strm, Z_FINISH);
-    if ((retCode != Z_OK) && (retCode != Z_STREAM_END)) {
-      deflateEnd(&strm);
-      return nil;
-    }
-    // Collect what we got.
-    unsigned gotBack = kChunkSize - strm.avail_out;
-    if (gotBack > 0) {
-      [result appendBytes:output length:gotBack];
-    }
-
-  } while (retCode == Z_OK);
-
-  // If the loop exits, it used all input and the stream ended.
-  NSAssert(strm.avail_in == 0,
-           @"Should have finished deflating without using all input, %u bytes left", strm.avail_in);
-  NSAssert(retCode == Z_STREAM_END,
-           @"thought we finished deflate w/o getting a result of stream end, code %d", retCode);
-
-  // Clean up.
-  deflateEnd(&strm);
-
-  return result;
-}
-
 /** Constructs data given an upload package.
  *
  * @param package The upload package used to construct the request proto bytes.

GoogleDataTransportCCTSupport/GDTCCTLibrary/Private/GDTCCTCompressionHelper.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2020 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** A class with methods to help with gzipped data. */
+@interface GDTCCTCompressionHelper : NSObject
+
+/** Compresses the given data and returns a new data object.
+ *
+ * @note Reduced version from GULNSData+zlib.m of GoogleUtilities.
+ * @return Compressed data, or nil if there was an error.
+ */
++ (nullable NSData *)gzippedData:(NSData *)data;
+
+/** Returns YES if the data looks like it was gzip compressed by checking for the gzip magic number.
+ *
+ * @note: From https://en.wikipedia.org/wiki/Gzip, gzip's magic number is 1f 8b.
+ * @return YES if the data appears gzipped, NO otherwise.
+ */
++ (BOOL)isGzipped:(NSData *)data;
+
+@end
+
+NS_ASSUME_NONNULL_END