[packager] Optimize archive accesses.

python/mozbuild/mozpack/copier.py

@@ -535,7 +535,7 @@ class Jarrer(FileRegistry, BaseFile):
             dest = Dest(dest)
         assert isinstance(dest, Dest)
 
-        from mozpack.mozjar import JarWriter, JarReader
+        from mozpack.mozjar import JarWriter, JarReader, JAR_BROTLI
         try:
             old_jar = JarReader(fileobj=dest)
         except Exception:

python/mozbuild/mozpack/mozjar.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import
 
 from io import BytesIO
 import struct
+import subprocess
 import zlib
 import os
 from zipfile import (
@@ -15,9 +16,11 @@ from zipfile import (
 from collections import OrderedDict
 from urlparse import urlparse, ParseResult
 import mozpack.path as mozpath
+from mozbuild.util import memoize
 
 JAR_STORED = ZIP_STORED
 JAR_DEFLATED = ZIP_DEFLATED
+JAR_BROTLI = 0x81
 MAX_WBITS = 15
 
 
@@ -262,13 +265,14 @@ class JarFileReader(object):
         corresponding to the file in the jar archive, data a buffer containing
         the file data.
         '''
-        assert header['compression'] in [JAR_DEFLATED, JAR_STORED]
+        assert header['compression'] in [JAR_DEFLATED, JAR_STORED, JAR_BROTLI]
         self._data = data
         # Copy some local file header fields.
         for name in ['filename', 'compressed_size',
                      'uncompressed_size', 'crc32']:
             setattr(self, name, header[name])
-        self.compressed = header['compression'] == JAR_DEFLATED
+        self.compressed = header['compression'] != JAR_STORED
+        self.compress = header['compression']
 
     def read(self, length=-1):
         '''
@@ -317,7 +321,11 @@ class JarFileReader(object):
         if hasattr(self, '_uncompressed_data'):
             return self._uncompressed_data
         data = self.compressed_data
-        if self.compressed:
+        if self.compress == JAR_STORED:
+            data = data.tobytes()
+        elif self.compress == JAR_BROTLI:
+            data = Brotli.decompress(data.tobytes())
+        elif self.compress == JAR_DEFLATED:
             data = zlib.decompress(data.tobytes(), -MAX_WBITS)
         else:
             data = data.tobytes()
@@ -360,6 +368,13 @@ class JarReader(object):
         '''
         del self._data
 
+    @property
+    def compression(self):
+        entries = self.entries
+        if not entries:
+            return JAR_STORED
+        return max(f['compression'] for f in entries.itervalues())
+
     @property
     def entries(self):
         '''
@@ -473,6 +488,8 @@ class JarWriter(object):
             self._data = fileobj
         else:
             self._data = open(file, 'wb')
+        if compress is True:
+            compress = JAR_DEFLATED
         self._compress = compress
         self._compress_level = compress_level
         self._contents = OrderedDict()
@@ -574,12 +591,13 @@ class JarWriter(object):
         '''
         Add a new member to the jar archive, with the given name and the given
         data.
-        The compress option indicates if the given data should be compressed
+        The compress option indicates how the given data should be compressed
-        (True), not compressed (False), or compressed according to the default
+        (one of JAR_STORED, JAR_DEFLATE or JAR_BROTLI), or compressed according
-        defined when creating the JarWriter (None).
+        to the default defined when creating the JarWriter (None). True and
-        When the data should be compressed (True or None with self.compress ==
+        False are allowed values for backwards compatibility, mapping,
-        True), it is only really compressed if the compressed size is smaller
+        respectively, to JAR_DEFLATE and JAR_STORED.
-        than the uncompressed size.
+        When the data should be compressed, it is only really compressed if
+        the compressed size is smaller than the uncompressed size.
         The mode option gives the unix permissions that should be stored
         for the jar entry.
         If a duplicated member is found skip_duplicates will prevent raising
@@ -594,8 +612,12 @@ class JarWriter(object):
             raise JarWriterError("File %s already in JarWriter" % name)
         if compress is None:
             compress = self._compress
-        if (isinstance(data, JarFileReader) and data.compressed == compress) \
+        if compress is True:
-                or (isinstance(data, Deflater) and data.compress == compress):
+            compress = JAR_DEFLATED
+        if compress is False:
+            compress = JAR_STORED
+        if (isinstance(data, (JarFileReader, Deflater)) and \
+                data.compress == compress):
             deflater = data
         else:
             deflater = Deflater(compress, compress_level=self._compress_level)
@@ -619,7 +641,7 @@ class JarWriter(object):
         if deflater.compressed:
             entry['min_version'] = 20  # Version 2.0 supports deflated streams
             entry['general_flag'] = 2  # Max compression
-            entry['compression'] = JAR_DEFLATED
+            entry['compression'] = deflater.compress
         else:
             entry['min_version'] = 10  # Version 1.0 for stored streams
             entry['general_flag'] = 0
@@ -659,14 +681,21 @@ class Deflater(object):
     '''
     def __init__(self, compress=True, compress_level=9):
         '''
-        Initialize a Deflater. The compress argument determines whether to
+        Initialize a Deflater. The compress argument determines how to
-        try to compress at all.
+        compress.
         '''
         self._data = BytesIO()
+        if compress is True:
+            compress = JAR_DEFLATED
+        elif compress is False:
+            compress = JAR_STORED
         self.compress = compress
-        if compress:
+        if compress in (JAR_DEFLATED, JAR_BROTLI):
-            self._deflater = zlib.compressobj(compress_level, zlib.DEFLATED,
+            if compress == JAR_DEFLATED:
-                                              -MAX_WBITS)
+                self._deflater = zlib.compressobj(
+                    compress_level, zlib.DEFLATED, -MAX_WBITS)
+            else:
+                self._deflater = BrotliCompress()
             self._deflated = BytesIO()
         else:
             self._deflater = None
@@ -759,6 +788,46 @@ class Deflater(object):
         return self._data.getvalue()
 
 
+class Brotli(object):
+    @staticmethod
+    @memoize
+    def brotli_tool():
+            from buildconfig import topobjdir, substs
+            return os.path.join(topobjdir, 'dist', 'host', 'bin',
+                               'brotli' + substs.get('BIN_SUFFIX', ''))
+
+    @staticmethod
+    def run_brotli_tool(args, input):
+        proc = subprocess.Popen([Brotli.brotli_tool()] + args,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE)
+        (stdout, _) = proc.communicate(input)
+        ret = proc.wait()
+        if ret != 0:
+            raise Exception("Brotli compression failed")
+        return stdout
+
+    @staticmethod
+    def compress(data):
+        return Brotli.run_brotli_tool(['--lgwin=17'], data)
+
+    @staticmethod
+    def decompress(data):
+        return Brotli.run_brotli_tool(['--decompress'], data)
+
+
+class BrotliCompress(object):
+    def __init__(self):
+        self._buf = BytesIO()
+
+    def compress(self, data):
+        self._buf.write(data)
+        return b''
+
+    def flush(self):
+        return Brotli.compress(self._buf.getvalue())
+
+
 class JarLog(dict):
     '''
     Helper to read the file Gecko generates when setting MOZ_JAR_LOG_FILE.

python/mozbuild/mozpack/packager/l10n.py

@@ -37,6 +37,7 @@ from mozpack.chrome.manifest import (
     Manifest,
 )
 from mozpack.errors import errors
+from mozpack.mozjar import JAR_DEFLATED
 from mozpack.packager.unpack import UnpackFinder
 from createprecomplete import generate_precomplete
 
@@ -241,16 +242,17 @@ def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
             finders[base] = UnpackFinder(path)
         l10n_finder = ComposedFinder(finders)
     copier = FileCopier()
+    compress = min(app_finder.compressed, JAR_DEFLATED)
     if app_finder.kind == 'flat':
         formatter = FlatFormatter(copier)
     elif app_finder.kind == 'jar':
         formatter = JarFormatter(copier,
                                  optimize=app_finder.optimizedjars,
-                                 compress=app_finder.compressed)
+                                 compress=compress)
     elif app_finder.kind == 'omni':
         formatter = OmniJarFormatter(copier, app_finder.omnijar,
                                      optimize=app_finder.optimizedjars,
-                                     compress=app_finder.compressed,
+                                     compress=compress,
                                      non_resources=non_resources)
 
     with errors.accumulate():

python/mozbuild/mozpack/packager/unpack.py

@@ -54,7 +54,7 @@ class UnpackFinder(BaseFinder):
         self.omnijar = None
         self.jarlogs = {}
         self.optimizedjars = False
-        self.compressed = True
+        self.compressed = False
 
         jars = set()
 
@@ -146,8 +146,7 @@ class UnpackFinder(BaseFinder):
         jar = JarReader(fileobj=file.open())
         if jar.is_optimized:
             self.optimizedjars = True
-        if not any(f.compressed for f in jar):
+        self.compressed = max(self.compressed, jar.compression)
-            self.compressed = False
         if jar.last_preloaded:
             jarlog = jar.entries.keys()
             self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]

toolkit/mozapps/installer/packager.mk

@@ -50,7 +50,7 @@ stage-package: $(MOZ_PKG_MANIFEST) $(MOZ_PKG_MANIFEST_DEPS)
 		) \
 		$(if $(JARLOG_DIR),$(addprefix --jarlog ,$(wildcard $(JARLOG_FILE_AB_CD)))) \
 		$(if $(OPTIMIZEJARS),--optimizejars) \
-		$(if $(DISABLE_JAR_COMPRESSION),--disable-compression) \
+		$(addprefix --compress ,$(JAR_COMPRESSION)) \
 		$(addprefix --unify ,$(UNIFY_DIST)) \
 		$(MOZ_PKG_MANIFEST) '$(DIST)' '$(DIST)'/$(STAGEPATH)$(MOZ_PKG_DIR)$(if $(MOZ_PKG_MANIFEST),,$(_BINPATH)) \
 		$(if $(filter omni,$(MOZ_PACKAGER_FORMAT)),$(if $(NON_OMNIJAR_FILES),--non-resource $(NON_OMNIJAR_FILES)))

toolkit/mozapps/installer/packager.py

@@ -23,6 +23,7 @@ from mozpack.copier import (
     Jarrer,
 )
 from mozpack.errors import errors
+from mozpack.mozjar import JAR_BROTLI
 from mozpack.unify import UnifiedBuildFinder
 import mozpack.path as mozpath
 import buildconfig
@@ -270,9 +271,9 @@ def main():
                         help='Enable jar optimizations')
     parser.add_argument('--unify', default='',
                         help='Base directory of another build to unify with')
-    parser.add_argument('--disable-compression', action='store_false',
+    parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
-                        dest='compress', default=True,
+                        default='deflate',
-                        help='Disable jar compression')
+                        help='Use given jar compression (default: deflate)')
     parser.add_argument('manifest', default=None, nargs='?',
                         help='Manifest file name')
     parser.add_argument('source', help='Source directory')
@@ -290,15 +291,21 @@ def main():
         for name, value in [split_define(d) for d in args.defines]:
             defines[name] = value
 
+    compress = {
+        'none': False,
+        'deflate': True,
+        'brotli': JAR_BROTLI,
+    }[args.compress]
+
     copier = FileCopier()
     if args.format == 'flat':
         formatter = FlatFormatter(copier)
     elif args.format == 'jar':
-        formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
+        formatter = JarFormatter(copier, compress=compress, optimize=args.optimizejars)
     elif args.format == 'omni':
         formatter = OmniJarFormatter(copier,
                                      buildconfig.substs['OMNIJAR_NAME'],
-                                     compress=args.compress,
+                                     compress=compress,
                                      optimize=args.optimizejars,
                                      non_resources=args.non_resource)
     else:

toolkit/mozapps/installer/upload-files.mk

@@ -385,7 +385,7 @@ ifneq (android,$(MOZ_WIDGET_TOOLKIT))
   OPTIMIZEJARS = 1
   ifneq (gonk,$(MOZ_WIDGET_TOOLKIT))
     ifdef NIGHTLY_BUILD
-      DISABLE_JAR_COMPRESSION = 1
+      JAR_COMPRESSION ?= none
     endif
   endif
 endif