Django-storages: Support for ManifestStaticFilesStorage

[pratyushsharma]

Does this library support versioning for static files

[leonsmith]

Yes it does add the ManifestFilesMixin to your storage class

https://github.com/django/django/blob/master/django/contrib/staticfiles/storage.py#L299

# storage.py
class StaticStorage(ManifestFilesMixin, S3Boto3Storage):
    pass

# settings.py
STATICFILES_STORAGE = 'storage.StaticStorage'

[monkut]

I was having an issue with the related manifest file not being found and added the following read_manifest(), save_manifest() methods:

class StaticStorage(ManifestFilesMixin, S3Boto3Storage):
    """
    Saves and looks up staticfiles.json in Project directory
    """

    location = settings.STATIC_LOCATION

    @property
    def key(self):
        return f"{self.location}/{self.manifest_name}"

    def read_manifest(self):
        contents = None
        try:
            buffer = BytesIO()
            S3_CLIENT.download_fileobj(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=self.key, Fileobj=buffer)
            contents = buffer.getvalue().decode("utf8")
        except Exception as e:
            logger.exception(e)
            logger.warning(f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/{self.location}/{self.manifest_name} not found!")
        return contents

    def save_manifest(self):
        payload = {"paths": self.hashed_files, "version": self.manifest_version}
        encoded_contents = json.dumps(payload).encode("utf-8")
        fo = BytesIO(encoded_contents)
        S3_CLIENT.upload_fileobj(fo, settings.AWS_STORAGE_BUCKET_NAME, self.key)

[WhyNotHugo]

@monkut Can you provide a diff? It's not obvious where the issue is there. :sweat:

[monkut]

Sorry looks like I forgot to add the error that occured... I'll try to revert and figure out what lead to this.

[monkut]

I think this one of the errors that lead me to implement the read_manifest() and save_manifest() methods.

  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/template/base.py", line 903, in render_annotated
    return self.render(context)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/wagtail/images/templatetags/wagtailimages_tags.py", line 118, in render
    return rendition.img_tag(resolved_attrs)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/wagtail/images/models.py", line 530, in img_tag
    attrs = self.attrs_dict.copy()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/wagtail/images/models.py", line 523, in attrs_dict
    ('src', self.url),
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/wagtail/images/models.py", line 503, in url
    return self.file.url
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/db/models/fields/files.py", line 62, in url
    return self.storage.url(self.name)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/utils/functional.py", line 224, in inner
    self._setup()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/core/files/storage.py", line 364, in _setup
    self._wrapped = get_storage_class()()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 379, in __init__
    self.hashed_files = self.load_manifest()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 389, in load_manifest
    content = self.read_manifest()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 383, in read_manifest
    with self.open(self.manifest_name) as manifest:
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/core/files/storage.py", line 36, in open
    return self._open(name, mode)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/storages/backends/s3boto3.py", line 526, in _open
    raise IOError('File does not exist: %s' % name)
OSError: File does not exist: media/staticfiles.json

Also, appears to occur on initial collectstatic (where the expected staticfiles.json doesn't yet exist):

...
File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/core/management/base.py", line 369, in execute
    output = self.handle(*args, **options)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 161, in handle
    if self.is_local_storage() and self.storage.location:
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/management/commands/collectstatic.py", line 215, in is_local_storage
    return isinstance(self.storage, FileSystemStorage)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/utils/functional.py", line 224, in inner
    self._setup()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 501, in _setup
    self._wrapped = get_storage_class(settings.STATICFILES_STORAGE)()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 379, in __init__
    self.hashed_files = self.load_manifest()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 389, in load_manifest
    content = self.read_manifest()
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/contrib/staticfiles/storage.py", line 383, in read_manifest
    with self.open(self.manifest_name) as manifest:
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/django/core/files/storage.py", line 36, in open
    return self._open(name, mode)
  File "/home/user/.local/share/virtualenvs/cms-gTxZnR9B/lib/python3.8/site-packages/storages/backends/s3boto3.py", line 526, in _open
    raise IOError('File does not exist: %s' % name)
OSError: File does not exist: static/staticfiles.json

[dabrowne]

Just noting my solution to this here:

# storage.py
class ManifestS3Boto3Storage(ManifestFilesMixin, S3Boto3Storage):
    def read_manifest(self):
        # ManifestFilesMixin compat with S3Boto3Storage
        try:
            return super().read_manifest()
        except IOError:
            return None

# settings.py
STATICFILES_STORAGE = 'storage.ManifestS3Boto3Storage'