Django 4.2 on AWS - collectstatic is not uploading files to S3

This has been a nightmare to try and figure out. Nothing is working. Here is my code. eb-hooks log is always sending the files to the localhost

1084 static files copied to '/var/app/current/static'

Settings.py

USE_S3 = os.getenv('LOGNAME') == 'webapp'
CLOUDFRONT_DOMAIN = 'https://d3g5b3y2rsspt8.cloudfront.net'
AWS_STORAGE_BUCKET_NAME = env('AWS_STORAGE_BUCKET_NAME')
AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME
AWS_ACCESS_KEY_ID = env('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env('AWS_SECRET_ACCESS_KEY')
AWS_S3_REGION_NAME = env('AWS_S3_REGION_NAME')


if USE_S3:

    STATICFILES_LOCATION = 'static'
    MEDIAFILES_LOCATION = 'media'
    AWS_S3_SIGNATURE_VERSION = 's3v4'
    AWS_S3_FILE_OVERWRITE = True
    AWS_DEFAULT_ACL =  None
    AWS_QUERYSTRING_AUTH = True
    AWS_QUERYSTRING_EXPIRE = 40000
    AWS_S3_VERIFY = True
    AWS_S3_OBJECT_PARAMETERS = {'CacheControl': 'max-age=86400'}
    AWS_CLOUDFRONT_KEY_ID = env.str('AWS_CLOUDFRONT_KEY_ID').strip()
    AWS_CLOUDFRONT_KEY = env.str('AWS_CLOUDFRONT_KEY', multiline=True).encode('ascii').strip()
    MEDIA_URL = f'{AWS_S3_CUSTOM_DOMAIN}/{MEDIAFILES_LOCATION}/'
    STATIC_URL = '%s/%s/' % (AWS_S3_CUSTOM_DOMAIN,STATICFILES_LOCATION)

    STORAGES = {
        "default": {"BACKEND": 'custom_storages.MediaStorage'},
        "staticfiles": {"BACKEND": 'custom_storages.StaticStorage'},
        "OPTIONS": {
            "bucket_name": AWS_STORAGE_BUCKET_NAME,
            "region_name": AWS_S3_REGION_NAME,
            "verify": AWS_S3_VERIFY,
            "signature_version": AWS_S3_SIGNATURE_VERSION,
            "cloudfront_key_id": AWS_CLOUDFRONT_KEY_ID,
            "cloudfront_key": AWS_CLOUDFRONT_KEY,
        },
    }
else:
    STATIC_URL = 'static/'
    STATIC_ROOT = 'static/'
    MEDIA_URL = '/media/'
    MEDIA_ROOT = 'media/'

The signed urls seem to be working, as when I go to the website, the urls are there, but exactly none of my static files are there.

EDIT: When I click a link to a static url on my webpages, I am now getting a "access denied" error, "Query-string authentication requires the Signature, Expires and AWSAccessKeyId parameters"

My S3 policy resembles the one from the site Restricting access to Amazon S3 origin

{
"Version": "2012-10-17",
"Statement": {
    "Sid": "AllowCloudFrontServicePrincipalReadWrite",
    "Effect": "Allow",
    "Principal": {
        "Service": "cloudfront.amazonaws.com"
    },
    "Action": [
        "s3:GetObject",
        "s3:PutObject"
    ],
    "Resource": "arn:aws:s3:::<S3 bucket name>/*",
    "Condition": {
        "StringEquals": {
            "AWS:SourceArn": "arn:aws:cloudfront::<AWS account ID>:distribution/<CloudFront distribution ID>"
        }
    }
}

}

Edit 2: "no such key, The specified key does not exist." when trying to access the file from the website. Again the URL seems fine. For example:

<Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Key>static/css/assets/bootstrap.min.css</Key>

Solution

For anyone using the latest Django 4.2 version this will be super useful, some things have changed substantially with django-storages, see the documentation for that.

Collectstatic is still not working to copy the files, but that's because if you have a lot of files, AWS is too slow to upload it, so it doesn't upload anything at all. You have to upload everything manually.

USE_S3 = os.getenv('LOGNAME') == 'webapp'
CLOUDFRONT_DOMAIN = 'xxxxxxxxxxxxxxx.cloudfront.net'
AWS_QUERYSTRING_AUTH = True
AWS_STORAGE_BUCKET_NAME = env('AWS_STORAGE_BUCKET_NAME')
AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME
AWS_ACCESS_KEY_ID = env('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env('AWS_SECRET_ACCESS_KEY')
AWS_S3_REGION_NAME = env('AWS_S3_REGION_NAME')
AWS_S3_SIGNATURE_VERSION = 's3v4'
AWS_CLOUDFRONT_KEY_ID = env.str('AWS_CLOUDFRONT_KEY_ID').strip()
AWS_CLOUDFRONT_KEY = env.str('AWS_CLOUDFRONT_KEY', multiline=True).encode('ascii').strip()


if USE_S3:
    STATICFILES_LOCATION = 'static'
    MEDIAFILES_LOCATION = 'media'
    MEDIA_URL = f'https://{CLOUDFRONT_DOMAIN}/{MEDIAFILES_LOCATION}/'
    MEDIA_ROOT = f'https://{AWS_S3_CUSTOM_DOMAIN}/static/{MEDIAFILES_LOCATION}/'
    STATIC_URL = 'https://%s/%s/' % (CLOUDFRONT_DOMAIN,STATICFILES_LOCATION)
    STATIC_ROOT = 'https://%s/%s/static/' % (AWS_S3_CUSTOM_DOMAIN,STATICFILES_LOCATION)

    STORAGES = {
        "default": {"BACKEND": 'custom_storages.MediaStorage'},
        "staticfiles": {"BACKEND": 'custom_storages.StaticStorage'},
        "OPTIONS": {
            "bucket_name": AWS_STORAGE_BUCKET_NAME,
            "region_name": AWS_S3_REGION_NAME,
            "signature_version": AWS_S3_SIGNATURE_VERSION,
            "cloudfront_key_id": AWS_CLOUDFRONT_KEY_ID,
            "cloudfront_key": AWS_CLOUDFRONT_KEY,
            "custom_domain": CLOUDFRONT_DOMAIN,
    },
}
else:
    STATIC_URL = 'static/'
    STATIC_ROOT = os.path.join(DATA_DIR, STATIC_URL)
    MEDIA_URL = 'media/'
    MEDIA_ROOT = os.path.join(DATA_DIR, MEDIA_URL)