Aws-cli: MemoryError thrown in aws cli 2 in buster-slim docker image

[passuied]

Running a simple aws ssm get-parameters command with aws cli 2 in ECS service throws the following error

[10] Failed to execute script pyiboot01_bootstrap
Traceback (most recent call last):
  File "site-packages/PyInstaller/loader/pyiboot01_bootstrap.py", line 127, in <module>
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "/codebuild/output/src194630121/src/repos/awscli/.tox/exe/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 627, in exec_module
  File "ctypes/__init__.py", line 543, in <module>
  File "ctypes/__init__.py", line 273, in _reset_cache

Here is a copy of the Dockerfile used

ARG TAG=3.1.2-buster-slim
FROM mcr.microsoft.com/dotnet/core/aspnet:${TAG} AS base-env
ENV DOTNET_USE_POLLING_FILE_WATCHER=true

RUN sed -i "s|DEFAULT@SECLEVEL=2|DEFAULT@SECLEVEL=1|g" /etc/ssl/openssl.cnf \
   && sed -i "s|TLSv1.2|TLSv1.0|g" /etc/ssl/openssl.cnf

RUN apt-get update \
    && apt-get install -y --no-install-recommends \
        unzip=6.0* \
        jq=1.5* \
        libidn2-0=2.0.5-1+deb10u1 -y \
    && apt-get clean && rm -rf /var/lib/apt/lists/*

WORKDIR /tmp

RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
    && unzip awscliv2.zip \
    && ./aws/install \
        && rm -rf /tmp/* /var/tmp/*

WORKDIR /

As a reference, here is the task definition for the service.

{
  "ipcMode": null,
  "executionRoleArn": null,
  "containerDefinitions": [
    {
      "dnsSearchDomains": [],
      "logConfiguration": {
        "logDriver": "awslogs",
        "secretOptions": [],
        "options": {
          "awslogs-group": "services",
          "awslogs-region": "us-west-1",
          "awslogs-stream-prefix": "qa-p-perf-user-rating-api"
        }
      },
      "entryPoint": [],
      "portMappings": [
        {
          "hostPort": 0,
          "protocol": "tcp",
          "containerPort": 8443
        },
        {
          "hostPort": 0,
          "protocol": "tcp",
          "containerPort": 8080
        }
      ],
      "command": [],
      "linuxParameters": {
        "capabilities": {
          "add": [],
          "drop": [
            "ALL"
          ]
        },
        "sharedMemorySize": null,
        "tmpfs": [
          {
            "mountOptions": [
              "rw",
              "noexec",
              "nosuid"
            ],
            "containerPath": "/tmp",
            "size": 65
          }
        ],
        "devices": [],
        "maxSwap": 0,
        "swappiness": 0,
        "initProcessEnabled": null
      },
      "cpu": 0,
      "environment": [
        // [...]
      ],
      "ulimits": [],
      "dnsServers": [],
      "mountPoints": [
        {
          "readOnly": null,
          "containerPath": "/var/log",
          "sourceVolume": "ServicesLog"
        }
      ],
      "workingDirectory": null,
      "secrets": [],
      "dockerSecurityOptions": [
        "no-new-privileges"
      ],
      "memory": 256,
      "memoryReservation": 255,
      "volumesFrom": [],
      "stopTimeout": null,
      "image": "XXXXXXXXXXXXXXX.dkr.ecr.us-west-1.amazonaws.com/perf-user-rating-api:XXXXXXXX",
      "startTimeout": null,
      "firelensConfiguration": null,
      "dependsOn": null,
      "disableNetworking": null,
      "interactive": null,
      "healthCheck": null,
      "essential": true,
      "links": [],
      "hostname": null,
      "extraHosts": [],
      "pseudoTerminal": null,
      "user": "csodservices",
      "readonlyRootFilesystem": true,
      "dockerLabels": {
        "environment": "qa-p",
        "product": "perf",
        "service": "perf-user-rating-api",
        "servicePath": "perf-user-rating-api"
      },
      "systemControls": [],
      "privileged": null,
      "name": "perf-user-rating-api"
    }
  ],
  "placementConstraints": [],
  "memory": null,
  "taskRoleArn": "arn:aws:iam::XXXXXXXXX:role/qa-p-perf-iam-TaskRoleUserRatingApi-198IYAM6NCREG",
  "compatibilities": [
    "EC2"
  ],
  "taskDefinitionArn": "arn:aws:ecs:us-west-1:XXXXXXXXX:task-definition/perf-user-rating-api-qa-p-common:3",
  "family": "perf-user-rating-api-qa-p-common",
  "requiresAttributes": [
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.ecr-auth"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.17"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.task-iam-role"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.22"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.23"
    },
    {
      "targetId": null,
      "targetType": null,
      "value": null,
      "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
    }
  ],
  "pidMode": null,
  "requiresCompatibilities": [],
  "networkMode": null,
  "cpu": null,
  "revision": 3,
  "status": "ACTIVE",
  "inferenceAccelerators": null,
  "proxyConfiguration": null,
  "volumes": [
    {
      "efsVolumeConfiguration": null,
      "name": "ServicesLog",
      "host": {
        "sourcePath": "/var/log"
      },
      "dockerVolumeConfiguration": null
    }
  ]
}

Please advise

[kyleknap]

We still need to investigate a bit more on this. I took the Dockerfile built it locally and was able to run the CLI just fine and did not encounter any memory errors. Are you able to get it working in a local container? Next we are going to try to reproduce it in ECS.

[passuied]

yes it's all working locally but only breaking when we deploy to ECS...

On Wed, Mar 11, 2020 at 1:56 PM Kyle Knapp [e-mail] wrote:

We still need to investigate a bit more on this. I took the Dockerfile
built it locally and was able to run the CLI just fine and did not
encounter any memory errors. Are you able to get it working in a local
container? Next we are going to try to reproduce it in ECS.

—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
https://github.com/aws/aws-cli/issues/5047#issuecomment-597874355, or
unsubscribe
https://github.com/notifications/unsubscribe-auth/AAHV5BSRYXOBHDPSF35FVNDRG73HDANCNFSM4LDWK4EQ
.

[passuied]

yes it's all working locally but only breaking when we deploy to ECS

[edlevin6612]

I get the same error when attempting to invoke aws cli v2 from an /etc/rc.local script on startup (Amazon Linux 2 EC2 instance, not Docker). Execution is successful if from the shell.

Ref:

aws --version aws-cli/2.0.8 Python/3.7.3 Linux/4.14.173-137.229.amzn2.x86_64 botocore/2.0.0dev12

[edlevin6612]

In my case, after much trial and error, overcame the problem by setting the HOME=/root environment var in the startup script. Perhaps this will work for Docker as well? If so this should definitely be documented 🙏

[akotsar]

@passuied et a.l,
I was able to reproduce this locally, it doesn't have to be in ECS. The problem is with read-only root file system.

Dockerfile:

FROM mcr.microsoft.com/dotnet/core/aspnet:3.1.3-buster-slim

RUN apt-get update \
    && apt-get install -y --no-install-recommends \
    unzip

WORKDIR /tmp

RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
    && unzip awscliv2.zip \
    && ./aws/install

Then run: docker run -it --rm --read-only --tmpfs /tmp <produced-image-id>

Inside the container, run aws:

root@835380e2f6cb:/tmp# aws
Traceback (most recent call last):
  File "PyInstaller/loader/pyiboot01_bootstrap.py", line 127, in <module>
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "/codebuild/output/src494604232/src/repos/awscli/.tox/exe/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 627, in exec_module
  File "ctypes/__init__.py", line 543, in <module>
  File "ctypes/__init__.py", line 273, in _reset_cache
MemoryError
[10] Failed to execute script pyiboot01_bootstrap

[passuied]

So is it both read only tmpfs and also the size of it? Or just the former that matters?

[akotsar]

I believe it's the read only root file system that is causing this. /tmp is R/W and the size doesn't matter in that case - at least I tried as much as 128MB and it didn't work (e.g. docker run -it --rm --read-only --mount type=tmpfs,destination=/tmp,tmpfs-size=134217728 <image-id>)

It's the --read-only flag that is killing it. Apparently, it tries to write/create something outside of /tmp. But what is it?

[passuied]

Makes sense. Will close the issue. Thanks guys

[akotsar]

I think the problem is still there. Why does it need a read/write root file system during runtime? We cannot use it as a result.

[ghunsl]

aws cli v2 has MemoryError failure if command line was executed during systemd bootup sequence (i.e. cloud-init) unless HOME environment variable is set. Thank you, @edlevin6612
for the workaround. Normally, this environment variable is not set unless you login to the instance. It didn't happen for aws cli v1. Please fix this issue. Why is this issue closed without providing proper fix for it?

[akotsar]

We are probably talking about different problems here.

1. Not having HOME defined breaking it.
2. Having a read-only root file system inside a container breaking it.

For example, I just tested it again and setting $HOME did not make any difference (note the first command echo $HOME:

root@b28956665675:/tmp# echo $HOME
/root
root@b28956665675:/tmp# aws
Traceback (most recent call last):
  File "PyInstaller/loader/pyiboot01_bootstrap.py", line 127, in <module>
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "/codebuild/output/src128717105/src/repos/awscli/.tox/exe/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 627, in exec_module
  File "ctypes/__init__.py", line 543, in <module>
  File "ctypes/__init__.py", line 273, in _reset_cache
MemoryError
[9] Failed to execute script pyiboot01_bootstrap

[yarzaryeni]

I have a php script that executes aws command.
<?php system("aws --version"); ?>

There is no problem to run php script as normal user:

[centos@server_name ~]$ php test.php
aws-cli/2.0.11 Python/3.7.3 Linux/3.10.0-1062.18.1.el7.x86_64 botocore/2.0.0dev15

When I tried to run the same php script via web server (I am using nginx web server and it is running as nginx user),

http://localhost/test.php

I got the following error.

[24660] Failed to execute script pyiboot01_bootstrap
Traceback (most recent call last):
  File "PyInstaller/loader/pyiboot01_bootstrap.py", line 127, in <module>
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "/codebuild/output/src166136877/src/repos/awscli/.tox/exe/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 627, in exec_module
  File "ctypes/__init__.py", line 543, in <module>
  File "ctypes/__init__.py", line 273, in _reset_cache
MemoryError

I am about to create a php web api server that run aws s3 sync command when it receive an api call. But I cannot proceed because of the above error.

[ashtuchkin]

I did an strace on it and here's where it fails:

[pid    22] open("/tmp/ffitWCYH9", O_RDWR|O_CREAT|O_EXCL, 0600) = -1 EROFS (Read-only file system)
[pid    22] open("/var/tmp/ffi3tUNPw", O_RDWR|O_CREAT|O_EXCL, 0600) = -1 EROFS (Read-only file system)
[pid    22] open("/dev/shm/ffiP9uEXT", O_RDWR|O_CREAT|O_EXCL, 0600) = 3
[pid    22] unlink("/dev/shm/ffiP9uEXT") = 0
[pid    22] ftruncate(3, 4096)          = 0
[pid    22] mmap(NULL, 4096, PROT_READ|PROT_EXEC, MAP_SHARED, 3, 0) = -1 EPERM (Operation not permitted)
[pid    22] close(3)                    = 0
[pid    22] open("/root/ffi3RkB5g", O_RDWR|O_CREAT|O_EXCL, 0600) = -1 EROFS (Read-only file system)

Basically pyinstaller looks at various places where it can put a temporary file and finds '/dev/shm'. But then it tries to memory-map it with PROT_EXEC flag (to unpack an executable there), and fails with EPERM. This happens if this filesystem is mounted with a no-exec flag.

I think confusion with $HOME variable happens because pyinstaller tries home directory as a last resort (see last line in the strace log above). It might be the case that home directory was mounted read-write and with 'exec' flag.

To fix this, you can mount a tmpfs with 'exec' flag: docker run --tmpfs=/tmp:exec ....