Terraform: Terraform failing when a log_group is manually deleted

Created on 7 Feb 2017  ·  5Comments  ·  Source: hashicorp/terraform

Terraform Version

Terraform 0.8.5
Terraform 0.7.3

Affected Resource(s)

Please list the resources as a list, for example:

  • aws_cloudwatch_log_subscription_filter
  • aws_cloudwatch_log_group

Terraform Configuration Files

Folder structure:
image

application/infra/main.tf

variable region {
}

variable env {
}

variable application {
}

variable unpublished_version {
}

variable fully_qualified_component_name {
}

module "component" {
    source = "../../infra"
    region="${var.region}"
    env="${var.env}"
    application="${var.application}"
    fully_qualified_component_name="${var.fully_qualified_component_name}"
    version="${var.unpublished_version}"
}

application/infra/vars.tfvars

account_id="12345"
region="ap-southeast-2"
env="dev"
application="ede6529e082a"
unpublished_version="0.0.3_1.gitcommit"
fully_qualified_component_name="component-api"

destroy/infra/main.tf

infra/main.tf

variable component_name {
  default = "component"
}

variable region {
}

variable env {
}


variable version {
}

variable fully_qualified_component_name {
}

variable application {
}

resource "aws_iam_role" "execution_role" {
  name = "${var.component_name}-${var.application}-${var.env}"
  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": [
            "lambda.amazonaws.com",
            "apigateway.amazonaws.com"
        ]
      }
    }
  ]
}
  EOF
}

resource "aws_iam_role_policy" "execution" {
  depends_on = ["aws_iam_role.execution_role"]
  name   = "${var.component_name}-${var.application}-${var.env}"
  role   = "${aws_iam_role.execution_role.id}"
  policy= <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
      {
          "Effect": "Allow",
          "Action": [
              "lambda:GetFunctionConfiguration",
              "lambda:InvokeFunction"
          ],
          "Resource": "arn:aws:lambda:*:*:*:*"
      },
      {
          "Effect": "Allow",
          "Action": [
              "s3:GetObject"
          ],
          "Resource": "arn:aws:s3:::*"
      }
  ]
}
  EOF
}

data "aws_s3_bucket_object" "lambda_package" {
  bucket = "my-bucket"
  key = "${var.fully_qualified_component_name}/${var.version}.zip"
}

resource "aws_lambda_function" "lambda_function" {
  function_name     = "${var.component_name}-${var.application}-${var.env}"
  description       = "description"
  role              = "${aws_iam_role.execution_role.arn}"
  handler           = "src/lambdas/Comp.handler"
  runtime           = "nodejs4.3"
  timeout           = 30
  s3_bucket         = "${data.aws_s3_bucket_object.lambda_package.bucket}"
  s3_key            = "${data.aws_s3_bucket_object.lambda_package.key}"
  s3_object_version = "${data.aws_s3_bucket_object.lambda_package.version_id}"
}

resource "aws_lambda_permission" "lambda_api_permission" {
  depends_on = ["aws_lambda_function.lambda_function"]
  statement_id  = "${var.component_name}-${var.env}-Lambda-AllowApiExecution"
  action        = "lambda:InvokeFunction"
  function_name = "${aws_lambda_function.lambda_function.arn}"
  principal     = "apigateway.amazonaws.com"
}

resource "aws_cloudwatch_log_group" "lambda" {
    name = "/aws/lambda/${aws_lambda_function.lambda_function.function_name}"
    retention_in_days = "7"
}

resource "aws_cloudwatch_log_subscription_filter" "kinesis" {
    depends_on = [ "aws_cloudwatch_log_group.lambda" ]
    name = "my-kinesis-subscription"
    log_group_name = "/aws/lambda/${aws_lambda_function.lambda_function.function_name}"
    filter_pattern = ""
    destination_arn = "arn:aws:kinesis:ap-southeast-2:xxxxxxxx:stream/cloudwatch-sumologic-stream-dev"
    role_arn = "arn:aws:iam::xxxxxxxxx:role/cloudwatch-sumo-role-dev"
}

generated .tfstate file

{
    "version": 3,
    "terraform_version": "0.8.5",
    "serial": 4,
    "lineage": "<guid>",
    "modules": [
        {
            "path": [
                "root"
            ],
            "outputs": {},
            "resources": {},
            "depends_on": []
        },
        {
            "path": [
                "root",
                "component"
            ],
            "outputs": {},
            "resources": {
                "aws_cloudwatch_log_subscription_filter.kinesis": {
                    "type": "aws_cloudwatch_log_subscription_filter",
                    "depends_on": [
                        "aws_cloudwatch_log_group.lambda",
                        "aws_lambda_function.lambda_function"
                    ],
                    "primary": {
                        "id": "cwlsf-<id>",
                        "attributes": {
                            "destination_arn": "arn:aws:kinesis:ap-southeast-2:<account id>:stream/cloudwatch-sumologic-stream-dev",
                            "id": "cwlsf-<id>",
                            "log_group_name": "/aws/lambda/component-<guid>-dev",
                            "name": "my-kinesis-subscription",
                            "role_arn": "arn:aws:iam::<account id>:role/cloudwatch-sumo-role-dev"
                        },
                        "meta": {},
                        "tainted": false
                    },
                    "deposed": [],
                    "provider": ""
                },
                "aws_iam_role.execution_role": {
                    "type": "aws_iam_role",
                    "depends_on": [],
                    "primary": {
                        "id": "<component name>",
                        "attributes": {
                            "arn": "arn:aws:iam::<account id>:role/<component name>",
                            "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"apigateway.amazonaws.com\",\"lambda.amazonaws.com\"]},\"Action\":\"sts:AssumeRole\"}]}",
                            "create_date": "2017-02-12T23:54:52Z",
                            "id": "<component name>",
                            "name": "<component name>",
                            "path": "/",
                            "unique_id": "<id>"
                        },
                        "meta": {},
                        "tainted": false
                    },
                    "deposed": [],
                    "provider": ""
                },
                "aws_lambda_permission.lambda_api_permission": {
                    "type": "aws_lambda_permission",
                    "depends_on": [
                        "aws_lambda_function.lambda_function",
                        "aws_lambda_function.lambda_function"
                    ],
                    "primary": {
                        "id": "component-dev-Lambda-AllowApiExecution",
                        "attributes": {
                            "action": "lambda:InvokeFunction",
                            "function_name": "arn:aws:lambda:ap-southeast-2:<account id>:function:<component name>",
                            "id": "component-dev-Lambda-AllowApiExecution",
                            "principal": "apigateway.amazonaws.com",
                            "statement_id": "component-dev-Lambda-AllowApiExecution"
                        },
                        "meta": {},
                        "tainted": false
                    },
                    "deposed": [],
                    "provider": ""
                }
            },
            "depends_on": []
        }
    ]
}

Debug Output

aws_cloudwatch_log_subscription_filter.kinesis: Error reading SubscriptionFilters for log group /aws/lambda/component-fde6529e082a-dev with name prefix my-kinesis-subscription: &awserr.requestError{awsError:(*awserr.baseError)(0xc4209a9280), statusCode:400, requestID:"2f13dc94-f17f-11e6-b08c-e161da5a1005"}

Panic Output

Expected Behavior

When running into the bug 11749 you should be able to run terraform destroy again and get back into a clean state

Actual Behavior

When a log_group is not found, terraform exits with a 400 error

image

Steps to Reproduce

  1. terraform get -update=true ./application/infra
  2. terraform apply -parallelism=1 -var-file=./application/vars.tfvars ./application/infra
  3. terraform apply -parallelism=1 ./destroy/infra
  4. see error
  5. terraform apply -parallelism=1 ./destroy/infra - continues to fail

Important Factoids

This was encountered when running into issue 11749. Issue 11749 caused some resources to be deleted before failing - log_group being one of them. With log_group removed we could no longer perform any terraform actions without getting the 400 error.

log_group has a kinesis stream subscribed.

References

Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example:

  • GH-11749
bug provideaws
Source
picture samuelkdavis

All 5 comments

@stack72 Maybe I'm reading this wrong, but the fix you put in doesn't seem to address the issue reported. Note the error message in your fixed lines and the reported error do not match.

Your fix accounts for a situation where the returned list of subscription filters does not contain the filter being refreshed,

where as the issue reported is that the loggroup itself is missing. The error is triggered on this line https://github.com/hashicorp/terraform/blob/v0.9.2/builtin/providers/aws/resource_aws_cloudwatch_log_subscription_filter.go#L137 preventing refresh from completing.

Let me know if you agree, I'll raise a new issue on the providers/aws repo.

foragerr picture foragerr on 7 Aug 2017
👍1

Experiencing the same thing with Terraform 0.9.8 - All Terraform actions are now blocked due to the log group issue which was caused by a destroy operation being disrupted by not being able to write state files.

brid picture brid on 14 Aug 2017

@brid my current workaround is to delete the subscription filter from state using terraform state rm

foragerr picture foragerr on 14 Aug 2017
👍1

opened new issue in the provider-aws repo. https://github.com/terraform-providers/terraform-provider-aws/issues/1405

foragerr picture foragerr on 14 Aug 2017

I'm going to lock this issue because it has been closed for _30 days_ ⏳. This helps our maintainers find and focus on the active issues.

If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.

hashibot[bot] picture hashibot[bot] on 7 Apr 2020
Was this page helpful?
0 / 5 - 0 ratings

Related issues

rnowosielski picture rnowosielski  ·  3Comments
ketzacoatl picture ketzacoatl  ·  3Comments
thebenwaters picture thebenwaters  ·  3Comments
Seraf picture Seraf  ·  3Comments
jrnt30 picture jrnt30  ·  3Comments