Terraform: cidrsubnet calculation not translating properly

Created on 1 Dec 2017  ·  6Comments  ·  Source: hashicorp/terraform

Terraform Version

Terraform 10.8
 provider.aws 0.1.4
 provider.random 0.1.0
 provider.template 0.1.1

Terraform Configuration Files

###########
# subnets
###########

/*
 ecs subnet public
*/
resource "aws_subnet" "ecs_subnet_pub" {
  count                   = "${var.amount_public_ecs_subnets}"
  cidr_block              = "${cidrsubnet(var.cidr_pub, var.newbits, (var.netnum_public_ecs + (32 * count.index)))}"
  vpc_id                  = "${aws_vpc.dev-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = true
  depends_on              = ["aws_internet_gateway.aee-dev-east-ig"]

  tags {
    Name = "${var.tags["Name"]}-ecs-pub-${count.index}-subnet"
  }
}

/*
 ecs subnet private
*/
resource "aws_subnet" "ecs_subnet_prv" {
  count                   = "${var.amount_private_ecs_subnets}"
  cidr_block              = "${cidrsubnet(var.cidr_prv, var.newbits, (var.netnum_private_ecs + (16 * count.index)))}"
  vpc_id                  = "${aws_vpc.dev-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = false

  tags {
    Name = "${var.tags["Name"]}-ecs-prv-${count.index}-subnet"
  }
}

/*
 database subnet private
*/
resource "aws_subnet" "db_subnet_prv" {
  count                   = "${var.amount_private_db_subnets}"
  cidr_block              = "${cidrsubnet(var.cidr_prv, var.newbits, (var.netnum_private_db + (16 * count.index)))}"
  vpc_id                  = "${aws_vpc.dev-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = false

  tags {
    Name = "${var.tags["Name"]}-db-prv-${count.index}-subnet"
  }
}

/*
 load balancer subnet private
*/
resource "aws_subnet" "lb_subnet_prv" {
  count                   = "${var.amount_private_lb_subnets}"
  cidr_block              = "${cidrsubnet(var.cidr_prv, var.newbits, (var.netnum_private_lb + (16 * count.index)))}"
  vpc_id                  = "${aws_vpc.dev-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = false

  tags {
    Name = "${var.tags["Name"]}-alb-prv-${count.index}-subnet"
  }
}

/*
  rds aurora customer notification database subnet group
*/
resource "aws_db_subnet_group" "custdb_subnet_group" {
  name        = "custdb_subnet_group"
  description = "custdb subnet group"
  subnet_ids  = ["${element(aws_subnet.db_subnet_prv.*.id, count.index)}"]
}

/*
  rds aurora email notification database subnet group
*/

resource "aws_db_subnet_group" "emaildb_subnet_group" {
  name        = "emaildb_subnet_group"
  description = "emaildb subnet group"
  subnet_ids  = ["${element(aws_subnet.db_subnet_prv.*.id, count.index)}"]
}

/*
  rds aurora text notification database subnet group
*/

resource "aws_db_subnet_group" "textdb_subnet_group" {
  name        = "textdb_subnet_group"
  description = "textdb subnet group"
  subnet_ids  = ["${element(aws_subnet.db_subnet_prv.*.id, count.index)}"]
}

/*
  rds aurora voice notification database subnet group
*/
resource "aws_db_subnet_group" "voicedb_subnet_group" {
  name        = "voicedb_subnet_group"
  description = "voicedb subnet group"
  subnet_ids  = ["${element(aws_subnet.db_subnet_prv.*.id, count.index)}"]
}

###########
# data
###########

/*
 availability zones data template
*/
data "aws_availability_zones" "available" {}

/*
 caller identity data template
*/
data "aws_caller_identity" "current" {}

/*
 ecs ami data template
*/
data "aws_ami" "ecs_optimized" {
  most_recent = true

  filter {
    name   = "name"
    values = ["*amazon-ecs-optimized"]
  }

  filter {
    name   = "architecture"
    values = ["x86_64"]
  }

  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }

  # Amazon
  owners = ["1234567890123"]
}

/*
 ecs task definition data template
*/
data "template_file" "task_definition" {
  template = "${file("${path.module}/files/task-definition.json")}"

  vars {
    image_name     = "ghost:latest"
    container_name = "api"
    container_port = 2368
    host_port      = 8080
  }
}

###########
# variables
###########

variable "netnum_public_ecs" {
  type    = "string"
  default = "0"
}
variable "netnum_private_ecs" {
  type    = "string"
  default = "96"
}

variable "netnum_private_db" {
  type    = "string"
  default = "144"
}

variable "netnum_private_lb" {
  type    = "string"
  default = "192"
}

variable "amount_private_db_subnets" {
  type    = "string"
  default = "3"
}

variable "amount_public_ecs_subnets" {
  type    = "string"
  default = "3"
}

variable "amount_private_ecs_subnets" {
  type    = "string"
  default = "3"
}

variable "amount_private_lb_subnets" {
  type    = "string"
  default = "3"
}

variable "newbits" {
  description = "see https://www.terraform.io/docs/configuration/interpolation.html#cidrsubnet_iprange_newbits_netnum_"
  type        = "string"
  default     = 8
}

variable "netnum" {
  description = "first number of subnet to start of (ex 10.1,10.2,10.3 subnet I specify 1) https://www.terraform.io/docs/configuration/interpolation.html#cidrsubnet_iprange_newbits_netnum_"
  type        = "string"
  default     = 0
}

/*
 cidr 
*/

variable "cidr" {
  description = "cidr for vpc"
  type        = "string"
  default     = "10.20.100.0/24"
}

variable "cidr_pub" {
  description = "cidr for public subnet range"
  type        = "string"
  default     = "10.20.100.0/19"
}

variable "cidr_prv" {
  description = "cidr for private subnet range"
  type        = "string"
  default     = "10.20.100.0/20"
}

variable "vpc_cidr" {
  description = "cidr for vpc"
  type        = "list"
  default     = ["10.20.100.0/24"]
}

Debug Output

Crash Output

Expected Behavior


subnet

public subnets starts with range "10.20.100.0/27"
private subnets starts with range ""10.20.100.0/27""

cidr should calculate
10.20.100.0/27
10.20.100.32/27
10.20.100.64/27
10.20.100.96/28
10.20.100.112/28
10.20.100.128/28
10.20.100.144/28
10.20.100.160/28
10.20.100.176/28
10.20.100.192/28
10.20.100.208/28
10.20.100.224/28
 

Actual Behavior


after running $ terraform plan
third decimal place number is not remaining static it increments as shown:

subnet ranges

10.20.96.0/27
10.20.100.0/27
10.20.104.0/27
10.20.102.0/28
10.20.103.0/28
10.20.104.0/28
10.20.105.0/28
10.20.106.0/28
10.20.107.0/28
10.20.108.0/28
10.20.109.0/28
10.20.110.0/28

Steps to Reproduce


$ terraform plan

Important Factoids

References

config question
Source
picture tonygyerr

Most helpful comment

Hi @tonygyerr,

Let's consider your "private DB" example here:

  cidr_block              = "${cidrsubnet(var.cidr_prv, var.newbits, (var.netnum_private_db + (16 * count.index)))}"

Substituting variables when count.index is 2, we get:

  cidr_block = "${cidrsubnet("10.20.100.0/20", 8, (144 + (16 * 2)))}"

That base prefix 10.20.100.0/20 is the following in binary:

00001010 . 00010100 . 01100100 . 00000000
NNNNNNNN . NNNNNNNN . NNNNHHHH . HHHHHHHH
   10    .    20    .   100    .   0

Note that this is already problematic because the prefix sets bit 22, but for a valid CIDR range specification all of the bits after the prefix should be zero. So Terraform is actually interpreting this as 10.20.96.0/20, which is the following in binary:

00001010 . 00010100 . 01100000 . 00000000
NNNNNNNN . NNNNNNNN . NNNNHHHH . HHHHHHHH
   10    .    20    .    96    .   0

Since newbits is set to 8, this asks Terraform to fill in an additonal 8 bits with the number given in the final argument. (144 + (16 * 2))) is 176, which is 10110000 in binary, so these additional bits get written into the digits indicated with S in the following binary representation:

00001010 . 00010100 . 01101011 . 00000000
NNNNNNNN . NNNNNNNN . NNNNSSSS . SSSSHHHH
   10    .    20    .   107    .   0

Because the first four bits of the new portion are in the third octet, that value becomes 107 in decimal, giving a result of 10.20.107.0/28 (since there are 20 "N" bits and 8 "S" bits, giving a total prefix length of 28).

So 10.20.107.0/28 is the _expected_ result for these arguments. If you want the third octet to remain constant for all of your assigned subnets then you must start with the prefix 10.20.100.0/24, which is the following in binary:

00001010 . 00010100 . 01100100 . 00000000
NNNNNNNN . NNNNNNNN . NNNNNNNN . HHHHHHHH
   10    .    20    .   100    .   0

Since the "N" bits now fill the whole of the third octet, the new portion added by cidrsubnet will not affect those bits. However, this involves reducing newbits because adding an additional 8 bits here would use up all of the remaining host ("H") bits.

To make the behavior easier to follow here, I'd recommend to build the addresses in two steps. First, calculate a /24 prefix for each of your public ECS, private ECS, database, load balancer ranges:

variable "netnum_public_ecs" {
  default = 1
}

variable "netnum_private_ecs" {
  default = 2
}

variable "netnum_private_db" {
  default = 3
}

variable "netnum_private_lb" {
  default = 4
}

locals {
  base_cidr              = "${aws_vpc.aee-ece-api-east-vpc.cidr_block}"
  cidr_block_public_ecs  = "${cidrsubnet(var.base_cidr, 4, var.netnum_public_ecs)}"
  cidr_block_private_ecs = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_ecs)}"
  netnum_private_db      = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_db)}"
  netnum_private_lb      = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_lb)}"
}

resource "aws_subnet" "ecs_subnet_pub" {
  count                   = "${var.amount_public_ecs_subnets}"
  cidr_block              = "${cidrsubnet(local.cidr_block_public_ecs, 4, count.index)}"
  vpc_id                  = "${aws_vpc.aee-ece-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = true
  depends_on              = ["aws_internet_gateway.aee-dev-east-ig"]

  tags {
    Name = "${var.tags["Name"]}-ecs-pub-${count.index}-subnet"
  }
}

# etc...

With the above:

  • local.base_cidr is the CIDR block from you VPC, which should have a /20 prefix e.g. 10.20.96.0/20
  • local.cidr_block_public_ecs extends that to a 24-bit prefix with the subnet number 1, giving 10.20.97.0/20
  • The cidr_block expression in aws_subnet.ecs_subnet_pub extends this to a 28-bit prefix with the subnet number count.index, so the generated prefixes for the different count.index values will be 10.20.97.0/28, 10.20.97.16/28, 10.20.97.32/28, , 10.20.97.33/28, ...

By splitting the calculation into two stages you can let the cidrsubnet function do the work of making sure the numbers fit into the number of bits you want, rather than you having to do complex expressions like var.netnum_public_ecs + (32 * count.index) to try to get things fit into the right places.

Please note that we use GitHub issues for tracking bugs and enhancements rather than for questions. While we may be able to help with certain simple problems here, it's better to use one of the community forums where there are far more people ready to help, whereas the GitHub issues here are generally monitored only by our few core maintainers. If you have follow-up questions, please ask them in one of the forums linked on that page.

picture apparentlymart on 6 Dec 2017
👍2 ❤1

All 6 comments

Hi @tonygyerr,

With the long config and the censorship of the IP addresses I'm not totally sure I follow correctly here, but I think the behavior you are seeing looks right to me for the config you've written.

Your two subnet cidr_blocks have /19 and /20 for their prefix length, which only covers part of the third octet of the IP address, in binary.

For example, with prefix length /20 (N = network, H = host):

NNNNNNNN . NNNNNNNN . NNNNHHHH . HHHHHHHH

You didn't include the values of var.newbits and var.netnum_private_db, but since your results have prefix /28 I assume var.newbits is 8, which reassigns the binary digits like this (S = subnet, the new bits added):

NNNNNNNN . NNNNNNNN . NNNNSSSS . SSSSHHHH

So your new subnet number covers the lower four bits of the third octet and the higher four bits of the forth octet. The third octet will therefore only stay constant if var.netnum_private_db, var.netnum_private_ecs and var.netnum_private_lb are all less than 128 and the amount of subnets in each group is small enough that + count.index doesn't create a number greater than 127, since otherwise some of the bits in the third octet must necessarily change to represent part of the calculated network number.

If you want the third octet to remain constant, I'd suggest starting with a prefix length of /24 and then setting "newbits" to 4, so your final prefix length is still 24 + 4 = 28 bits. This will ensure that the subnet calculation only affects the final eight bits, and thus the final octet in the address.

apparentlymart picture apparentlymart on 5 Dec 2017

@apparentlymart thank you for the help. I was able to get the last cidr block to read /27 and /28 by changing the size of the newbits. However I am still having issues with the numbers incrementing properly. I have updated this issue with the terraform configuration for the subnet, data, and variables.

tonygyerr picture tonygyerr on 6 Dec 2017

Hi @tonygyerr,

Let's consider your "private DB" example here:

  cidr_block              = "${cidrsubnet(var.cidr_prv, var.newbits, (var.netnum_private_db + (16 * count.index)))}"

Substituting variables when count.index is 2, we get:

  cidr_block = "${cidrsubnet("10.20.100.0/20", 8, (144 + (16 * 2)))}"

That base prefix 10.20.100.0/20 is the following in binary:

00001010 . 00010100 . 01100100 . 00000000
NNNNNNNN . NNNNNNNN . NNNNHHHH . HHHHHHHH
   10    .    20    .   100    .   0

Note that this is already problematic because the prefix sets bit 22, but for a valid CIDR range specification all of the bits after the prefix should be zero. So Terraform is actually interpreting this as 10.20.96.0/20, which is the following in binary:

00001010 . 00010100 . 01100000 . 00000000
NNNNNNNN . NNNNNNNN . NNNNHHHH . HHHHHHHH
   10    .    20    .    96    .   0

Since newbits is set to 8, this asks Terraform to fill in an additonal 8 bits with the number given in the final argument. (144 + (16 * 2))) is 176, which is 10110000 in binary, so these additional bits get written into the digits indicated with S in the following binary representation:

00001010 . 00010100 . 01101011 . 00000000
NNNNNNNN . NNNNNNNN . NNNNSSSS . SSSSHHHH
   10    .    20    .   107    .   0

Because the first four bits of the new portion are in the third octet, that value becomes 107 in decimal, giving a result of 10.20.107.0/28 (since there are 20 "N" bits and 8 "S" bits, giving a total prefix length of 28).

So 10.20.107.0/28 is the _expected_ result for these arguments. If you want the third octet to remain constant for all of your assigned subnets then you must start with the prefix 10.20.100.0/24, which is the following in binary:

00001010 . 00010100 . 01100100 . 00000000
NNNNNNNN . NNNNNNNN . NNNNNNNN . HHHHHHHH
   10    .    20    .   100    .   0

Since the "N" bits now fill the whole of the third octet, the new portion added by cidrsubnet will not affect those bits. However, this involves reducing newbits because adding an additional 8 bits here would use up all of the remaining host ("H") bits.

To make the behavior easier to follow here, I'd recommend to build the addresses in two steps. First, calculate a /24 prefix for each of your public ECS, private ECS, database, load balancer ranges:

variable "netnum_public_ecs" {
  default = 1
}

variable "netnum_private_ecs" {
  default = 2
}

variable "netnum_private_db" {
  default = 3
}

variable "netnum_private_lb" {
  default = 4
}

locals {
  base_cidr              = "${aws_vpc.aee-ece-api-east-vpc.cidr_block}"
  cidr_block_public_ecs  = "${cidrsubnet(var.base_cidr, 4, var.netnum_public_ecs)}"
  cidr_block_private_ecs = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_ecs)}"
  netnum_private_db      = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_db)}"
  netnum_private_lb      = "${cidrsubnet(var.base_cidr, 4, var.netnum_private_lb)}"
}

resource "aws_subnet" "ecs_subnet_pub" {
  count                   = "${var.amount_public_ecs_subnets}"
  cidr_block              = "${cidrsubnet(local.cidr_block_public_ecs, 4, count.index)}"
  vpc_id                  = "${aws_vpc.aee-ece-api-east-vpc.id}"
  availability_zone       = "${data.aws_availability_zones.available.names[count.index]}"
  map_public_ip_on_launch = true
  depends_on              = ["aws_internet_gateway.aee-dev-east-ig"]

  tags {
    Name = "${var.tags["Name"]}-ecs-pub-${count.index}-subnet"
  }
}

# etc...

With the above:

  • local.base_cidr is the CIDR block from you VPC, which should have a /20 prefix e.g. 10.20.96.0/20
  • local.cidr_block_public_ecs extends that to a 24-bit prefix with the subnet number 1, giving 10.20.97.0/20
  • The cidr_block expression in aws_subnet.ecs_subnet_pub extends this to a 28-bit prefix with the subnet number count.index, so the generated prefixes for the different count.index values will be 10.20.97.0/28, 10.20.97.16/28, 10.20.97.32/28, , 10.20.97.33/28, ...

By splitting the calculation into two stages you can let the cidrsubnet function do the work of making sure the numbers fit into the number of bits you want, rather than you having to do complex expressions like var.netnum_public_ecs + (32 * count.index) to try to get things fit into the right places.

Please note that we use GitHub issues for tracking bugs and enhancements rather than for questions. While we may be able to help with certain simple problems here, it's better to use one of the community forums where there are far more people ready to help, whereas the GitHub issues here are generally monitored only by our few core maintainers. If you have follow-up questions, please ask them in one of the forums linked on that page.

apparentlymart picture apparentlymart on 6 Dec 2017
👍2 ❤1

@apparentlymart thank you so much for the help and showing me how to streamline my approach. I greatly appreciate you putting the time and the detailed steps.

  • Tony Gyepi-Garbrah
tonygyerr picture tonygyerr on 8 Dec 2017

@apparentlymart i think there is someting that I am missing. I am trying to calculate the cidrsubnet with a useable range of 0-224 to produce and only increment the 4th octet:
The following is what I would like to calcuate. I've changed 100-> 138 in the range below. Please advise.

10.20.138.0/27
10.20.138.32/27
10.20.138.64/27
10.20.138.96/28
10.20.138.112/28
10.20.138.128/28
10.20.138.144/28
10.20.138.160/28
10.20.138.176/28
10.20.138.192/28
10.20.138.208/28

10.20.138.224/28

locals {
base_cidr = "10.20.138.0/24"
cidr_block_public_ecs = "${cidrsubnet(var.base_cidr, 4, 3)}"
cidr_block_private_ecs = "${cidrsubnet(var.base_cidr, 4, 3)}"
cidr_block_private_db = "${cidrsubnet(var.base_cidr, 4, 3)}"
cidr_block_private_lb = "${cidrsubnet(var.base_cidr, 4, 3)}"
}

/*
ecs subnet public
*/
resource "aws_subnet" "ecs_subnet_pub" {
count = "${var.amount_public_ecs_subnets}"
cidr_block = "${cidrsubnet(local.cidr_block_public_ecs, 3, count.index)}"
vpc_id = "${aws_vpc.ece-api-east-vpc.id}"
availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
map_public_ip_on_launch = true
depends_on = ["aws_internet_gateway.east-ig"]

tags {
Name = "${var.tags["Name"]}-ecs-pub-${count.index}-subnet"
}
}

/*
ecs subnet private
*/
resource "aws_subnet" "ecs_subnet_prv" {
count = "${var.amount_private_ecs_subnets}"
cidr_block = "${cidrsubnet(local.cidr_block_private_ecs, 4, count.index)}"
vpc_id = "${aws_vpc.api-east-vpc.id}"
availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
map_public_ip_on_launch = false

tags {
Name = "${var.tags["Name"]}-ecs-prv-${count.index}-subnet"
}
}

/*
database subnet private
*/
resource "aws_subnet" "db_subnet_prv" {
count = "${var.amount_private_db_subnets}"
cidr_block = "${cidrsubnet(local.cidr_block_private_db, 4, count.index)}"
vpc_id = "${aws_vpc.api-east-vpc.id}"
availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
map_public_ip_on_launch = false

tags {
Name = "${var.tags["Name"]}-db-prv-${count.index}-subnet"
}
}

/*
load balancer subnet private
*/
resource "aws_subnet" "lb_subnet_prv" {
count = "${var.amount_private_lb_subnets}"
cidr_block = "${cidrsubnet(local.cidr_block_private_lb, 4, count.index)}"
vpc_id = "${aws_vpc.api-east-vpc.id}"
availability_zone = "${data.aws_availability_zones.available.names[count.index]}"
map_public_ip_on_launch = false

tags {
Name = "${var.tags["Name"]}-alb-prv-${count.index}-subnet"
}
}

tonygyerr picture tonygyerr on 11 Dec 2017
👎1

I'm going to lock this issue because it has been closed for _30 days_ ⏳. This helps our maintainers find and focus on the active issues.

If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.

hashibot[bot] picture hashibot[bot] on 5 Apr 2020
Was this page helpful?
0 / 5 - 0 ratings

Related issues

allomov picture allomov  ·  3Comments
ronnix picture ronnix  ·  3Comments
ketzacoatl picture ketzacoatl  ·  3Comments
carl-youngblood picture carl-youngblood  ·  3Comments
rkulagowski picture rkulagowski  ·  3Comments