parametize connection parameters, reduce VPC stack needs, add app node

Author: jphaugla

cfns3deploy.sh

@@ -1 +1 @@
-aws cloudformation deploy --template-file templates/createS3.yaml --stack-name ${PROJECT_NAME}-s3 --parameter-overrides S3BucketName=${S3_BUCKET} 
+aws cloudformation deploy --template-file templates/createS3.yaml --stack-name ${PROJECT_NAME}-s3 --parameter-overrides S3BucketName=${S3_BUCKET} KeyPair=${KEY_PAIR}

src/main/java/com/jphaugla/redis/streams/RedisStreams101Consumer.java

@@ -14,12 +14,10 @@ public class RedisStreams101Consumer {
     public final static String MESSAGE_KEY = "weather_sensor:wind:message:";
 
     public static void main(String[] args) {
-        String portNumber = "12000";
-        if (args != null && args.length != 0 ) {
-            portNumber = args[0];
-        }
-        System.out.println("port number is " + portNumber);
-        RedisClient redisClient = RedisClient.create("redis://localhost:" + portNumber); // change to reflect your environment
+
+        String REDIS_URI=System.getenv("REDIS_URI");
+        System.out.println("Redis URI=" + REDIS_URI);
+        RedisClient redisClient = RedisClient.create(REDIS_URI);
         StatefulRedisConnection<String, String> connection = redisClient.connect();
         RedisCommands<String, String> syncCommands = connection.sync();
 

src/main/java/com/jphaugla/redis/streams/RedisStreams101ConsumerLUA.java

@@ -22,12 +22,10 @@ public class RedisStreams101ConsumerLUA {
     public final static String APPLICATION_NAME = "application_1";
 
     public static void main(String[] args) throws JsonProcessingException {
-        String portNumber = "12000";
-        if (args != null && args.length != 0 ) {
-            portNumber = args[0];
-        }
-        System.out.println("port number is " + portNumber);
-        RedisClient redisClient = RedisClient.create("redis://localhost:" + portNumber); // change to reflect your environment
+
+        String REDIS_URI=System.getenv("REDIS_URI");
+        System.out.println("Redis URI=" + REDIS_URI);
+        RedisClient redisClient = RedisClient.create(REDIS_URI);
         StatefulRedisConnection<String, String> connection = redisClient.connect();
         RedisCommands<String, String> syncCommands = connection.sync();
         InputStream luaInputStream =

src/main/java/com/jphaugla/redis/streams/RedisStreams101ProducerSingle.java

@@ -22,9 +22,10 @@ public static void main(String[] args) {
         }
 
         System.out.printf("\n Sending %s message(s)%n", nbOfMessageToSend);
+        String REDIS_URI=System.getenv("REDIS_URI");
+        System.out.println("Redis URI=" + REDIS_URI);
+        RedisClient redisClient = RedisClient.create(REDIS_URI);
 
-
-        RedisClient redisClient = RedisClient.create("redis://localhost:6379"); // change to reflect your environment
         StatefulRedisConnection<String, String> connection = redisClient.connect();
         RedisCommands<String, String> syncCommands = connection.sync();
 

templates/MasterWithRedis.yaml

@@ -3,37 +3,39 @@ Transform: 'AWS::Serverless-2016-10-31'
 Description: Deploy Redis Instances in a region
 
 Mappings:
+
   RegionMap:
     us-east-1: # Virginia
-      "EC2AMI" : "ami-0d894ea5521c64557"
+      "REDISEC2AMI" : "ami-0d894ea5521c64557"
+      "EC2AMI" : "ami-08e4e35cccc6189f4"
     us-east-2: # Ohio
-      "EC2AMI" : "ami-082a25bc38620fc4d"
+      "REDISEC2AMI" : "ami-082a25bc38620fc4d"
     us-east-3: # NCALIF
-      "EC2AMI" : "ami-01773f954c3875420"
+      "REDISEC2AMI" : "ami-01773f954c3875420"
     us-west-2: # Oregon
-      "EC2AMI" : "ami-05037ab6f618ed336"
+      "REDISEC2AMI" : "ami-05037ab6f618ed336"
     ap-south-1: # Mumbai
-      "EC2AMI" : "ami-0eefc21aa52001198"
+      "REDISEC2AMI" : "ami-0eefc21aa52001198"
     ap-northeast-2: # Seoul
-      "EC2AMI" : "ami-0967b7ff9a7ba331e"
+      "REDISEC2AMI" : "ami-0967b7ff9a7ba331e"
     ap-southeast-1: # Singapore
-      "EC2AMI" : "ami-052d3e5a59fc4a877"
+      "REDISEC2AMI" : "ami-052d3e5a59fc4a877"
     ap-southeast-2: # Sydney 
-      "EC2AMI" : "ami-097cbef0451a30da5"
+      "REDISEC2AMI" : "ami-097cbef0451a30da5"
     ap-northeast-1: # Tokyo
-      "EC2AMI" : "ami-0ce04ab8bcd5de2e9"
+      "REDISEC2AMI" : "ami-0ce04ab8bcd5de2e9"
     eu-central-1: # Frankfurt
-      "EC2AMI" : "ami-09304ee2fe70b3aad"
+      "REDISEC2AMI" : "ami-09304ee2fe70b3aad"
     eu-west-1: # Ireland
-      "EC2AMI" : "ami-03b7776ede7f47475"
+      "REDISEC2AMI" : "ami-03b7776ede7f47475"
     eu-west-2: # London
-      "EC2AMI" : "ami-0c2b997e5c91153e0"
+      "REDISEC2AMI" : "ami-0c2b997e5c91153e0"
     eu-west-3: # Paris  
-      "EC2AMI" : "ami-0176733f200a89b17"
+      "REDISEC2AMI" : "ami-0176733f200a89b17"
     eu-north-1: # Stockholm 
-      "EC2AMI" : "ami-040e6fa2d290e7b9b"
+      "REDISEC2AMI" : "ami-040e6fa2d290e7b9b"
     sa-east-1: # Sao Paulo 
-      "EC2AMI" : "ami-06d197a564493b29b"
+      "REDISEC2AMI" : "ami-06d197a564493b29b"
 
 Parameters:
   RedisDatabasePort: 
@@ -72,7 +74,7 @@ Parameters:
     MinValue: 1
     MaxValue: 3
     Default: 3
-  EC2InstanceType:
+  RedisEC2InstType:
     Description: Amazon EC2 Instance Type  m4 and m5.xlarge 4-16 m5 has better network, 2xlarge 8-13,  r4.xlarge 4 30.5, 8 61
     Type: String
     Default: r4.2xlarge
@@ -100,6 +102,37 @@ Parameters:
       - i3.16xlarge
     ConstraintDescription: Must be a valid EC2 instance type.
 
+  EC2InstType:
+    Description: Amazon EC2 Instance Type for app node 
+    Type: String
+    Default: i3.large
+    AllowedValues:
+      - r4.large
+      - r4.xlarge
+      - r4.2xlarge
+      - r4.4xlarge
+      - r4.8xlarge
+      - r4.16xlarge
+      - m4.large
+      - m4.xlarge
+      - m4.2xlarge
+      - m4.4xlarge
+      - m4.10xlarge
+      - m4.16xlarge
+      - c5a.large
+      - c5a.xlarge
+      - c5a.2xlarge
+      - c5a.4xlarge
+      - c5a.8xlarge
+      - c5a.16xlarge
+      - i3.large
+      - i3.xlarge
+      - i3.2xlarge
+      - i3.4xlarge
+      - i3.8xlarge
+      - i3.16xlarge
+    ConstraintDescription: Must be a valid EC2 instance type.
+
 Conditions:
   Create-One: !Equals [ !Ref NumberInstances, 1 ]
   Create-Two: !Equals [ !Ref NumberInstances, 2 ]
@@ -121,7 +154,7 @@ Resources:
       'AWS::CloudFormation::Designer':
         id: 7aa57bac-380e-4523-a72b-ef898d0a0367
 
-  RedisEC2N1:
+  RedisEC2VM1:
     Type:  AWS::EC2::Instance
     DependsOn: 
       - VPCStack
@@ -130,8 +163,8 @@ Resources:
         Fn::FindInMap:
         - RegionMap
         - !Ref AWS::Region
-        - EC2AMI
-      InstanceType: !Ref EC2InstanceType
+        - REDISEC2AMI
+      InstanceType: !Ref RedisEC2InstType
       SubnetId: !GetAtt VPCStack.Outputs.PublicSubnet1Id
       KeyName: !Ref KeyPair
       SecurityGroupIds:
@@ -160,19 +193,19 @@ Resources:
              tar xvf redislabs-6.2.8-53-rhel7-x86_64.tar
              ./install.sh -y
 
-  RedisEC2N2:
+  RedisEC2VM2:
     Type:  AWS::EC2::Instance
     Condition: Do-Two
     DependsOn: 
       - VPCStack
-      - RedisEC2N1
+      - RedisEC2VM1
     Properties:
       ImageId: 
         Fn::FindInMap:
         - RegionMap
         - !Ref AWS::Region
-        - EC2AMI
-      InstanceType: !Ref EC2InstanceType
+        - REDISEC2AMI
+      InstanceType: !Ref RedisEC2InstType
       SubnetId: !GetAtt VPCStack.Outputs.PublicSubnet1Id
       KeyName: !Ref KeyPair
       SecurityGroupIds:
@@ -194,19 +227,19 @@ Resources:
              tar xvf redislabs-6.2.8-53-rhel7-x86_64.tar
              ./install.sh -y
 
-  RedisEC2N3:
+  RedisEC2VM3:
     Condition: Create-Three
     Type:  AWS::EC2::Instance
     DependsOn: 
       - VPCStack
-      - RedisEC2N1
+      - RedisEC2VM1
     Properties:
       ImageId: 
         Fn::FindInMap:
         - RegionMap
         - !Ref AWS::Region
-        - EC2AMI
-      InstanceType: !Ref EC2InstanceType
+        - REDISEC2AMI
+      InstanceType: !Ref RedisEC2InstType
       SubnetId: !GetAtt VPCStack.Outputs.PublicSubnet1Id
       KeyName: !Ref KeyPair
       SecurityGroupIds:
@@ -228,46 +261,74 @@ Resources:
              tar xvf redislabs-6.2.8-53-rhel7-x86_64.tar
              ./install.sh -y
 
+  AppEC2Instance:
+    Type:  AWS::EC2::Instance
+    DependsOn: 
+      - VPCStack
+    Properties:
+      ImageId: 
+        Fn::FindInMap:
+        - RegionMap
+        - !Ref AWS::Region
+        - EC2AMI
+      InstanceType: !Ref EC2InstType
+      SubnetId: !GetAtt VPCStack.Outputs.PublicSubnet1Id
+      KeyName: !Ref KeyPair
+      SecurityGroupIds:
+        - !GetAtt VPCStack.Outputs.WorkshopSecurityGroup
+      Tags:
+        - Key: 'Name'
+          Value: !Join ['',  [!Ref AWS::StackName, 'APPEC2'] ]
+      UserData:
+        Fn::Base64:
+          !Sub |
+             #!/bin/bash -xe
+             sudo  bash -l
+             yum -y install git wget bind-utils maven java-1.8.0-openjdk
+             cd /home/ec2-user
+             git clone https://github.com/jphaugla/redisSMSLuaA-A.git
+             sudo chown -R ec2-user:ec2-user redisSMSLuaA-A
+
   RedisARecord1:
     Type: AWS::Route53::RecordSet
     DependsOn: 
-      - RedisEC2N1
+      - RedisEC2VM1
     Properties:
       HostedZoneId: !Ref HostedZoneId
       Comment: DNS name for my instance
       Name: !Join ['',  ['node1.', !Ref ClusterName, ., !Ref HostedZoneName ] ]  
       Type: A
       TTL: 900
       ResourceRecords:
-        - !GetAtt RedisEC2N1.PublicIp
+        - !GetAtt RedisEC2VM1.PublicIp
 
   RedisARecord2:
     Condition: Do-Two
     Type: AWS::Route53::RecordSet
     DependsOn: 
-      - RedisEC2N2
+      - RedisEC2VM2
     Properties:
       HostedZoneId: !Ref HostedZoneId
       Comment: DNS name for my instance
       Name: !Join ['',  ['node2.', !Ref ClusterName, ., !Ref HostedZoneName ] ]  
       Type: A
       TTL: 900
       ResourceRecords:
-        - !GetAtt RedisEC2N2.PublicIp
+        - !GetAtt RedisEC2VM2.PublicIp
 
   RedisARecord3:
     Condition: Create-Three
     Type: AWS::Route53::RecordSet
     DependsOn: 
-      - RedisEC2N3
+      - RedisEC2VM3
     Properties:
       HostedZoneId: !Ref HostedZoneId
       Comment: DNS name for my instance
       Name: !Join ['',  ['node3.', !Ref ClusterName, ., !Ref HostedZoneName ] ]  
       Type: A
       TTL: 900
       ResourceRecords:
-        - !GetAtt RedisEC2N3.PublicIp
+        - !GetAtt RedisEC2VM3.PublicIp
 
   RedisNSRecord:
     Type: AWS::Route53::RecordSet
@@ -285,31 +346,27 @@ Outputs:
 
   Subnets:
     Description: Subnets for workshop
-    Value: !Join [',', [!GetAtt VPCStack.Outputs.PublicSubnet1Id,!GetAtt VPCStack.Outputs.PublicSubnet2Id,!GetAtt VPCStack.Outputs.PublicSubnet3Id]]  
+    Value: !Join [',', [!GetAtt VPCStack.Outputs.PublicSubnet1Id]]  
 
   WorkshopSecurityGroup:
     Description: 'Security Group'
     Value: !GetAtt VPCStack.Outputs.WorkshopSecurityGroup
 
-  EMRSecurityGroup:
-    Description: 'Security Group'
-    Value: !GetAtt VPCStack.Outputs.EMRSecurityGroup 
-
   RedisInstancePrivateDNS:
     Description: 'Private DNS for Redis EC2 instance'
-    Value: !GetAtt RedisEC2N1.PrivateDnsName
+    Value: !GetAtt RedisEC2VM1.PrivateDnsName
 
   RedisInstancePublicDNS:
     Description: 'Public DNS for Redis EC2 instance'
-    Value: !GetAtt RedisEC2N1.PublicDnsName
+    Value: !GetAtt RedisEC2VM1.PublicDnsName
 
   RedisInstancePrivateIP:
     Description: 'Private IP for Redis EC2 instance'
-    Value: !GetAtt RedisEC2N1.PrivateIp
+    Value: !GetAtt RedisEC2VM1.PrivateIp
 
   RedisInstancePublicIP:
     Description: 'Public IP for Redis EC2 instance'
-    Value: !GetAtt RedisEC2N1.PublicIp
+    Value: !GetAtt RedisEC2VM1.PublicIp
 
   RedisNSRecord:
     Description: 'The NS Record Name for Redis'