initial commit

Author: D.Kang

.dockerignore

@@ -0,0 +1,11 @@
+*.swp
+*.bak
+.git
+.gitignore
+.dockerignore
+dist
+build
+test
+*.egg-info
+__pycache__
+.idea

.gitignore

@@ -0,0 +1,14 @@
+*.swp
+*.bak
+disk
+build
+*.egg-info
+*.egg
+*.whl
+*.log
+.idea
+__pycache__
+venv
+local-conf.yml
+test/reports
+test/_trial_temp

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3
+
+ENV PYTHONUNBUFFERED 1
+ENV SPACEONE_PORT 50051
+ENV SERVER_TYPE grpc
+ENV PKG_DIR /tmp/pkg
+ENV SRC_DIR /tmp/src
+
+COPY src ${SRC_DIR}
+ARG CACHEBUST=1
+WORKDIR ${SRC_DIR}
+RUN python3 setup.py install && \
+    rm -rf /tmp/*
+
+EXPOSE ${SPACEONE_PORT}
+
+ENTRYPOINT ["spaceone"]
+CMD ["grpc", "monitoring"]

bin/build.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+# Build a docker image
+cd ..
+docker build -t pyengine/aws-cloudwatch . --no-cache
+docker tag pyengine/aws-cloudwatch pyengine/google-cloud-stackdriver:1.0
+docker tag pyengine/aws-cloudwatch spaceone/google-cloud-stackdriver:1.0

bin/push.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# How to upload
+./build.sh
+docker push pyengine/google-cloud-stackdriver:1.0
+docker push spaceone/google-cloud-stackdriver:1.0

pkg/pip_requirements.txt

@@ -0,0 +1,6 @@
+spaceone-core
+spaceone-api
+spaceone-tester
+google-auth
+google-api-python-client
+schematics

src/VERSION

@@ -0,0 +1 @@
+1.0.0

src/setup.py

@@ -0,0 +1,42 @@
+#
+#   Copyright 2020 The SpaceONE Authors.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+from setuptools import setup, find_packages
+
+with open('VERSION', 'r') as f: 
+    VERSION = f.read().strip()
+    f.close()
+
+setup(
+    name='plugin-googlecloud-stackdriver',
+    version=VERSION,
+    description='Google Cloud Stackdriver monitoring plugin',
+    long_description='',
+    url='https://www.spaceone.dev/',
+    author='MEGAZONE SpaceONE Team',
+    author_email='[email protected]',
+    license='Apache License 2.0',
+    packages=find_packages(),
+    install_requires=[
+        'spaceone-core',
+        'spaceone-api',
+        'spaceone-tester',
+        'google-auth',
+        'google-api-python-client',
+        'schematics'
+    ],
+    zip_safe=False,
+)

src/spaceone/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

src/spaceone/monitoring/__init__.py

@@ -0,0 +1 @@
+name = 'monitoring'

src/spaceone/monitoring/api/__init__.py

@@ -0,0 +1,16 @@
+from spaceone.api.monitoring.plugin import data_source_pb2, data_source_pb2_grpc
+from spaceone.core.pygrpc import BaseAPI
+
+
+class DataSource(BaseAPI, data_source_pb2_grpc.DataSourceServicer):
+
+    pb2 = data_source_pb2
+    pb2_grpc = data_source_pb2_grpc
+
+    def verify(self, request, context):
+        params, metadata = self.parse_request(request, context)
+
+        with self.locator.get_service('DataSourceService', metadata) as data_source_service:
+            response_stream = data_source_service.verify(params)
+            for response in response_stream:
+                yield self.locator.get_info('PluginVerifyResponse', response)

src/spaceone/monitoring/api/plugin/__init__.py

@@ -0,0 +1,8 @@
+from spaceone.api.monitoring.plugin import log_pb2, log_pb2_grpc
+from spaceone.core.pygrpc import BaseAPI
+
+
+class Log(BaseAPI, log_pb2_grpc.LogServicer):
+
+    pb2 = log_pb2
+    pb2_grpc = log_pb2_grpc

src/spaceone/monitoring/api/plugin/data_source.py

@@ -0,0 +1,24 @@
+from spaceone.api.monitoring.plugin import metric_pb2, metric_pb2_grpc
+from spaceone.core.pygrpc import BaseAPI
+
+
+class Metric(BaseAPI, metric_pb2_grpc.MetricServicer):
+
+    pb2 = metric_pb2
+    pb2_grpc = metric_pb2_grpc
+
+    def list(self, request, context):
+        params, metadata = self.parse_request(request, context)
+
+        with self.locator.get_service('MetricService', metadata) as metric_service:
+            response_stream = metric_service.list(params)
+            for response in response_stream:
+                yield self.locator.get_info('PluginMetricsResponse', response)
+
+    def get_data(self, request, context):
+        params, metadata = self.parse_request(request, context)
+
+        with self.locator.get_service('MetricService', metadata) as metric_service:
+            response_stream = metric_service.get_data(params)
+            for response in response_stream:
+                yield self.locator.get_info('PluginMetricDataResponse', response)

src/spaceone/monitoring/api/plugin/log.py

@@ -0,0 +1,3 @@
+CONNECTORS = {
+    'AWSBotoConnector': {}
+}

src/spaceone/monitoring/api/plugin/metric.py

@@ -0,0 +1,5 @@
+PROTO = {
+    'spaceone.monitoring.api.plugin.data_source': ['DataSource'],
+    'spaceone.monitoring.api.plugin.metric': ['Metric'],
+    'spaceone.monitoring.api.plugin.log': ['Log'],
+}

src/spaceone/monitoring/conf/__init__.py

@@ -0,0 +1 @@
+from spaceone.monitoring.connector.aws_boto_connector import AWSBotoConnector

src/spaceone/monitoring/conf/global_conf.py

@@ -0,0 +1,69 @@
+import logging
+import boto3
+
+from spaceone.core import utils
+from spaceone.core.connector import BaseConnector
+from spaceone.monitoring.error import *
+from spaceone.monitoring.connector.aws_boto_connector.cloud_watch import CloudWatch
+
+__all__ = ['AWSBotoConnector']
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class AWSBotoConnector(BaseConnector):
+
+    def __init__(self, transaction, config):
+        super().__init__(transaction, config)
+
+    def create_session(self, options: dict, secret_data: dict):
+        self._check_secret_data(secret_data)
+
+        aws_access_key_id = secret_data['aws_access_key_id']
+        aws_secret_access_key = secret_data['aws_secret_access_key']
+        region_name = secret_data.get('region_name')
+        role_arn = secret_data.get('role_arn')
+
+        try:
+            if role_arn:
+                self._create_session_with_assume_role(aws_access_key_id, aws_secret_access_key, region_name, role_arn)
+            else:
+                self._create_session_with_access_key(aws_access_key_id, aws_secret_access_key, region_name)
+        except Exception as e:
+            raise ERROR_INVALID_CREDENTIALS()
+
+    @staticmethod
+    def _check_secret_data(secret_data):
+        if 'aws_access_key_id' not in secret_data:
+            raise ERROR_REQUIRED_PARAMETER(key='secret.aws_access_key_id')
+
+        if 'aws_secret_access_key' not in secret_data:
+            raise ERROR_REQUIRED_PARAMETER(key='secret.aws_secret_access_key')
+
+    def _create_session_with_access_key(self, aws_access_key_id, aws_secret_access_key, region_name):
+        self.session = boto3.Session(aws_access_key_id=aws_access_key_id,
+                                     aws_secret_access_key=aws_secret_access_key,
+                                     region_name=region_name)
+
+        sts = self.session.client('sts')
+        sts.get_caller_identity()
+
+    def _create_session_with_assume_role(self, aws_access_key_id, aws_secret_access_key, region_name, role_arn):
+        self._create_session_with_access_key(aws_access_key_id, aws_secret_access_key, region_name)
+
+        sts = self.session.client('sts')
+        assume_role_object = sts.assume_role(RoleArn=role_arn, RoleSessionName=utils.generate_id('AssumeRoleSession'))
+        credentials = assume_role_object['Credentials']
+
+        self.session = boto3.Session(aws_access_key_id=credentials['AccessKeyId'],
+                                     aws_secret_access_key=credentials['SecretAccessKey'],
+                                     region_name=region_name,
+                                     aws_session_token=credentials['SessionToken'])
+
+    def list_metrics(self, *args, **kwargs):
+        cw = CloudWatch(self.session)
+        return cw.list_metrics(*args, **kwargs)
+
+    def get_metric_data(self, *args, **kwargs):
+        cw = CloudWatch(self.session)
+        return cw.get_metric_data(*args, **kwargs)

src/spaceone/monitoring/conf/proto_conf.py

@@ -0,0 +1,121 @@
+import logging
+import time
+from datetime import datetime, timedelta
+
+from spaceone.core import utils
+from spaceone.monitoring.error import *
+
+__all__ = ['CloudWatch']
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class CloudWatch(object):
+
+    def __init__(self, session):
+        self.session = session
+        self.client = self.session.client('cloudwatch')
+
+    def list_metrics(self, namespace, dimensions):
+        paginator = self.client.get_paginator('list_metrics')
+
+        responses = paginator.paginate(Namespace=namespace, Dimensions=dimensions)
+
+        metrics_info = []
+
+        for response in responses:
+            for metric in response['Metrics']:
+                metric_name = metric['MetricName']
+                unit = self._get_metric_unit(namespace, dimensions, metric_name)
+                chart_type, chart_option = self._get_chart_info(namespace, dimensions, metric_name)
+
+                metric_info = {
+                    'key': metric_name,
+                    'name': metric_name,
+                    'unit': unit,
+                    'chart_type': chart_type,
+                    'chart_options': chart_option
+                }
+
+                metrics_info.append(metric_info)
+
+        return {
+            'metrics': metrics_info
+        }
+
+    def get_metric_data(self, namespace, dimensions, metric_name, start, end, period, stat, limit=None):
+        metric_id = f'metric_{utils.random_string()[:12]}'
+
+        extra_opts = {}
+
+        if limit:
+            extra_opts['MaxDatapoints'] = limit
+
+        response = self.client.get_metric_data(
+            MetricDataQueries=[{
+                'Id': metric_id,
+                'MetricStat': {
+                    'Metric': {
+                        'Namespace': namespace,
+                        'MetricName': metric_name,
+                        'Dimensions': dimensions
+                    },
+                    'Period': period,
+                    'Stat': stat
+                }
+            }],
+            StartTime=start,
+            EndTime=end,
+            ScanBy='TimestampAscending',
+            **extra_opts
+        )
+
+        metric_data_info = {
+            'labels': [],
+            'values': []
+        }
+
+        for metric_data in response.get('MetricDataResults', []):
+            metric_data_info['labels'] = list(map(self._convert_timestamp, metric_data['Timestamps']))
+            metric_data_info['values'] += metric_data['Values']
+
+        return metric_data_info
+
+    @staticmethod
+    def _convert_timestamp(metric_datetime):
+        timestamp = int(time.mktime(metric_datetime.timetuple()))
+        return {
+            'seconds': timestamp
+        }
+
+    def _get_metric_unit(self, namespace, dimensions, metric_name):
+        end = datetime.utcnow()
+        start = end - timedelta(minutes=60)
+
+        response = self.client.get_metric_statistics(
+            Namespace=namespace,
+            Dimensions=dimensions,
+            MetricName=metric_name,
+            StartTime=start,
+            EndTime=end,
+            Period=600,
+            Statistics=['SampleCount']
+        )
+
+        return_dict = {
+            'x': 'Timestamp',
+            'y': ''
+        }
+
+        for data_point in response.get('Datapoints', []):
+            unit = data_point['Unit']
+
+            if unit != 'None':
+                return_dict['y'] = unit
+                return return_dict
+
+        return return_dict
+
+    @staticmethod
+    def _get_chart_info(namespace, dimensions, metric_name):
+        return 'line', {}

src/spaceone/monitoring/connector/__init__.py

@@ -0,0 +1 @@
+from spaceone.monitoring.error.google_cloud import *

src/spaceone/monitoring/connector/google_cloud_api_connector/__init__.py

@@ -0,0 +1,16 @@
+from spaceone.core.error import *
+
+class ERROR_INVALID_CREDENTIALS(ERROR_INVALID_ARGUMENT):
+    _message = 'Google Cloud credentials is invalid.'
+
+
+class ERROR_INVALID_RESOURCE_FORMAT(ERROR_INVALID_ARGUMENT):
+    _message = 'Resource format is invalid. (format = ARN).'
+
+
+class ERROR_NOT_SUPPORT_RESOURCE(ERROR_INVALID_ARGUMENT):
+    _message = 'This Resource is not supported by Google Cloud Stackdriver. (resource = {resource})'
+
+
+class ERROR_NOT_SUPPORT_STAT(ERROR_INVALID_ARGUMENT):
+    _message = 'Statistics option is invalid. (supported_stat = {supported_stat})'

src/spaceone/monitoring/connector/google_cloud_api_connector/stackdriver.py

@@ -0,0 +1,2 @@
+from spaceone.monitoring.info.metric_info import *
+from spaceone.monitoring.info.data_source_info import *