chore: remove sysadmin part2

JIRA:CLOUDSEC-12

devops/jenkins/extend-sandbox-termination.py

@@ -0,0 +1,89 @@
+__author__ = 'arbab'
+'''
+This script will be used to modify/extend the termination date on the sandbox.
+'''
+import boto
+from datetime import datetime
+from datetime import timedelta
+import logging
+import argparse
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description="Modify/extend the termination date on the sandbox.")
+
+    parser.add_argument('-n', '--noop', action='store_true',
+                        help="don't actually run the commands", default=False)
+
+    parser.add_argument('-p', '--profile', default=None,
+                        help="AWS profile to use when connecting.")
+
+    extend_group = parser.add_mutually_exclusive_group(required=True)
+
+    extend_group.add_argument('-d', '--day', default=None,
+                        help="number of days", type=int)
+
+    extend_group.add_argument('-a', '--always', default=False,
+                        help="Do not terminate this Sandbox")
+
+    group = parser.add_mutually_exclusive_group(required=True)
+
+    group.add_argument('-u', '--username', default=None,
+                       help="GitHub username")
+
+    group.add_argument('-c', '--custom', default=None,
+                       help="Custom name, if the sandbox was not created with the default options")
+
+    group.add_argument('-i', '--instance-id', default=None,
+                       help="Sandbox Instance ID")
+
+    args = parser.parse_args()
+
+    ec2 = boto.connect_ec2(profile_name=args.profile)
+
+    days_to_increase = args.day
+
+    if args.username:
+        sandbox_name = args.username + '-sandbox'
+        reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name})
+    if args.custom:
+        sandbox_name = args.custom
+        reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name})
+    if args.instance_id:
+        instance_id = args.instance_id
+        reservations = ec2.get_all_instances(instance_ids=[instance_id])
+
+    instance = reservations[0].instances[0]
+
+    if args.noop:
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} will extend by {} days".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    days_to_increase
+                    )
+                    )
+    elif args.always:
+        instance.add_tag('do_not_terminate', 'true')
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} will not be terminate".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    )
+                    )
+    else:
+        # modified the terminate time
+        terminate_time = datetime.strptime(str(instance.tags['instance_termination_time']), "%m-%d-%Y %H:%M:%S")
+        terminate_time = terminate_time + timedelta(days=days_to_increase)
+        instance.add_tag('instance_termination_time', terminate_time.strftime("%m-%d-%Y %H:%M:%S"))
+        logger.info("Sandbox ID:{} with Name: {} and Owner: {} has been extended by {} days".format(
+                    instance.id,
+                    instance.tags['Name'],
+                    instance.tags['owner'],
+                    days_to_increase
+                    )
+                    )

devops/jenkins/mailchimp/courses.csv

@@ -0,0 +1,8 @@
+05f99ad611,HarvardX/SW12x/2013_SOND,production,edx
+7dfc19a6ad,HarvardX/SW25x/1T2014,production,edx
+fdb57528e6,HarvardX/SW12.2x/1T2014,production,edx
+bc3991d1aa,HarvardX/SW12.3x/1T2014,production,edx
+295787c2eb,HarvardX/USW30x/2T2014,production,edx
+d20f7569e3,HarvardX/SW12.4x/1T2014,production,edx
+0f52d660fa,HarvardX/SW12.5x/2T2014,production,edx
+22a674a898,HarvardX/SW12.6x/2T2014,production,edx

devops/jenkins/mailchimp/sync-courselists.bash

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -x
+
+if [[ -z $WORKSPACE ]]; then
+    echo "Environment incorrect for this wrapper script"
+    env
+    exit 1
+fi
+
+cd "$WORKSPACE/edx-platform"
+
+# install requirements
+# These requirements will be installed into the shinginpanda
+# virtualenv on the jenkins server and are necessary to run
+# management commands locally.
+
+pip install --exists-action w -r requirements/edx/pre.txt
+pip install --exists-action w -r requirements/edx/base.txt
+pip install --exists-action w -r requirements/edx/post.txt
+pip install --exists-action w -r requirements/edx/github.txt
+pip install --exists-action w -r requirements/edx/local.txt
+
+cd "$WORKSPACE/configuration"
+
+pip install --exist-action w -r pre-requirements.txt
+pip install --exist-action w -r requirements.txt
+
+cd "$WORKSPACE/configuration/playbooks"
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${deployment}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${deployment}.yml"
+fi
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${environment}-${deployment}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${environment}-${deployment}.yml"
+fi
+
+for extra_var in $extra_vars; do
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/$extra_var"
+done
+
+extra_var_args+=" -e edxapp_app_dir=${WORKSPACE}"
+extra_var_args+=" -e EDXAPP_CFG_DIR=${WORKSPACE}"
+extra_var_args+=" -e edxapp_code_dir=${WORKSPACE}/edx-platform"
+extra_var_args+=" -e edxapp_user=jenkins"
+
+# Generate the json configuration files
+ansible-playbook -c local $extra_var_args --tags edxapp_cfg -i localhost, -s -U jenkins edxapp.yml
+
+# Run migrations and replace literal '\n' with actual newlines to make the output
+# easier to read
+
+
+EDX_PATH="${WORKSPACE}/edx-platform"
+#DJANGO_ADMIN="${JENKINS_HOME}/.virtualenvs/mailchimp/bin/django-admin.py"
+DJANGO_ADMIN="${VIRTUAL_ENV}/bin/python ${EDX_PATH}/manage.py lms --settings=production"
+
+get_key () {
+  case $1 in
+      "edx"       ) ORG_KEY="7b87ccd203b973d87d0ac4423192afa6-us5";;
+  esac
+  echo $ORG_KEY
+}
+
+
+sync_announcements () {
+    LIST_ID=$1
+    CONFIGURATION=$2
+    ORG_NAME=$3
+    ORG_KEY=$(get_key ${ORG_NAME})
+    CMD="${DJANGO_ADMIN} mailchimp_sync_announcements --key=${ORG_KEY} --list=${LIST_ID}"
+    $CMD
+}
+
+sync_course () {
+    LIST_ID=$1
+    COURSE_ID=$2
+    CONFIGURATION=$3
+    ORG_NAME=$4
+    ORG_KEY=$(get_key ${ORG_NAME})
+    SEGMENTS=${5:-0}
+    CMD="${DJANGO_ADMIN} mailchimp_sync_course --key=${ORG_KEY} --list=${LIST_ID} --course=${COURSE_ID} --segments=${SEGMENTS}"
+    $CMD
+}
+
+OLD_IFS=${IFS}
+
+cd ${WORKSPACE}/sysadmin/jenkins/mailchimp
+
+while read -r line
+do
+    IFS=","
+    set $line
+    IFS=${OLD_IFS}
+     sync_course $1 $2 $3 $4
+done < "courses.csv"
+
+sync_announcements 237694b56d production edx

devops/jenkins/saml-ssl-expiration-check.py

@@ -0,0 +1,40 @@
+import argparse
+import logging
+import OpenSSL
+from datetime import datetime, timedelta
+import sys
+import yaml
+from os.path import basename
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-e', '--region', default='us-east-1', required=True,
+                        help="AWS Region")
+
+    parser.add_argument('-d', '--days', type=int,
+                        help="Alert if SSL certificate will expire within these days", default=90)
+    parser.add_argument('-i','--file',
+                        help="input YAML file to parse and get SAML cert")
+
+
+    args = parser.parse_args()
+
+    time_now = datetime.now()
+    ssl_expire_check = time_now + timedelta(days=args.days)
+    saml_cert_file = args.file
+    expired_ssl =  basename(saml_cert_file).strip('.yml')
+
+    with open(saml_cert_file) as f:
+        secure_config = yaml.safe_load(f)
+    cert = secure_config['EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT']
+    x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+    cert_expire_date = datetime.strptime(x509.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%S%fZ").date()
+
+    if ssl_expire_check.date() > cert_expire_date:
+        logger.info("{} SAML certificate will be expired on  {}".format(expired_ssl,cert_expire_date))
+        sys.exit(1)

devops/jenkins/ssl-expiration-check.py

@@ -0,0 +1,106 @@
+import boto3
+import argparse
+import logging
+import ssl
+import OpenSSL
+import smtplib
+from datetime import date, datetime, timedelta
+from socket import socket
+from pprint import pformat
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def send_an_email(toaddr, fromaddr, expired_ssl_message, not_check_message, region):
+    client = boto3.client('ses', region_name=region)
+
+    message = """
+
+    <p>Hello,</p>
+
+    <p>Certificates that are associated with these load-balancers will be expired in next 30 days: </p>
+
+    <p> {expired_ssl_message} </p>
+
+    <p> These ELBs have SSL certificate but don't have any instance associated to them: </p>
+
+    <p> {not_check_message} </p>
+
+    """.format(expired_ssl_message=expired_ssl_message, not_check_message=not_check_message)
+    client.send_email(
+        Source=fromaddr,
+        Destination={
+            'ToAddresses': [
+                toaddr
+            ]
+        },
+        Message={
+            'Subject': {
+                'Data': 'These Certificates will be expired in the next 30 days',
+                'Charset': 'utf-8'
+            },
+            'Body': {
+                'Html':{
+                    'Data': message,
+                    'Charset': 'utf-8'
+                }
+            }
+        }
+    )
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(
+        description="Find the SSL Certificates that will expire after X days.")
+
+    parser.add_argument('-e', '--region', default='us-east-1', required=True,
+                        help="AWS Region for getting the records", type=str)
+
+    parser.add_argument('-d', '--days', type=int,
+                        help="Alert if SSL certificate will expire within these days", default=30)
+
+    email_args = parser.add_argument_group("Email Arguments",
+                                                "Args for sending email.")
+
+    email_args.add_argument('-r', '--recipient', type=str,
+                        help='Recipient email address')
+
+    email_args.add_argument('-f', '--from-email', type=str,
+                                 help="Sender email address for email notifications. "
+                                      "Email notifications will be disabled if not provided")
+
+    args = parser.parse_args()
+
+    expire_ssl = []
+    time_now = datetime.now()
+    ssl_expire_check = time_now + timedelta(days=args.days)
+
+    elb_conn = boto3.client('elb', region_name=args.region)
+    elbs = elb_conn.describe_load_balancers()['LoadBalancerDescriptions']
+
+    elbs_with_ssl = [elb for elb in elbs for listener in elb['ListenerDescriptions'] if (listener['Listener']['LoadBalancerPort'] == 443)]
+
+    elbs_to_check = [(elb['LoadBalancerName'],elb['DNSName']) for elb in elbs_with_ssl if elb['Instances']]
+
+    elbs_not_need_to_check = [elb['DNSName'] for elb in elbs_with_ssl if not elb['Instances']]
+
+    for elb in elbs_to_check:
+        elb_tags = elb_conn.describe_tags(LoadBalancerNames=[elb[0]])['TagDescriptions'][0]['Tags']
+        for tag in elb_tags:
+          if 'kubernetes.io' in tag["Key"]:
+              break
+        else:
+            print("Checking {}".format(elb[1]))
+            cert = ssl.get_server_certificate((elb[1], 443))
+            x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+            cert_expire_date = datetime.strptime(x509.get_notAfter().decode(), "%Y%m%d%H%M%S%fZ").date()
+            if ssl_expire_check.date() > cert_expire_date:
+                print("Expires {}".format(cert_expire_date))
+                expire_ssl.append((elb[1],cert_expire_date))
+
+    if expire_ssl or elbs_not_need_to_check:
+        expired_ssl_message = pformat(expire_ssl)
+        not_check_message = pformat(elbs_not_need_to_check)
+        print(not_check_message)
+        if args.from_email and args.recipient:
+            send_an_email(args.recipient, args.from_email, expired_ssl_message, not_check_message, args.region)

devops/requirements/base.in

@@ -0,0 +1,10 @@
+# Core requirements for using this application
+-c constraints.txt
+boto
+boto3
+python-gnupg
+pyyaml
+pymysql
+pymongo
+requests
+pyOpenSSL