Merge branch 'merge_work' of /Users/carlosfrias/Google Drive/Client Workspaces/Verizon/OneAPI/vz-workspace/apigee-opdk-playbook-setup-ansible into merge_updates
diff --git a/.gitignore b/.gitignore index 75a0526..8160975 100644 --- a/.gitignore +++ b/.gitignore
@@ -110,3 +110,8 @@ .idea *iml *retry +*.db +*~ +ansible.cfg +backups +
diff --git a/README-CONTROL-SERVER-SYSTEM-PACKAGES.md b/README-CONTROL-SERVER-SYSTEM-PACKAGES.md new file mode 100644 index 0000000..2f24f23 --- /dev/null +++ b/README-CONTROL-SERVER-SYSTEM-PACKAGES.md
@@ -0,0 +1,4 @@ +# Control Server System Packages +Required system packages can be installed with + + ansible-playbook install-system-packages.yml -K
diff --git a/README-CYGWIN.md b/README-CYGWIN.md new file mode 100644 index 0000000..50e9c11 --- /dev/null +++ b/README-CYGWIN.md
@@ -0,0 +1,84 @@ +# Setting Up Ansible with Cygwin + +Running Ansible commands from within Windows is unsupported at the time +of this writing. However, windows users can setup Ansible with Cygwin. + +# Installation Steps +Here are steps to getting Ansible (and it's related commands, like +ansible-playbook) running on Windows: +Download and install [Cygwin](http://cygwin.com/install.html), with at +least the following packages selected (you can select the packages +during the install process): + +* curl +* python (2.7.x) +* python-jinja +* python-crypto +* python-openssl +* python-setuptools +* git +* vim +* openssh +* openssl +* openssl-devel +* wget +* gcc-core +* gcc-g++ +* python2-devel +* python2-cython +* make +* libffi-devel + +# Working Behind a Proxy +If you are working behind a proxy (as is the case in many corporate +networks), edit the .bash_profile used by Cygwin either using vim (open +Cygwin and enter vim .bash_profile), or with whatever editor you'd like, +and add in lines like the following: + + export http_proxy=http://username:password@proxy-address-here:80/ + export https_proxy=https://username:password@proxy-address-here:80/ + +# Download and Install PyYAML and Jinja2 Separately +Download and install separately PyYAML and Jinja2 separately, as they're +not available via Cygwin's installer: + +## Open Cygwin + +#### Download PyYAML + + curl -O https://pypi.python.org/packages/source/P/PyYAML/PyYAML-3.10.tar.gz + +#### Download Jinja2 + + curl -O https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.6.tar.gz + +#### Untar both downloads + + tar -xvf PyYAML-3.10.tar.gz && tar -xvf Jinja2-2.6.tar.gz + +### Change directory into each of the expanded folders and run `python setup.py install` to install each package. + +### Generate an SSH key for use later: `ssh-keygen` then hit enter to skip adding a password until you get back to the command prompt. + +### Clone ansible from its repository on GitHub: + + git clone https://github.com/ansible/ansible /opt/ansible + +### Add the following lines into your Cygwin .bash_profile after the proxy settings you added previously: + + # Ansible Settings + ANSIBLE=/opt/ansible + export PATH=$PATH:$ANSIBLE/bin + export PYTHONPATH=$ANSBILE/lib + export ANSIBLE_LIBRARY=$ANSIBLE/library + +### Restart Cygwin and verify that `ansible --version` displays the ansible version. + +# Configuring Pip on Cygwin +Pip does not work by default on Cygwin. Please follow these steps to enable pip: + + python -m ensure pip + + +# Reference +[Running Ansible within Windows](https://www.jeffgeerling.com/blog/running-ansible-within-windows) \ No newline at end of file
diff --git a/README-EC2.md b/README-EC2.md new file mode 100644 index 0000000..65599f5 --- /dev/null +++ b/README-EC2.md
@@ -0,0 +1,28 @@ +# Overview: EC2 Managed Setup +The minimum setup must be completed prior to configuring the EC2 managed +setup. The EC2 managed setup will update perform the following +activities: +* Update ~/.bashrc to export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY +values you provide. +* Update ~/.bashrc to export EC2_INI_PATH. +* Create a template .boto file and place it your home directory +* Create a template ec2.ini and place it in your ~/.ansible folders +* Provide a copy of the ec2.py script that must be included with any AWS +managed inventory file. + +# Usage: Setup for EC2 Managed Images +Please follow the usage instructions: + + ansible-playbook setup-ec2.yml \ + -e secure_aws_region={{ secure_aws_region }} \ + -e aws_access_key_id={{ aws_access_key_id }} \ + -e aws_secret_access_key={{ aws_secret_access_key }} + +## Variables Used + +Variable Name | Description | +--- | --- | +secure_aws_region | Please provide your ec2_region. | +aws_access_key_id | Please provide your AWS Access Key ID. | +aws_secret_access_key | Please provide your AWS Secret Access Key. | +
diff --git a/inventory.md b/README-INVENTORY-FILE.md similarity index 65% rename from inventory.md rename to README-INVENTORY-FILE.md index 117c98b..4183764 100644 --- a/inventory.md +++ b/README-INVENTORY-FILE.md
@@ -15,9 +15,9 @@ ## Inventory File Conventions These roles depend on use of conventions in the inventory file. -Specifically inventory file conventions are ansible groups names that +Specifically inventory file conventions are ansible group names that are defined around Apigee semantics. These ansible groups are -semantically linked to the documentation. The ansible groups used as +semantically linked to the Apigee documentation. The ansible groups used as conventions correspond to the installation roles and server categorizations called out in the Apigee Private Cloud Installation and Configuration Guide. It has been useful to use planet and region @@ -32,10 +32,10 @@ followed. The inventory file group names for planet level semantics are listed in the template inventory file below. -A region represents subset of a planet. The semantics used for -installation roles are congruent with a region. Region have been +A region represents a subset of a planet. The semantics used for +installation roles are congruent with a region. Regions have been referenced as data centers. The internal configurations of OPDK and BaaS -support many regions as dc-1, dc-2 and so forth. Following this +support many regions such as dc-1, dc-2 ... dc-n. Following this historical precedent we also define the regions with their corresponding installation role to provide a semantic model as follows: @@ -77,10 +77,47 @@ [dc-1-ui] # Listing of the UI node in dc-1 -## Zookeeper Observer Nodes +# Zookeeper Observer Nodes Zookeeper nodes can be designated as an observer node. Ansible inventory files allow variables to be assigned to servers. These roles will update the silent installation configuration file correctly for any zookeeper node that is assigned the variable zk_observer. zk_observer=true + +## Example Zookeeper Observer Node Configuration +This is a sample of a configuration of a Zookeeper node as an observer. Assuming that the first three nodes listed are +Cassandra/Zookeeper nodes then we can configure a node as a zookeeper observer as follows: + + [planet] + dc-1-n1 + dc-1-n2 + dc-1-n3 zk_observer=true + + [dc-1-ds] + dc-1-n[1:3] + +# Cassandra Rackaware Configuration +Cassandra nodes can be configured to be rackaware. This leverages the Cassandra built in functionality +for managing itself when the Cassandra ring is distributed across several availability zones. We would +set the variable rack for each member of the Cassandra ring with the designated location we want for +that node in the following way: + + rack=1,1 + +Please note that this follows the Cassandra rack aware configuration syntax: + + rack=<Availability Zone>,<Position on Rack> + +## Example Cassandra Rackaware Configuration +This is a sample of a configuration of Cassandra as a node that is rackaware. Assuming that the first three nodes listed are +Cassandra/Zookeeper nodes then we can configure a node as rackaware as follows: + + [planet] + dc-1-n1 rack=1,1 + dc-1-n2 rack=2,1 + dc-1-n3 rack=3,1 + + [dc-1-ds] + dc-1-n[1:3] +
diff --git a/README-credentials.md b/README-credentials.md new file mode 100644 index 0000000..3ec38d9 --- /dev/null +++ b/README-credentials.md
@@ -0,0 +1,58 @@ +# Configuring Credentials +Credentials are stored at `~/.apigee-secure/credentials.yml`. A template file is created that includes +commented out attributes as placeholders for values that could be needed. + +## Credentials Location +The `credentials.yml` file has been placed in your user home which is typically referenced by +`~/.apigee`. This location has been selected because of the security it affords the user. Please +note that the scripts expect to find the `credentials.yml` file in the folder `~/.apigee`. + +## Required Credentials +The following table depicts those credentials that are required to complete an +Apigee installation. If these are not provided, then the apigee installation +scripts will prompt you for them. Given that this is an automated framework +that will orchestrate your Apigee installation it is necessary to provide these +attributes ahead of time in the `~/.apigee-secure/credentials.yml` file. + +| Required Attribute | Value | Description | +|---|---|---| +| apigee_repo_user | APIGEE_REPOSITORY_USERNAME | The customer login to the software.apigee.com | +| apigee_repo_password | APIGEE_REPOSITORY_PASSWORD | The login password to software.apigee.com | +| opdk_user_name | apigee | The Apigee administrator username and operating system username | +| opdk_group_name | apigee | The operating system group name | +| opdk_user_email | APIGEE_ADMIN_EMAIL | The email address for the Apigee administrator | +| opdk_user_pass | APIGEE_ADMIN_PASSWORD | The password used by the Apigee Administrator | +| opdk_cass_username | cassandra | The Cassandra username | +| opdk_cass_password | cassandra | The Cassandra password | +| opdk_ldap_pass | APIGEE_LDAP_PASSWORD | The Apigee LDAP password | +| pg_user | admin | The Postgres database administrator username | +| pg_pass | postgres | The Postgres database administrator password | + +## Optional Credentials + +The following table depicts those credentials that are not a part of the Apigee installation. +These can be provided here or they may be provided by any of the other mechanisms that Ansible +uses to accept parameters such as `vars_files`, `-e ATTRIBUTE_NAME=ATTRIBUTE_VALUE`, an inventory +file or a group or host variable files. + +| Optional Attribute | Value | Description | +|---|---|---| +| service_account_authorization_token | "Basic BASE64_ENCODED_USERNAME_PASSWORD" | Authorization header value to be used when registering the instance with Cyberark | +| aws_access_key | AWS_ACCESS_KEY | AWS access key | +| aws_secret_key | AWS_SECRET | AWS secret | +| aws_region | AWS_REGION | AWS region | +| security_group | AWS_SECURITY_GROUP_NAME | AWS security group name | +| key_name | AWS_KEY_PAIR_NAME | AWS key pair name used for access | +| grafana_username | GRAFANA_USERNAME | Used if installing the Apigee monitoring dashboard | +| grafana_password | GRAFANA_PASSWORD | Used if installing the Apigee monitoring dashboard | +| opdk_smtp_user | SMTP_USERNAME | Username of the SMTP server that Edge will use to send notifications to developers | +| opdk_smtp_password | SMTP_Password | Password for the SMTP server that Edge will use to send notifications to developers | +| baas_admin_name | BAAS_ADMIN_NAME | The Apigee Baas administrator username, please note that baas_admin_email cannot be the same as baas_superuser_email | +| baas_admin_password | BAAS_ADMIN_PASSWORD | The Apigee Baas administrator password | +| baas_admin_email | BAAS_ADMIN_EMAIL | The Apigee Baas administrator email | +| baas_admin_pass | BAAS_ADMIN_PASS | The Apigee Baas administrator email password | +| baas_superuser_email | BAAS_SUPERUSER_EMAIL | Email of the Baas Superuser, please note that baas_superuser_email cannot be the same as baas_admin_email | +| baas_superuser_name | BAAS_SUPERUSER_NAME | Name of the Baas Superuser | +| baas_superuser_pass | BAAS_SUPERUSER_PASSWORD | Password of the Baas Superuser | +| baas_smtp_user | BAAS_SMTP_USER | Username of the SMTP server that Baas will use to send notifications to developers | +| baas_smtp_user_pass | BAAS_SMTP_USER_PASS | Password for the SMTP server that Baas will use to send notification to developers |
diff --git a/README.md b/README.md index b7d5e8e..25f61a2 100644 --- a/README.md +++ b/README.md
@@ -1,67 +1,87 @@ -Apigee OPDK Ansible Setup Playbook -================================== - -The purpose of this project is to help configure Ansible for the use of [Apigee OPDK roles](https://github.com/carlosfrias/apigee-opdk-playbook-samples). +# Apigee OPDK Ansible Setup Playbook +The purpose of this project is to help configure Ansible for the use of [Apigee OPDK roles](https://github.com/carlosfrias/apigee-opdk-playbook-samples). The Apigee OPDK roles follow conventions in the naming of files and where those files are located. This project sets up those locations and performs the git clone to setup your -environment with the templates that should be customized for your use. +environment with the templates that should be customized for your use. -Usage: Minimum Setup -==================== +# Quick Installation +Assuming ansible is already installed then you can setup with the following steps: +1. `mkdir ~/apigee-workspace` +1. `cd ~/apigee-workspace` +1. Make sure your public ssh key has been added to stash. +1. `git clone git@github.com:carlosfrias/apigee-opdk-playbook-setup-ansible.git` +1. `cd apigee-opdk-playbook-setup-ansible` +1. `ansible-playbook setup.yml` +1. Update the credentials.yml file as indicated below. +1. Add the license.txt file to `~/.apigee-secure/license.txt` +1. Configure ansible to work with the correct configuration using: `export ANSIBLE_CONFIG=~/.ansible/configurations/vz-edge-1701-aio.cfg` +1. `cd ~/apigee-workspace/apigee-opdk-playbook-workspace/apigee-opdk-playbook-installation-single-region` +1. `ansible-galaxy install -r install-edge-requirements.yml` +1. The first invocation should be: `ansible-playbook install-edge.yml --become --become-method=pbrun --tags=cache,ds,ms,rmp,qs,org` + +# Installing Ansible on Windows +Please see the provided [Cygwin readme](README-CYGWIN.md) for instructions on installing Ansible on Windows. + +# Installing Ansible +We use provide a [pip requirements file](ansible-pip-requirements.txt) for pip to install Ansible and its dependencies. This requires the +installation of pip on your system. Once pip is installed then the provided requirements file will +install Ansible and the necessary dependencies on your system. + +# Pip Configuration +It may be necessary to configure pip to work with a repository that is internally managed. This can be accomplished by +updating the pip.conf file with the url of your managed repository. To accomplish this please follow these steps: + + mkdir ~/.pip + touch ~/.pip/pip.conf + vi ~/.pip/pip.conf + + [global] + index-url = {{ pip_index_url }} + +## Managed Pip Url +Please note that the url can be found in the your ~/apigee-workspace/apigee/custom-properties.yml. Please replace as follows: + + {{ pip_index_url }} >> "https://private.repository.com/api/pypi/Pypi-remote/simple" + +# Pip Usage + +* Upgrade your pip installation: `pip install -U pip` +* Install ansible and dependencies using the requirements file: `pip install -r ansible-pip-requirements.txt` + +# Usage: Minimum Setup Set up your a workspace at ~/apigee-workspace/ if you already have git and pip installed: ansible-playbook setup-base.yml -Requirements -============ - -* Ansible 2.1 or greater -* Linux control server version 6 or greater -* Access to github.com -* Git +# Requirements +Requirements are declared in the [pip requirements file](ansible-pip-requirements.txt). The requirements +are listed as follows: +* Ansible 2.3.x * Pip +* Boto, if using Ansible to manage EC2 +* Markupsafe +* Paramiko -Overview: Minimum Setup -======================= - +# Overview: Minimum Setup This playbook will perform the following activities: 1. Scaffold required folders - 1. Git clone of Ansible configuration file templates - -1. Git clone of inventory file templates. [Updating your Inventory File] -(inventory.md) - +1. Git clone of inventory file templates. 1. Create credentials.yml template file -System Packages -=============== - -Required system packages can be installed with - - ansible-playbook install-system-packages.yml -K - -Default Workspace Folders -========================= - +# Default Workspace Folders The default workspace folders can be modified by updating workspace-folders.yml. -Default Github Repository List -============================== - +# Default Github Repository List The github repositories that you include can be modified by updating workspace-github-repos.yml -Inventory Semantics -=================== +# Inventory File Semantics +A description of the how the [Inventory File](README-INVENTORY-FILE.md) should be setup. -[A full description of inventory semantics](inventory.md). - -Scaffold required folders -========================= - -This playbook will create the following folders for you: +# Scaffold required folders +This playbook will create the correct folders for you. Folder location can be modified by updating the [workspace-folders.yml](workspace-folders.yml). ## ~/.ansible/configurations Sample configuration templates for single and multi-dc planets @@ -72,6 +92,8 @@ ## ~/.apigee Folder to contain credentials, downloaded logs, configs and maintain the edge license.txt file. +The [setup-apigee-license.yml](setup-apigee-license.yml) playbook will take a specified license file and +install it in the correct location for your this workspace. ## ~/apigee-workspace/apigee-opdk-playbook-workspace Starter templates for playbooks. Use these playbooks as accelerators for your own. @@ -79,32 +101,11 @@ ## ~/apigee-workspace/apigee-odk-role-workspace Workspace containing the roles used by the playbooks. -Overview: EC2 Managed Setup -=========================== +# Configure Credentials +Configuring of installation credentials is described in the [credentials readme](README-credentials.md). -The minimum setup must be completed prior to configuring the EC2 managed setup. The EC2 managed setup will -update perform the following activities: -* Update ~/.bashrc to export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY values you provide. -* Update ~/.bashrc to export EC2_INI_PATH. -* Create a template .boto file and place it your home directory -* Create a template ec2.ini and place it in your ~/.ansible folders -* Provide a copy of the ec2.py script that must be included with any AWS managed inventory file. +# Configure Ansible +Ansible can be configured to make use of the above folders by providing an Ansible configuration file. +The Ansible configuration file can be generated in using the `setup-ansible-config.yml` playbook. -Usage: Setup for EC2 Managed Images -=================================== - -Please follow the usage instructions: - - ansible-playbook setup-ec2.yml \ - -e secure_aws_region={{ secure_aws_region }} \ - -e aws_access_key_id={{ aws_access_key_id }} \ - -e aws_secret_access_key={{ aws_secret_access_key }} - -## Variables Used - -Variable Name | Description | ---- | --- | -secure_aws_region | Please provide your ec2_region. | -aws_access_key_id | Please provide your AWS Access Key ID. | -aws_secret_access_key | Please provide your AWS Secret Access Key. | - + ansible-playbook setup-ansible-config.yml
diff --git a/ansible-pip-requirements.txt b/ansible-pip-requirements.txt new file mode 100644 index 0000000..12f18ec --- /dev/null +++ b/ansible-pip-requirements.txt
@@ -0,0 +1,4 @@ +ansible==2.3.0 +boto==2.45.0 +markupsafe==0.23 +paramiko==2.0.2
diff --git a/ansible.cfg b/ansible.cfg deleted file mode 100755 index 3b01d09..0000000 --- a/ansible.cfg +++ /dev/null
@@ -1,5 +0,0 @@ -[defaults] -retry_files_enabled = False -# executable = /bin/bash -ansible_shell_type=sh -module_name = shell
diff --git a/backup.yml b/backup.yml new file mode 100644 index 0000000..53ddc03 --- /dev/null +++ b/backup.yml
@@ -0,0 +1,20 @@ +--- +- name: Backup custom-properties, credentials and license files + hosts: "{{ target_hosts }}" + vars_files: + - repository-custom-properties.yml + tasks: + - name: Create backup folder + file: + path: backups + state: directory + + - name: Backup files + fetch: + src: "{{ item.dir }}/{{ item.name }}" + dest: "backups/{{ item.name }}" + flat: yes + with_items: + - { dir: "{{ apigee_config }}", name: license.txt } + - { dir: "{{ apigee_config }}/apigee-opdk-custom-properties", name: custom-properties.yml } + - { dir: "{{ apigee_security }}", name: credentials.yml }
diff --git a/bitbucket-create-update-repos.yml b/bitbucket-create-update-repos.yml deleted file mode 100644 index 3538a72..0000000 --- a/bitbucket-create-update-repos.yml +++ /dev/null
@@ -1,49 +0,0 @@ ---- -- name: Create and update repositories in BitBucket - hosts: localhost - connection: local - vars_files: - - workspace-github-repos.yml - - workspace-folders.yml - - bitbucket-repo-user-list.yml - vars: - bitbucket_project_api_endpoint: "https://onestash.verizon.com/rest/api/1.0/projects/CELVW" - bitbucket_repo_api_endpoint: "{{ bitbucket_project_api_endpoint }}/repos" - tasks: - - name: Create repositories if necessary - uri: - url: "{{ bitbucket_repo_api_endpoint }}" - method: POST - user: "{{ vz_windows_user }}" - password: "{{ vz_windows_password }}" - force_basic_auth: yes - status_code: 201,409 - body_format: json - body: - name: "{{ item.repo_name }}" - scmId: git - forkable: True - with_items: "{{ repo_names }}" - - - name: Add permissions to repos - uri: - url: "{{ bitbucket_project_api_endpoint }}/permissions/users?name={{ item }}&permission=PROJECT_ADMIN" - method: PUT - user: "{{ vz_windows_user }}" - password: "{{ vz_windows_password }}" - force_basic_auth: yes - status_code: 204,409 - with_items: "{{ repo_users }}" - - - name: Add Onestash as remote repo - ignore_errors: true - shell: "git remote add vz ssh://git@onestash.verizon.com:7999/celvw/{{ item.repo_name }}.git" - args: - chdir: "{{ item.workspace }}/{{ item.repo_name }}" - with_items: "{{ repo_names }}" - - - name: Commit updated repos to onestash - shell: "git checkout master && git push -u vz master" - args: - chdir: "{{ item.workspace }}/{{ item.repo_name }}" - with_items: "{{ repo_names }}"
diff --git a/bitbucket-remove-repos.yml b/bitbucket-remove-repos.yml deleted file mode 100644 index 1b30add..0000000 --- a/bitbucket-remove-repos.yml +++ /dev/null
@@ -1,15 +0,0 @@ ---- -- name: Create repositories in BitBucket - hosts: localhost - connection: local - vars: - onestash_url: "https://onestash.verizon.com/rest/api/1.0/projects/CELVW/repos" - tasks: - - name: Remove repo - uri: - url: "{{ onestash_url }}/delete_test_repo" - method: DELETE - user: "{{ vz_windows_user }}" - password: "{{ vz_windows_password }}" - status_code: 202,204 - force_basic_auth: yes
diff --git a/bitbucket-repo-user-list.yml b/bitbucket-repo-user-list.yml deleted file mode 100644 index 680aeaf..0000000 --- a/bitbucket-repo-user-list.yml +++ /dev/null
@@ -1,8 +0,0 @@ ---- -repo_users: -- friasca -- v636401 -- v532218 -- v412810 -- v674375 -- v682557 \ No newline at end of file
diff --git a/control-server.yml b/control-server.yml index de25fe3..a878696 100644 --- a/control-server.yml +++ b/control-server.yml
@@ -1,7 +1,6 @@ --- - hosts: '{{ hosts }}' become: yes - become_flags: "beroot" vars_files: - custom-properties.yml tasks: @@ -31,8 +30,3 @@ with_items: - pip - ansible -# - markupsafe -# - boto -# environment: -# http_proxy: "{{ http_proxy }}" -# https_proxy: "{{ https_proxy }}"
diff --git a/pip-requirements.txt b/pip-requirements.txt deleted file mode 100644 index 60bf906..0000000 --- a/pip-requirements.txt +++ /dev/null
@@ -1,3 +0,0 @@ -ansible==2.2.1.0 -boto==2.45.0 -markupsafe==0.23
diff --git a/private-repository-create-update.yml b/private-repository-create-update.yml new file mode 100644 index 0000000..383994e --- /dev/null +++ b/private-repository-create-update.yml
@@ -0,0 +1,53 @@ +--- +- name: Create and update repositories in BitBucket + hosts: localhost + connection: local + vars_files: + - repository-mapping-name-folder.yml + - repository-custom-properties.yml + - private-repository-user-list.yml + tasks: + - name: Create repositories if necessary + uri: + url: "{{ repository_secure_endpoint_https }}" + method: POST + user: "{{ username }}" + password: "{{ password }}" + force_basic_auth: yes + status_code: 201,409 + body_format: json + body: + name: "{{ item.repo_name }}" + scmId: git + forkable: True + with_items: "{{ repo_names }}" + + - name: Add permissions to repos + uri: + url: "{{ repository_secure_endpoint_https }}/permissions/users?name={{ item }}&permission=PROJECT_ADMIN" + method: PUT + user: "{{ username }}" + password: "{{ password }}" + force_basic_auth: yes + status_code: 204,409 + with_items: "{{ repo_users }}" + + - name: Add Onestash as remote repo + ignore_errors: true + shell: "git remote add target {{ repository_secure_endpoint_ssh }}/{{ item.repo_name }}.git" + args: + chdir: "{{ item.workspace }}/{{ item.repo_name }}" + with_items: "{{ repo_names }}" + + - name: Pull from Onestash if the repo is already there + ignore_errors: yes + shell: "git pull target master" + args: + chdir: "{{ item.workspace }}/{{ item.repo_name }}" + with_items: "{{ repo_names }}" + + - name: Commit updated repos to onestash + shell: "git checkout master && git push -u target master" + args: + chdir: "{{ item.workspace }}/{{ item.repo_name }}" + with_items: "{{ repo_names }}"
diff --git a/private-repository-remove.yml b/private-repository-remove.yml new file mode 100644 index 0000000..cf91a19 --- /dev/null +++ b/private-repository-remove.yml
@@ -0,0 +1,15 @@ +--- +- name: Create repositories in BitBucket + hosts: localhost + connection: local + vars_files: + - repository-custom-properties.yml + tasks: + - name: Remove repo + uri: + url: "{{ repository_api_endpoint }}/repos/delete_test_repo" + method: DELETE + user: "{{ username }}" + password: "{{ password }}" + status_code: 202,204 + force_basic_auth: yes
diff --git a/private-repository-user-list.yml b/private-repository-user-list.yml new file mode 100644 index 0000000..b503e3c --- /dev/null +++ b/private-repository-user-list.yml
@@ -0,0 +1,3 @@ +--- +repo_users: +- friasca
diff --git a/repository-custom-properties.yml b/repository-custom-properties.yml new file mode 100644 index 0000000..2ca0a91 --- /dev/null +++ b/repository-custom-properties.yml
@@ -0,0 +1,17 @@ +--- +ansible_config: ~/.ansible +apigee_config: ~/.apigee +apigee_security: ~/.apigee-secure +apigee_workspace: ~/apigee-workspace +playbook_workspace: "{{ apigee_workspace }}/apigee-opdk-playbook-workspace" +role_workspace: "{{ apigee_workspace }}/apigee-opdk-role-workspace" +repository_secure_endpoint_https: "https://github.com/carlosfrias/apigee-opdk-playbook-setup-ansible.git" +repository_secure_endpoint_ssh: "git@github.com:carlosfrias/apigee-opdk-playbook-setup-ansible.git" +configuration_folders: +- '{{ ansible_config }}/configurations' +- '{{ ansible_config }}/inventory' +- '{{ ansible_config }}/tmp/logs' +- '{{ apigee_config }}/logs_configs' +- '{{ apigee_security }}' +- '{{ playbook_workspace }}' +- '{{ role_workspace }}'
diff --git a/workspace-github-repos.yml b/repository-mapping-name-folder.yml similarity index 83% rename from workspace-github-repos.yml rename to repository-mapping-name-folder.yml index 14c4333..aad1157 100755 --- a/workspace-github-repos.yml +++ b/repository-mapping-name-folder.yml
@@ -1,8 +1,11 @@ --- -repo_names: -- { workspace: '{{ playbook_dir }}/..', repo_name: local-workspace-maintenance } +config_repos: - { workspace: '{{ playbook_dir }}/..', repo_name: apigee-opdk-playbook-setup-ansible } -- { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-installation-aio } +- { workspace: '{{ apigee_config }}', repo_name: apigee-opdk-custom-properties } +- { workspace: '{{ playbook_dir }}/..', repo_name: local-workspace-maintenance } + +playbook_repos: +- { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-installation-aio } # moved into apigee-opdk-playbook-installation-single-region - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-installation-baas } - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-installation-microgateway } - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-installation-mirror } @@ -22,6 +25,8 @@ - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-maintenance-validate-port-availability } - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-samples } - { workspace: '{{ playbook_workspace }}', repo_name: apigee-opdk-playbook-maintenance-cassandra-replication-factor } + +role_repos: - { workspace: '{{ role_workspace }}', repo_name: apigee-fetch-files } - { workspace: '{{ role_workspace }}', repo_name: apigee-internal-port-connectivity-validator-cassandra } - { workspace: '{{ role_workspace }}', repo_name: apigee-internal-port-connectivity-validator-ldap } @@ -50,6 +55,7 @@ - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-set-reachable } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-analytics-group-add } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-apigee-user } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-apigee-user-ownership } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-bootstrap } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-bootstrap-archive-installer } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-bootstrap-create-archive } @@ -96,3 +102,19 @@ - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-cache-update } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-ansible-configuration-samples } - { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-ansible-inventory-samples } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-java-home } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-target-links } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-ssh-user } +- { workspace: '{{ role_workspace }}', repo_name: apigee-server-restart } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-cwc-update } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-postgres-db-upgrade } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-yum-repository-enabled } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-yum-repository-proxy-config } +- { workspace: '{{ role_workspace }}', repo_name: apigee-lvm-increase-volume-size } +- { workspace: '{{ role_workspace }}', repo_name: apigee-lvm-create-physical-volume } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-yum-packages } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-mirror-nginx } +- { workspace: '{{ role_workspace }}', repo_name: apigee-opdk-setup-component-install } +- { workspace: '{{ role_workspace }}', repo_name: vz-application-netgroup-access } +- { workspace: '{{ role_workspace }}', repo_name: vz-cyberark-register-ec2 } +- { workspace: '{{ role_workspace }}', repo_name: vz-firstboot-operations-verification }
diff --git a/resources/ansible.cfg.j2 b/resources/ansible.cfg.j2 new file mode 100644 index 0000000..585ec40 --- /dev/null +++ b/resources/ansible.cfg.j2
@@ -0,0 +1,36 @@ +[defaults] +hostfile = {{ ansible_config }}/inventory/ +fact_caching_connection = {{ ansible_config }}/tmp/cache/ +log_path = {{ ansible_config }}/tmp/logs/ansible-default.log + +# remote_user = {{ username }} +# private_key_file = ~/.ssh/id_rsa + +roles_path = {{ ansible_config }}/roles +local_tmp = {{ ansible_config }}/tmp +library = {{ ansible_config }}/library/ + +host_key_checking = false +forks = 10 +retry_files_enabled = False + +ansible_shell_type=sh +gathering = smart +fact_caching = jsonfile +module_name = shell + +merge_multiple_cli_tags=True +gather_subset = !ohai,!facter +display_args_to_stdout = True + +timeout = 60 + +[ssh_connection] +pipelining = True +# Sample ssh_args with extended arguments +# ssh_args = -o ControlMaster=auto -o ControlPersist=7200s -o KbdInteractiveAuthentication=no -o PreferredAuthentications=gssapi-with-mic,gssapi-keyex,hostbased,publickey -o PasswordAuthentication=no -q + +# Plain vanilla ssh configuration +ssh_args = -o ControlMaster=auto -o ControlPersist=7200s -q +control_path=%(directory)s/ansible-ssh-%%h-%%p-%%r +
diff --git a/resources/credentials.yml b/resources/credentials.yml index ddf8c05..b720d05 100644 --- a/resources/credentials.yml +++ b/resources/credentials.yml
@@ -1,4 +1,7 @@ --- +# Basic authorization header used to authenticate with Cyberark service +# service_account_authorization_token: "Basic BASE64_ENCODED_USERNAME" + # AWS credentials # aws_access_key: # aws_secret_key:
diff --git a/resources/custom-properties.yml b/resources/custom-properties.yml index 449105f..4ed7815 100644 --- a/resources/custom-properties.yml +++ b/resources/custom-properties.yml
@@ -1,7 +1,7 @@ --- opdk_version: 4.17.01 -# pip_index_url: https://pip.artifactory-server.com/artifactory/api/pypi/Pypi-remote/simple -# pip_conf_dir: ~/.pip +# pip_index_url: "https://private.repository.com/api/pypi/Pypi-remote/simple" +# pip_conf_dir: "~/.pip" # Used with restarting servers # start_check_delay: 0
diff --git a/setup-ansible-config.yml b/setup-ansible-config.yml new file mode 100644 index 0000000..d4ff2d1 --- /dev/null +++ b/setup-ansible-config.yml
@@ -0,0 +1,17 @@ +--- +- hosts: localhost + connection: local + vars_files: + - repository-custom-properties.yml + vars_prompt: + - name: username + prompt: "Enter your the username for ansible" + private: no + tasks: + - name: Create ansible.cfg file from template + template: + src: resources/ansible.cfg.j2 + dest: "{{ apigee_workspace }}/apigee-opdk-playbook-setup-ansible/ansible.cfg" + backup: yes + force: yes +
diff --git a/setup-apigee-license.yml b/setup-apigee-license.yml index 4878f7b..89e635c 100644 --- a/setup-apigee-license.yml +++ b/setup-apigee-license.yml
@@ -1,9 +1,8 @@ --- - hosts: localhost connection: local - vars: - apigee_config: ~/.apigee - + vars_files: + - repository-custom-properties.yml vars_prompt: - name: license_path prompt: "Enter path to license file"
diff --git a/setup-base.yml b/setup-base.yml index 2640b62..4cab921 100644 --- a/setup-base.yml +++ b/setup-base.yml
@@ -2,43 +2,77 @@ - hosts: localhost connection: local vars_files: - - workspace-github-repos.yml - - workspace-folders.yml + - repository-mapping-name-folder.yml + - repository-custom-properties.yml tasks: - - name: Create ansible configuration folders + - name: Ensure ansible configuration folders exist file: path: '{{ item }}' state: directory with_items: '{{ configuration_folders }}' + tags: ['config'] - name: Git checkout of sample configurations + tags: ['config'] become: false ignore_errors: yes git: - repo: '{{ host_repo }}/apigee-opdk-ansible-configuration-samples.git' + repo: '{{ repository_secure_endpoint_ssh }}/apigee-opdk-ansible-configuration-samples.git' dest: "{{ ansible_config }}/configurations" accept_hostkey: yes - - name: Git checkout of sample inventories + - name: Git checkout of custom properties + tags: ['config'] become: false ignore_errors: yes git: - repo: '{{ host_repo }}/apigee-opdk-ansible-inventory-samples.git' + repo: '{{ repository_secure_endpoint_ssh }}/apigee-opdk-custom-properties.git' + dest: "{{ apigee_config }}/apigee-opdk-custom-properties" + accept_hostkey: yes + + - name: Git checkout of sample inventories + tags: ['inventory'] + become: false + ignore_errors: yes + git: + repo: '{{ repository_secure_endpoint_ssh }}/apigee-opdk-ansible-inventory-samples.git' dest: "{{ ansible_config }}/inventory" accept_hostkey: yes - - name: Git checkout of sample playbooks + - name: Git checkout of configuration repositories + tags: ['config'] become: false ignore_errors: yes git: - repo: '{{ host_repo }}/{{ item.repo_name }}.git' + repo: '{{ repository_secure_endpoint_ssh }}/{{ item.repo_name }}.git' dest: "{{ item.workspace }}/{{ item.repo_name }}" accept_hostkey: yes - with_items: "{{ repo_names }}" + with_items: "{{ config_repos }}" + + - name: Git checkout of playbook repositories + tags: ['playbooks'] + become: false + ignore_errors: yes + git: + repo: '{{ repository_secure_endpoint_ssh }}/{{ item.repo_name }}.git' + dest: "{{ item.workspace }}/{{ item.repo_name }}" + accept_hostkey: yes + with_items: "{{ playbook_repos }}" + + - name: Git checkout of role repositories + tags: ['roles'] + become: false + ignore_errors: yes + git: + repo: '{{ repository_secure_endpoint_ssh }}/{{ item.repo_name }}.git' + dest: "{{ item.workspace }}/{{ item.repo_name }}" + accept_hostkey: yes + with_items: "{{ role_repos }}" + when: role_repos is defined - name: Add empty credentials.yml file to .apigee copy: src: resources/credentials.yml - dest: '{{ apigee_config }}/credentials.yml' + dest: '{{ apigee_security }}/credentials.yml' force: no
diff --git a/setup.yml b/setup.yml index f5d0dee..867efb8 100644 --- a/setup.yml +++ b/setup.yml
@@ -1,6 +1,8 @@ --- +# - include: setup-ansible-config.yml + #- include: control-server.yml - include: setup-base.yml -#- include: setup-ec2.yml \ No newline at end of file +#- include: setup-ec2.yml
diff --git a/workspace-folders.yml b/workspace-folders.yml deleted file mode 100755 index 99f777d..0000000 --- a/workspace-folders.yml +++ /dev/null
@@ -1,14 +0,0 @@ ---- -apigee_workspace: /tmp/apigee-workspace -ansible_config: "{{ apigee_workspace }}/ansible" -apigee_config: "{{ apigee_workspace }}/apigee" -playbook_workspace: "{{ apigee_workspace }}/apigee-opdk-playbook-workspace" -role_workspace: "{{ apigee_workspace }}/apigee-opdk-role-workspace" -host_repo: https://github.com/carlosfrias -configuration_folders: -- '{{ ansible_config }}/configurations' -- '{{ ansible_config }}/inventory' -- '{{ ansible_config }}/tmp/logs' -- '{{ apigee_config }}' -- '{{ playbook_workspace }}' -- '{{ role_workspace }}'