Removed GSI specifics, making everything configurable. Added template files...

Removed GSI specifics, making everything configurable. Added template files for configuration orientation. Added root user option to credentials.yml

.gitignore

 .vagrant
 tmp/*
 *.retry
+*.inv
+GSI_*
+OS_*
+!templates/*

.hosts

-172.16.0.150
-172.16.0.151
-172.16.0.152
-172.16.0.108
-172.16.0.115
-172.16.0.118

Vagrantfile

-Vagrant.configure("2") do |config|
-
-  config.vm.define "datamanager" do |datamanager|
-    datamanager.vm.hostname="datamanager"
-    datamanager.vm.box = "debian/contrib-jessie64"
-    datamanager.vm.network "forwarded_port", guest: 80, host: 1097
-    datamanager.vm.network "private_network", ip: "172.16.1.200",netmask:"255.240.0.0"
-    datamanager.vm.provision :ansible do |ansible|
-          ansible.playbook= "playbook.yml"
-    end
-  end
-
-  config.vm.define "dataserver" do |dataserver|
-    dataserver.vm.hostname="dataserver"
-    dataserver.vm.box = "debian/contrib-jessie64"
-    dataserver.vm.network "forwarded_port", guest: 1094, host: 1094
-    dataserver.vm.network "forwarded_port", guest: 80, host: 1096
-    dataserver.vm.network "private_network", ip: "172.16.1.201",netmask:"255.240.0.0"
-    dataserver.vm.provision :ansible do |ansible|
-          ansible.playbook= "playbook.yml"
-    end
-  end
-
-  config.vm.define "client" do |client|
-    client.vm.hostname="client"
-    client.vm.box = "debian/contrib-jessie64"
-    client.vm.network "private_network", ip: "172.16.1.203",netmask:"255.240.0.0"
-    client.vm.provision :ansible do |ansible|
-          ansible.playbook= "playbook.yml"
-    end
-  end
-
-  config.vm.define "dynafed" do |dynafed|
-    client.vm.hostname="dynafed"
-    client.vm.box = "debian/contrib-jessie64"
-    client.vm.network "private_network", ip: "172.16.1.204",netmask:"255.240.0.0"
-    client.vm.provision :ansible do |ansible|
-          ansible.playbook= "playbook.yml"
-    end
-  end
-end

boxes

-[debian]
-172.16.0.151
-172.16.0.115
-172.16.0.150
-
-[xrootd]
-172.16.0.151
-172.16.0.115
-172.16.0.150
-
-[dynafed]
-172.16.0.152
-172.16.0.156
-172.16.0.155
-172.16.0.157
-172.16.0.159
-172.16.0.160
-
-[proxies]
-172.16.0.153
-172.16.0.161
-172.16.0.162
-
-[datamanagers]
-
-[dataservers]
-172.16.0.151
-172.16.0.115
-172.16.0.150

credentials.yml

-http_proxy: lxproxy01.gsi.de
+#YUM mirror/http proxy settings
+http_proxy: lxproxy01.gsi.de #SET TO "none" if no proxy needed. The Proxy through which the YUM mirrors have to be accessed
 http_port: 8080
-xrootd_interface: http #http or xrootd
-xrootd_port: 80 # This is the same always. It's a bug
+
+#centos settings
+running_user_centos: pkramp #the user with root permissions on the three centos LAP machines. Can be root
+
+#dynafed_settings:
+dynafed_json: GSI_dynafedInstances.json #the file that specifies details to the dynafed instances
+
+#XRootD Dataserver config
+xrootd_interface: http
+xrootd_port: 80
+running_user_debian: debian

dynafedInstances.json

-{
-  "networks":[
-    {
-      "name": "GROdynafed",
-      "ip": "172.16.11.100",
-      "proxy": "172.16.11.102",
-        "dataservers": [
-
-        ],
-        "dynafeds": [
-          {
-            "name": "GROdynafed_2",
-            "ip": "172.16.11.101"
-          },
-          {
-            "name": "GSIdynafed_2",
-            "ip": "172.16.6.101"
-          },
-          {
-            "name": "KIdynafed_2",
-            "ip": "172.16.10.101"
-          }
-        ]
-    },
-    {
-      "name": "GROdynafed_2",
-      "ip": "172.16.11.101",
-      "proxy": "none",
-        "dataservers": [
-          {
-            "name": "dataserver1",
-            "ip": "172.16.3.100"
-          }
-        ],
-        "dynafeds": [
-
-        ]
-    },
-    {
-      "name": "GSIdynafed",
-      "ip": "172.16.6.100",
-      "proxy": "172.16.6.102",
-        "dataservers": [
-
-        ],
-        "dynafeds": [
-          {
-            "name": "GROdynafed_2",
-            "ip": "172.16.11.101"
-          },
-          {
-            "name": "GSIdynafed_2",
-            "ip": "172.16.6.101"
-          },
-          {
-            "name": "KIdynafed_2",
-            "ip": "172.16.10.101"
-          }
-        ]
-    },
-    {
-      "name": "GSIdynafed_2",
-      "ip": "172.16.6.101",
-      "proxy": "none",
-        "dataservers": [
-          {
-            "name": "dataserver1",
-            "ip": "172.16.2.100"
-          }
-        ],
-        "dynafeds": [
-
-        ]
-    },
-    {
-      "name": "KIdynafed",
-      "ip": "172.16.10.100",
-      "proxy": "172.16.10.102",
-        "dataservers": [
-
-        ],
-        "dynafeds": [
-          {
-            "name": "GROdynafed_2",
-            "ip": "172.16.11.101"
-          },
-          {
-            "name": "GSIdynafed_2",
-            "ip": "172.16.6.101"
-          },
-          {
-            "name": "KIdynafed_2",
-            "ip": "172.16.10.101"
-          }
-        ]
-    },
-    {
-      "name": "KIdynafed_2",
-      "ip": "172.16.10.101",
-      "proxy": "none",
-        "dataservers": [
-          {
-            "name": "dataserver1",
-            "ip": "172.16.9.100"
-          }
-        ],
-        "dynafeds": [
-
-        ]
-    }
-  ]
-}

playbook.yml

 ---
 - hosts: debian
-  user: debian
-  sudo: true
+  user: "{{ running_user_debian }}"
+  become: yes
   vars_files:
     - credentials.yml
   roles:
@@ -9,16 +9,16 @@
    - debian
 
 - hosts: xrootd
-  user: debian
-  sudo: true
+  user: "{{ running_user_debian }}"
+  become: yes
   vars_files:
     - credentials.yml
   roles:
    - common
 
 - hosts: proxies
-  user: root
-  sudo: true
+  user: "{{ running_user_centos }}"
+  become: yes
   vars_files:
     - credentials.yml
   roles:
@@ -26,10 +26,10 @@
    - proxy
 
 - hosts: dynafed
-  user: root
+  user: "{{ running_user_centos }}"
+  become: yes
   vars:
-    endpoints: "{{ lookup('file','dynafedInstances.json')|from_json }}"
-  sudo: true
+    endpoints: "{{ lookup('file','{{ dynafed_json }}')|from_json }}"
   vars_files:
     - credentials.yml
   roles:
@@ -37,17 +37,9 @@
    - dynafed
 
 - hosts: dataservers
-  user: debian
-  sudo: true
+  user: "{{ running_user_debian }}"
+  become: yes
   vars_files:
     - credentials.yml
   roles:
    - dataserver
-#
-#- hosts: 172.16.0.118
-#  user: debian
-#  sudo: true
-#  vars_files:
-#    - credentials.yml
-#  roles:
-#   - client

readme.md

 # Datalake Local Access Point Setup
+git clone https://git.gsi.de/dc/datalake-reps/ansible_lap.git
+cd ansible_lap
+
+Creation of hosts.inv file from templates/template_hosts.inv
+Creation of dynafedInstances.inv file from templates/template_dynafedInstances.inv
+
+configure credentials.yml with the user you want to run as
+  -root or user with root permissions
+    -create user with root permissions
+      -on centos: adduser USER && passwd USER && usermod -aG wheel USER
+    -put user public key in /home/USER/.ssh/.authorized_keys
+      -mkdir /home/USER/.ssh && vi /home/USER/.ssh/authorized_keys
+
+
+ansible-playbook playbook.yml -i YOUR_HOSTS_FILE --private-key ~/.ssh/USER_PRIVATE_KEY --ask-become-pass

roles/dynafed/tasks/install_dynafed.yml

@@ -3,6 +3,7 @@
    template:
     src: yum.conf.j2
     dest: /etc/yum.conf
+   when:  http_proxy != "none"
 
  - name: overwrite /etc/yum.repos.d/epel.repo
    copy:

roles/dynafed/templates/yum.conf.j2

 [main]
-proxy=http://{{ http_proxy }}
+proxy=http://{{ http_proxy }}:{{ http_port }}

roles/proxy/tasks/configureSquid.yml

@@ -17,3 +17,8 @@
     notify: restart squid
     tags:
       - config
+
+  - name: Add the user 'squid' as a systemuser
+    user:
+      name: squid
+      system: yes

roles/proxy/tasks/installSquid.yml

+---
+ - name: download squid install file zip
+   run_once: true
+   local_action:
+     get_url
+       url=https://git.gsi.de/dc/datalake-reps/squid-proxy-docker-container/uploads/94fdd6d63082db5d0cdf9382a11e0220/squid-4.7.tar.gz
+       dest=/tmp/squid-4.7.tar.gz
+       mode=666
+
+ - name: Copy file with owner and permissions
+   copy:
+     src: /tmp/squid-4.7.tar.gz
+     dest: /tmp/squid-4.7.tar.gz
+     mode: '0744'
+
+ - name: Extract /tmp/squid-4.7.tar.gz into /tmp/
+   unarchive:
+      src: /tmp/squid-4.7.tar.gz
+      dest: /tmp/
+      remote_src: true
+
+ - name: install squid from src /tmp/squid-4.7'
+   shell: 'make install -C /tmp/squid-4.7'

roles/proxy/tasks/proxySetup.yml

@@ -3,18 +3,7 @@
    template:
     src: yum.conf.j2
     dest: /etc/yum.conf
-
-# - name: copy os-mirror.repo
-#   copy:
-#     src: files/yum.repos.d/os-mirror.repo
-#     dest: /etc/yum.repos.d/os-mirror.repo
-#     mode: '0744'
-
- #- name: copy update-mirror.repo
-#   copy:
-#     src: files/yum.repos.d/update-mirror.repo
-#     dest: /etc/yum.repos.d/update-mirror.repo
-#     mode: '0744'
+   when:  http_proxy != "none"
 
  - name: overwrite /etc/yum.repos.d/epel.repo
    copy:
@@ -44,116 +33,33 @@
     - libecap-devel
     - make
     - cmake
-   tags:
-     - install_pre
-
- - name: download squid install file zip
-   run_once: true
-   local_action:
-     get_url
-       url=https://git.gsi.de/dc/datalake-reps/squid-proxy-docker-container/uploads/046b7d463f8c22cfae0f66f41ae7c269/squid-4.7.tar.gz
-       dest=/tmp/squid-4.7.tar.gz
-       mode=666
-   tags:
-      - install_pre
-
-#--------------WORKFLOW remote unpack------------------------
- - name: Copy file with owner and permissions
-   copy:
-     src: /tmp/squid-4.7.tar.gz
-     dest: /home/centos/
-     owner: centos
-     group: centos
-     mode: '0744'
-   tags:
-     - remote_unpack
-
- - name: Create a directory if it does not exist
-   file:
-     path: /home/centos/tmpbuild
-     state: directory
-     mode: '0755'
-   tags:
-     - remote_unpack
-
- - name: Extract /home/centos/squid-4.7.tar.gz into /home/centos/squid-4.7
-   unarchive:
-      src: /home/centos/squid-4.7.tar.gz
-      dest: /home/centos/tmpbuild
-      remote_src: true
-   tags:
-      - remote_unpack
-
- - name: Move /home/centos/tmpbuild/build/squid-4.7 to /home/centos/
-   command: mv /home/centos/tmpbuild/build/squid-4.7 /home/centos/
-   tags:
-      - remote_unpack
-
-#--------------WORKFLOW remote unpack END------------------------
-
-#--------------WORKFLOW local unpack ------------------------
-# - name: Extract /home/centos/squid-4.7.tar.gz into /home/centos/squid-4.7
-#   run_once: true
-#   local_action:
-#     unarchive
-#      src=/tmp/squid-4.7.tar.gz
-#      dest=/tmp
-#   tags:
-#      - local_unpack
-#
-# - name: Copy file with owner and permissions
-#   copy:
-#     src: /tmp/build/squid-4.7
-#     dest: /home/centos/squid-4.7
-#     owner: centos
-#     group: centos
-#     mode: '0744'
-#   tags:
-#      - local_unpack
-#--------------WORKFLOW local unpack END------------------------
-
- - name: install squid from src /usr/src/squid
-   shell: 'make install -C /home/centos/squid-4.7'
-   tags:
-      - install
 
  - name: Install ecap modifying adapter libraries
    copy:
      src: "{{ item }}"
      dest: /usr/local/lib
-     owner: centos
-     group: centos
      mode: '0744'
    with_items:
      - files/lib/ecap_adapter_modifying.a
      - files/lib/ecap_adapter_modifying.la
      - files/lib/ecap_adapter_modifying.so
-   tags:
-     - config
 
- - name: Add the user 'squid' as a systemuser
-   user:
-     name: squid
-     system: yes
-   tags:
-      - config
+ - name: install squid
+   vars:
+     endpoints: "{{ lookup('file','{{ dynafed_json }}')|from_json }}"
+   include: installSquid.yml
 
- - name: copy squid.conf
+ - name: configure squid
    vars:
-     endpoints: "{{ lookup('file','dynafedInstances.json')|from_json }}"
+     endpoints: "{{ lookup('file','{{ dynafed_json }}')|from_json }}"
    include: configureSquid.yml
    notify: restart squid
-   tags:
-     - config
-
 
  - name: Change file ownership, group and permissions
    file:
      path: /var/logs/
      owner: root
      mode: '0777'
-   tags:
-     - config
 
  - name: start squid
    service:

templates/template_dynafed_instances.json

+{
+  "networks":[
+    {
+      "name": "your_dynafed_1_hostname",
+      "ip": "your_dynafed_1_ip",
+      "proxy": "your_squid_proxy_ip",
+        "dataservers": [
+
+        ],
+        "dynafeds": [
+          {
+            "name": "GSIdynafed_2",
+            "ip": "172.16.6.101"
+          }
+        ]
+    },
+    {
+      "name": "your_dynafed_2_hostname",
+      "ip": "your_dynafed_2_ip",
+      "proxy": "none",
+        "dataservers": [
+          {
+            "name": "your_http_storage_hostname",
+            "ip": "your_http_storage_ip"
+          }
+        ],
+        "dynafeds": [
+
+        ]
+    }
+  ]
+}

templates/template_hosts.inv

+[debian]
+(your_debian_host1) only if XRootD-Dataserver is the Storageelement
+[xrootd]
+(your_debian_host1) only if XRootD-Dataserver is the Storageelement
+[dataservers]
+(your_debian_host1) only if XRootD-Dataserver is the Storageelement
+
+[dynafed]
+your_centos_host2
+your_centos_host3
+
+[proxies]
+your_centos_host4