Dev: Added functionality to automate the docker-compose automation.

docker_automate_script.py

+if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
+import argparse
+import os
+import sys
+import time
+
+from scripts.core import ILensVersionHandler
+from scripts.core.docker_handler import DockerHandler
+from scripts.core.git_handler import GitHandler
+from scripts.logging import logging
+from scripts.utils.common_utils import CommonUtils
+
+common_util = CommonUtils()
+variables_file_path = "docker-compose.yml"
+_module_data = common_util.convert_yaml_to_define_obj(variables_file_path)
+default_link = "https://gitlab-pm.knowledgelens.com/"
+git_user_name = os.environ.get("GIT_USERNAME", default="harshavardhan.c")
+git_access_token = os.environ.get("GIT_TOKEN", default="FEMA6PnP63fJCs6DrtZJ")
+config_variables = os.environ.get("CONFIG_MAP_VARIABLES", default="").split(",")
+docker_repo = os.environ.get("DOCKER_REPO",
+                             default="https://gitlab-pm.knowledgelens.com/KnowledgeLens/Products/iLens-2.0/core/devops/docker-compose.git")
+global_configmap = os.environ.get("GLOBAL_VARIABLES_FILE", default="ilens-env-spec-variables.yml")
+
+git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
+# for module in _module_data:
+
+
+print(_module_data)
+ap = argparse.ArgumentParser()
+db_handler = ILensVersionHandler()
+docker_handler = DockerHandler(git_user_name=git_user_name, git_access_token=git_access_token)
+if __name__ == '__main__':
+    ap.add_argument(
+        "--ilens_version",
+        "-iv",
+        required=False,
+        default=None,
+        help="ILens Version Tag",
+    )
+    ap.add_argument(
+        "--release_version",
+        "-rv",
+        required=False,
+        default=None,
+        help="ILens Release Tag",
+    )
+    ap.add_argument(
+        "--client_name",
+        "-cn",
+        required=False,
+        default=None,
+        help="Client Name Tag"
+    )
+    ap.add_argument(
+        "--branch_name",
+        "-b",
+        required=False,
+        default=None,
+        help="Branch Name"
+    )
+    ap.add_argument(
+        "--git_repos",
+        "-gr",
+        required=False,
+        default=None,
+        help="Git repos to be added in helm",
+        nargs="+"
+    )
+    ap.add_argument(
+        "--module_names",
+        "-mn",
+        required=False,
+        default=None,
+        help="Module names to be added in helm",
+        nargs="+"
+    )
+    DOCKER_TEMP_PATH = f"{int(time.time())}_docker_tmp_path"
+    GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
+    OUTPUT_PATH = f"{int(time.time())}_docker-compose"
+    try:
+        arguments = vars(ap.parse_args())
+        _release_version = arguments["release_version"]
+        _client_name = arguments['client_name']
+        _git_repos = arguments["git_repos"]
+        _module_names = arguments["module_names"]
+        _branch_name = arguments['branch_name'] or "master"
+        _ilens_version = arguments["ilens_version"]
+        if not _ilens_version or not _release_version or not _client_name or not (
+                _git_repos or _module_names) or not global_configmap:
+            print(
+                "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
+            sys.exit()
+        _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
+        variables_file = "variables.yml"
+        if not os.path.exists(DOCKER_TEMP_PATH):
+            os.makedirs(DOCKER_TEMP_PATH)
+        if not os.path.exists(OUTPUT_PATH):
+            os.makedirs(OUTPUT_PATH)
+        docker_compose_path = os.path.join(DOCKER_TEMP_PATH, "docker-compose")
+        if not git_handler_obj.clone_repository(repo_link=docker_repo, module_output_path=docker_compose_path,
+                                                clone_branch=_client_name):
+            logging.error(f"Cannot clone helm repo with branch: {_client_name}")
+            sys.exit()
+        files_info = os.listdir(docker_compose_path)
+        sorted_files = list(filter(lambda f: f.endswith(".yml") and "docker" in f, files_info))
+        global_config_data = common_util.convert_yaml_to_define_obj(os.path.join(docker_compose_path, global_configmap))
+
+        for _file in sorted_files:
+            docker_compose_data = common_util.convert_yaml_to_define_obj(os.path.join(docker_compose_path, _file),
+                                                                         load_type=None)
+            service_dict = docker_compose_data.get("services")
+            if not service_dict:
+                logging.debug(f'Services not found for current docker compose file  - {_file}')
+            if response_data := docker_handler.process_module_request(compose_data=docker_compose_data,
+                                                                      variables_file=variables_file,
+                                                                      general_path=GENERAL_TEMP_PATH, **arguments):
+                compose_out_file_path = os.path.join(OUTPUT_PATH, _file)
+                common_util.convert_json_to_yaml(json_data=response_data, output_file_path=compose_out_file_path)
+        git_handler_obj.push_deployments_to_git(repo_link=docker_repo, private_token=git_access_token, branch=_branch,
+                                                folder_path=OUTPUT_PATH,helm_deployment=False,
+                                                base_path=docker_compose_path)
+        git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=_branch,
+                                             destination_branch=_client_name)
+    except Exception as e:
+        logging.error(f'Exception occurred while preparing the docker compose {e.args}')

helm_automate_script.py

@@ -109,7 +109,8 @@ if __name__ == '__main__':
             sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
             _module_names = [_each.replace(".yml", "") for _each in sorted_files]
 
-        global_config_data = common_util.convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
+        global_config_data = common_util.convert_yaml_to_define_obj(
+            os.path.join(base_helm_directory_path, global_configmap))
         # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
 
         variables_file = "variables.yml"
@@ -135,7 +136,7 @@ if __name__ == '__main__':
                                                                       clone_file_path=variables_file):
                 logging.debug("Failed to clone module!! Skipping Helm File Preparation")
                 continue
-            _module_data = common_util.convert_yaml_to_json(variables_file_path)
+            _module_data = common_util.convert_yaml_to_define_obj(variables_file_path)
             module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
             module_env_variables = {_v['name']: _v for _v in module_env_variables}
             template_file = os.path.join(template_path, f'{_module}.yml')
@@ -166,9 +167,10 @@ if __name__ == '__main__':
                                                                   helm_out_file_path=helm_out_file_path,
                                                                   global_config_data=global_config_data,
                                                                   module_name=_file.split(".yml")[0])
-        git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _branch, final_helm_path=OUTPUT_PATH,
-                                              base_path=helm_path)
-        git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch, destination_branch=_client_name)
+        git_handler_obj.push_deployments_to_git(helm_repo, git_access_token, _branch, folder_path=OUTPUT_PATH,
+                                                base_path=helm_path)
+        git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch,
+                                             destination_branch=_client_name)
     except Exception as e:
         logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
     finally:

helm_register.py

@@ -130,7 +130,7 @@ if __name__ == '__main__':
             sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
             _module_names = [_each.replace(".yml", "") for _each in sorted_files]
 
-        global_config_data = common_util.convert_yaml_to_json(os.path.join(base_helm_directory_path, global_configmap))
+        global_config_data = common_util.convert_yaml_to_define_obj(os.path.join(base_helm_directory_path, global_configmap))
         # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
         if repo_info:
             global_config_data.update(repo_info)
@@ -158,7 +158,7 @@ if __name__ == '__main__':
                                                                       clone_file_path=variables_file):
                 logging.debug("Failed to clone module!! Skipping Helm File Preparation")
                 continue
-            _module_data = common_util.convert_yaml_to_json(variables_file_path)
+            _module_data = common_util.convert_yaml_to_define_obj(variables_file_path)
             module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
 
             module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if

scripts/core/docker_handler.py

+import os
+
+from scripts.core import ILensVersionHandler
+from scripts.core.git_handler import GitHandler
+from scripts.logging import logging
+from scripts.utils.common_utils import CommonUtils
+
+
+class DockerHandler:
+
+    def __init__(self, git_user_name, git_access_token):
+        self.git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
+        self.common_util = CommonUtils()
+        self.db_handler = ILensVersionHandler()
+
+    def process_module_request(self, compose_data: dict, variables_file, general_path,
+                               **kwargs):
+        try:
+            ilens_version = kwargs.get("ilens_version")
+            branch_name = kwargs.get("branch_name")
+            release_version = kwargs.get("release_version")
+            for _module, module_dict in compose_data['services'].items():
+                variable_name = f'x-{_module}-image'
+                module_path = os.path.join(general_path)
+                module_path = os.path.join(module_path, _module)
+                if not os.path.exists(module_path):
+                    os.makedirs(module_path)
+                variables_file_path = os.path.join(module_path, variables_file)
+                git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
+                if not git_info:
+                    logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
+                    continue
+                if not self.git_handler_obj.clone_repository_with_defined_file(repo_link=git_info,
+                                                                               clone_branch=branch_name,
+                                                                               file_output_path=variables_file_path,
+                                                                               clone_file_path=variables_file):
+                    logging.debug("Failed to clone module!! Skipping Helm File Preparation")
+                    continue
+                _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
+                # session_obj = get_db_for_func()
+                # module_info = self.db_handler.get_module_versions(
+                #     input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version,
+                #                           release_version=release_version), db=session_obj)
+                # session_obj.close()
+                _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
+                module_info = {}
+                module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
+                module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables}
+                image_url = module_info.get("image_url", '') if module_info else ''
+                if image_url:
+                    module_dict['image'] = image_url
+                    compose_data[variable_name] = image_url
+                existing_env_variables = module_dict.get('environment', {})
+                diff_keys = list(
+                    set(existing_env_variables.keys()).symmetric_difference(set(module_env_variables.keys())))
+                for _each in diff_keys:
+                    value = module_env_variables.get(_each)
+                    if _each.lower() in {'port', 'service_port', 'module_port'} or not value:
+                        continue
+                    if value.startswith('{{') and value.endswith('}}'):
+                        value = f"${{{value.lstrip('{{').rstrip('}}').rstrip(' ').lstrip(' ')}}}"
+                    module_dict['environment'].update({_each: value})
+            return compose_data
+
+        except Exception as e:
+            logging.exception(f'Exception occurred while process each module {e.args}')
+            return False

scripts/core/git_handler.py

@@ -89,7 +89,7 @@ class GitHandler:
             return False
         items = pl.repository_tree(path=HELM_PATH, ref=branch)
         print(items)
-        
+
     def create_merge_request(self, repo_link: str, source_branch: str, destination_branch: str):
         base_url = os.environ.get("GIT_BASE_URL", default=default_link)
         repo_link_split = repo_link.split(base_url)
@@ -108,7 +108,8 @@ class GitHandler:
         })
 
     @staticmethod
-    def push_helm_deployments(repo_link: str, private_token: str, branch: str, final_helm_path, base_path: str):
+    def push_deployments_to_git(repo_link: str, private_token: str, branch: str, folder_path, base_path: str,
+                                helm_deployment=True):
         try:
             base_url = os.environ.get("GIT_BASE_URL", default=default_link)
             repo_link_split = repo_link.split(base_url)
@@ -121,9 +122,10 @@ class GitHandler:
                 return False
             pl = pl[0]
             commit_actions = []
-            files_list = os.listdir(final_helm_path)
+            files_list = os.listdir(folder_path)
             branches = pl.branches.list(get_all=True)
             branches_names = [x.name for x in branches]
+
             if branch not in branches_names:
                 pl.branches.create({'branch': branch,
                                     'ref': 'master' if branch.split("_")[0] not in branches_names else
@@ -132,14 +134,17 @@ class GitHandler:
             if not files_list:
                 logging.debug('Files not found for pushing to git.')
             for file in files_list:
+                file_path = f'{file}'
+                if helm_deployment:
+                    file_path = f"{HELM_PATH}/{file}"
                 _action = {
                     'action': 'update',
-                    'file_path': f"{HELM_PATH}/{file}",
-                    'content': open(f'{final_helm_path}/{file}').read()
+                    'file_path': file_path,
+                    'content': open(f'{folder_path}/{file}').read()
                 }
                 commit_actions.append(_action)
             commit_data = {'branch': branch, 'commit_message': f"{branch} helm creation"} | {'actions': commit_actions}
 
             pl.commits.create(commit_data)
         except Exception as e:
-            logging.exception(f'Exception while pushing helm deployments: {e.args}')
+            logging.exception(f'Exception while pushing deployments to git: {e.args}')

scripts/core/helm_handler.py

@@ -20,7 +20,7 @@ class HelmHandler:
             if not os.path.exists(template_yml_path):
                 logging.debug(f"{template_yml_path} not found!! Skipping Helm File Preparation")
                 return False
-            existing_data = self.common_utils.convert_yaml_to_json(template_yml_path)
+            existing_data = self.common_utils.convert_yaml_to_define_obj(template_yml_path)
             global_config_vars = global_config_data.get('data', {})
             existing_env_variables = {_v['name']: _v for _v in
                                       existing_data.get('deployment', {}).get('environmentVar', [])}

scripts/utils/common_utils.py

@@ -8,12 +8,13 @@ from scripts.logging import logging
 class CommonUtils:
 
     @staticmethod
-    def convert_yaml_to_json(yaml_file_path):
+    def convert_yaml_to_define_obj(yaml_file_path, load_type='safe'):
         try:
             if not os.path.exists(yaml_file_path):
                 return {}
-            _yaml = ruamel.yaml.YAML(typ='safe')
+            _yaml = ruamel.yaml.YAML(typ=load_type)
             _yaml.preserve_quotes = True
+            _yaml.allow_duplicate_keys = True
             with open(yaml_file_path) as fpi:
                 yaml_dict = _yaml.load(fpi)
             return yaml_dict
@@ -31,3 +32,5 @@ class CommonUtils:
         except Exception as e:
             logging.exception(f"Exception Occurred while reading the yaml file {e.args}")
             return False
+
+