Skip to content

H
Helm-Automation-Script
Commit 29ed9a0f authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

Dev: integration fixes while generating helm-scripts dynamically.

parent f413d47e
Show whitespace changes
Inline Side-by-side
Showing with 288 additions and 354 deletions
automation_script.py
View file @ 29ed9a0f
...@@ -4,9 +4,11 @@ import sys ...@@ -4,9 +4,11 @@ import sys
from dotenv import load_dotenv from dotenv import load_dotenv
from docker_automation.docker_automate_script import DockerVersionUpgrade
load_dotenv() load_dotenv()
from helm_automation.helm_register import HelmRegistration
from docker_automation.docker_automate_script import DockerVersionUpgrade
from helm_automation.helm_automate_script import HelmVersionUpgrade
from docker_automation.docker_register import DockerRegistration from docker_automation.docker_register import DockerRegistration
from scripts.config import EnvironmentVariables from scripts.config import EnvironmentVariables
from scripts.logging import logger from scripts.logging import logger
...@@ -25,6 +27,10 @@ def func_mapper(operation_type, parameters): ...@@ -25,6 +27,10 @@ def func_mapper(operation_type, parameters):
"docker_registration": DockerRegistration(arguments=parameters, git_user_name=git_user_name, "docker_registration": DockerRegistration(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration, git_access_token=git_access_token).module_registration,
"docker_version_upgrade": DockerVersionUpgrade(arguments=parameters, git_user_name=git_user_name, "docker_version_upgrade": DockerVersionUpgrade(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration,
"helm_version_upgrade": HelmVersionUpgrade(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration,
"helm_registration": HelmRegistration(arguments=parameters, git_user_name=git_user_name,
git_access_token=git_access_token).module_registration git_access_token=git_access_token).module_registration
} }
return func_mapper_dict.get(operation_type) return func_mapper_dict.get(operation_type)
......
docker_automation/docker_automate_script.py
View file @ 29ed9a0f
...@@ -37,9 +37,9 @@ class DockerVersionUpgrade: ...@@ -37,9 +37,9 @@ class DockerVersionUpgrade:
_module_names = self.arguments["module_names"] _module_names = self.arguments["module_names"]
_branch_name = self.arguments['branch_name'] or "master" _branch_name = self.arguments['branch_name'] or "master"
_ilens_version = self.arguments["ilens_version"] _ilens_version = self.arguments["ilens_version"]
if not _ilens_version or not _release_version or not _client_name or not _git_repos and not _module_names or not global_configmap: if not _ilens_version or not _release_version or not _client_name or not global_configmap:
print( print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details " "global_configmap, client_name, ilens_version and release_version details "
"not found!!!!!") "not found!!!!!")
sys.exit() sys.exit()
variables_file = "variables.yml" variables_file = "variables.yml"
......
helm_automation/helm_automate_script.py
View file @ 29ed9a0f
from scripts.config import EnvironmentVariables
from scripts.core.helm_handler import HelmHandler
if __name__ == "__main__": if __name__ == "__main__":
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
import argparse
import logging import logging
import os import os
import sys import sys
...@@ -10,7 +12,6 @@ import time ...@@ -10,7 +12,6 @@ import time
from scripts.core import ILensVersionHandler from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler from scripts.core.git_handler import GitHandler
from scripts.core.helm_handler import HelmHandler
from scripts.db.psql.databases import get_db_for_func from scripts.db.psql.databases import get_db_for_func
from scripts.schemas import GetRequest from scripts.schemas import GetRequest
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
...@@ -18,131 +19,95 @@ from scripts.utils.common_utils import CommonUtils ...@@ -18,131 +19,95 @@ from scripts.utils.common_utils import CommonUtils
HELM_PATH = "/ilens-core/ilens-modules" HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts" HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token) global_configmap = EnvironmentVariables.global_configmap
helm_repo = EnvironmentVariables.helm_repo
ap = argparse.ArgumentParser() class HelmVersionUpgrade:
db_handler = ILensVersionHandler() def __init__(self, arguments: dict, git_user_name: str, git_access_token: str):
common_util = CommonUtils() self.arguments = arguments
helm_handler = HelmHandler() self.db_handler = ILensVersionHandler()
if __name__ == '__main__': self.helm_handler = HelmHandler()
ap.add_argument( self.git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
"--ilens_version", self.common_util = CommonUtils()
"-iv",
required=False, def module_registration(self):
default=None, helm_temp_path = f"{int(time.time())}_helm_tmp_path"
help="ILens Version Tag", general_temp_path = f"{int(time.time())}_tmp"
) output_path = f"{int(time.time())}_helm-charts"
ap.add_argument(
"--release_version",
"-rv",
required=False,
default=None,
help="ILens Release Tag",
)
ap.add_argument(
"--client_name",
"-cn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--branch_name",
"-b",
required=False,
default=None,
help="Branch Name"
)
ap.add_argument(
"--git_repos",
"-gr",
required=False,
default=None,
help="Git repos to be added in helm",
nargs="+"
)
ap.add_argument(
"--module_names",
"-mn",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
OUTPUT_PATH = f"{int(time.time())}_helm-charts"
try: try:
arguments = vars(ap.parse_args()) _ilens_version = self.arguments["ilens_version"]
_ilens_version = arguments["ilens_version"] _release_version = self.arguments["release_version"]
_release_version = arguments["release_version"] _client_name = self.arguments['client_name']
_client_name = arguments['client_name'] _git_repos = self.arguments["git_repos"]
_git_repos = arguments["git_repos"] _module_names = self.arguments["module_names"]
_module_names = arguments["module_names"] _branch_name = self.arguments['branch_name'] or "master"
_branch_name = arguments['branch_name'] or "master"
if not _ilens_version or not _release_version or not _client_name or not ( if not _ilens_version or not _release_version or not _client_name or not (
_git_repos or _module_names) or not global_configmap: _git_repos or _module_names) or not global_configmap:
print( print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!") "global_configmap, git_repos, client_name, ilens_version and release_version details not found!!!!!")
sys.exit() sys.exit()
_branch = f"{_client_name}_{_ilens_version}.{_release_version}" _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not os.path.exists(HELM_TEMP_PATH): if not os.path.exists(helm_temp_path):
os.makedirs(HELM_TEMP_PATH) os.makedirs(helm_temp_path)
if not os.path.exists(OUTPUT_PATH): if not os.path.exists(output_path):
os.makedirs(OUTPUT_PATH) os.makedirs(output_path)
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts") helm_path = os.path.join(helm_temp_path, "helm-charts")
if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path, if not self.git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
clone_branch=_client_name): clone_branch=_client_name):
logging.error(f"Cannot clone helm repo with branch: {_client_name}") logging.error(f"Cannot clone helm repo with branch: {_client_name}")
sys.exit() sys.exit()
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules") base_helm_directory_path = os.path.join(helm_temp_path, "helm-charts", "ilens-core", "ilens-modules")
if len(_module_names) == 1 and _module_names[0].lower() == "all": if len(_module_names) == 1 and _module_names[0].lower() == "all":
updated_list = [] updated_list = []
files_info = os.listdir(base_helm_directory_path) files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info)) sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files] _module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = common_util.convert_yaml_to_define_obj( global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(base_helm_directory_path, global_configmap)) os.path.join(base_helm_directory_path, global_configmap))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml") # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
variables_file = "variables.yml" variables_file = "variables.yml"
template_path = os.path.join(GENERAL_TEMP_PATH, "../templates") template_path = os.path.join(general_temp_path, "../templates")
if not os.path.exists(template_path): if not os.path.exists(template_path):
os.makedirs(template_path) os.makedirs(template_path)
for _module in _module_names: for _module in _module_names:
module_path = os.path.join(GENERAL_TEMP_PATH) module_path = os.path.join(general_temp_path)
module_path = os.path.join(module_path, _module) module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path): if not os.path.exists(module_path):
os.makedirs(module_path) os.makedirs(module_path)
helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml') helm_out_file_path = os.path.join(output_path, f'{_module}.yml')
if os.path.exists(helm_out_file_path): if os.path.exists(helm_out_file_path):
logging.debug(f"Helm Deployment File found for selected the module {_module}") logging.debug(f"Helm Deployment File found for selected the module {_module}")
continue continue
variables_file_path = os.path.join(module_path, variables_file) variables_file_path = os.path.join(module_path, variables_file)
git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module) git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
if not git_info: if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation") logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue continue
if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name, if not self.git_handler_obj.clone_repository_with_defined_file(repo_link=git_info,
clone_branch=_branch_name,
file_output_path=variables_file_path, file_output_path=variables_file_path,
clone_file_path=variables_file): clone_file_path=variables_file):
logging.debug("Failed to clone module!! Skipping Helm File Preparation") logging.debug("Failed to clone module!! Skipping Helm File Preparation")
continue continue
_module_data = common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v for _v in module_env_variables} module_env_variables = {_v['name']: _v for _v in module_env_variables}
template_file = os.path.join(template_path, f'{_module}.yml') template_file = os.path.join(template_path, f'{_module}.yml')
session_obj = get_db_for_func() session_obj = get_db_for_func()
module_info = db_handler.get_module_versions( module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version, input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
release_version=_release_version), db=session_obj) release_version=_release_version), db=session_obj)
session_obj.close() session_obj.close()
image_url = module_info.get("image_url", '') if module_info else '' image_url = module_info.get("image_url", '') if module_info else ''
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml') existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path,
image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
template_file=template_file, template_path=template_path, template_file=template_file,
template_path=template_path,
helm_out_file_path=helm_out_file_path, helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, global_config_data=global_config_data,
module_name=_module) module_name=_module)
...@@ -151,24 +116,26 @@ if __name__ == '__main__': ...@@ -151,24 +116,26 @@ if __name__ == '__main__':
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x] updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
for _file in updated_files_info: for _file in updated_files_info:
sub_file_path = os.path.join(base_helm_directory_path, _file) sub_file_path = os.path.join(base_helm_directory_path, _file)
helm_out_file_path = os.path.join(OUTPUT_PATH, _file) helm_out_file_path = os.path.join(output_path, _file)
template_file = os.path.join(template_path, _file) template_file = os.path.join(template_path, _file)
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, image_tag=image_url, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path,
image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
template_file=template_file, template_file=template_file,
template_path=template_path, template_path=template_path,
helm_out_file_path=helm_out_file_path, helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, global_config_data=global_config_data,
module_name=_file.split(".yml")[0]) module_name=_file.split(".yml")[0])
git_handler_obj.push_deployments_to_git(helm_repo, git_access_token, _branch, folder_path=OUTPUT_PATH, self.git_handler_obj.push_deployments_to_git(repo_link=helm_repo, branch=_branch, folder_path=output_path,
base_path=helm_path) base_path=helm_path,
git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch, private_token=EnvironmentVariables.git_access_token)
self.git_handler_obj.create_merge_request(repo_link=helm_repo, source_branch=_branch,
destination_branch=_client_name) destination_branch=_client_name)
except Exception as e: except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}") logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
finally: finally:
# shutil.rmtree(HELM_STORE_PATH) # shutil.rmtree(HELM_STORE_PATH)
# shutil.rmtree(HELM_TEMP_PATH) # shutil.rmtree(helm_temp_path)
# shutil.rmtree(GENERAL_TEMP_PATH) # shutil.rmtree(general_temp_path)
# shutil.rmtree(OUTPUT_PATH) # shutil.rmtree(output_path)
... ...
helm_automation/helm_register.py
View file @ 29ed9a0f
...@@ -10,7 +10,6 @@ if __name__ == "__main__": ...@@ -10,7 +10,6 @@ if __name__ == "__main__":
from scripts.db.psql.databases import get_db_for_func from scripts.db.psql.databases import get_db_for_func
from scripts.schemas import GetRequest from scripts.schemas import GetRequest
import argparse
import logging import logging
import os import os
import sys import sys
...@@ -22,174 +21,118 @@ from scripts.core.helm_handler import HelmHandler ...@@ -22,174 +21,118 @@ from scripts.core.helm_handler import HelmHandler
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
default_link = EnvironmentVariables.default_link default_link = EnvironmentVariables.default_link
git_user_name = EnvironmentVariables.git_user_name
git_access_token = EnvironmentVariables.git_access_token
config_variables = EnvironmentVariables.config_variables
helm_repo = EnvironmentVariables.helm_repo helm_repo = EnvironmentVariables.helm_repo
global_configmap = EnvironmentVariables.global_configmap global_configmap = EnvironmentVariables.global_configmap
HELM_PATH = "/ilens-core/ilens-modules" HELM_PATH = "/ilens-core/ilens-modules"
HELM_STORE_PATH = "./helm-charts" HELM_STORE_PATH = "./helm-charts"
git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
ap = argparse.ArgumentParser() class HelmRegistration:
db_handler = ILensVersionHandler() def __init__(self, arguments: dict, git_user_name: str, git_access_token: str):
common_util = CommonUtils() self.arguments = arguments
helm_handler = HelmHandler() self.db_handler = ILensVersionHandler()
if __name__ == '__main__': self.helm_handler = HelmHandler()
ap.add_argument( self.git_handler_obj = GitHandler(user_name=git_user_name, access_token=git_access_token)
"--ilens_version", self.common_util = CommonUtils()
"-iv",
required=False, def module_registration(self):
default=None, helm_temp_path = f"{int(time.time())}_helm_tmp_path"
help="ILens Version Tag", general_temp_path = f"{int(time.time())}_tmp"
) output_path = f"{int(time.time())}_helm-charts"
ap.add_argument(
"--release_version",
"-rv",
required=False,
default=None,
help="ILens Release Tag",
)
ap.add_argument(
"--reference_client_name",
"-rcn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--client_name",
"-cn",
required=False,
default=None,
help="Client Name Tag"
)
ap.add_argument(
"--branch_name",
"-b",
required=False,
default=None,
help="Branch Name"
)
ap.add_argument(
"--git_repos",
"-gr",
required=False,
default=None,
help="Git repos to be added in helm",
nargs="+"
)
ap.add_argument(
"--module_names",
"-mn",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
ap.add_argument(
"--repo_info",
"-ri",
required=False,
default=None,
help="Module names to be added in helm",
nargs="+"
)
HELM_TEMP_PATH = f"{int(time.time())}_helm_tmp_path"
GENERAL_TEMP_PATH = f"{int(time.time())}_tmp"
OUTPUT_PATH = f"{int(time.time())}_helm-charts"
try: try:
arguments = vars(ap.parse_args()) _ilens_version = self.arguments["ilens_version"]
_ilens_version = arguments["ilens_version"] _release_version = self.arguments["release_version"]
_release_version = arguments["release_version"] _client_name = self.arguments['client_name']
_client_name = arguments['client_name'] _reference_branch = self.arguments['reference_client_name']
_reference_branch = arguments['reference_client_name'] _git_repos = self.arguments["git_repos"]
_git_repos = arguments["git_repos"] _module_names = self.arguments["module_names"]
repo_info = arguments["repo_info"] _branch_name = self.arguments['branch_name'] or "master"
_module_names = arguments["module_names"] repo_info = self.arguments["repo_info"]
_branch_name = arguments['branch_name'] or "master"
if repo_info: if repo_info:
repo_info = json.loads(repo_info) repo_info = json.loads(repo_info)
if not _ilens_version or not _release_version or not _client_name or not ( if not _ilens_version or not _release_version or not _client_name or not _git_repos and not _module_names or not global_configmap:
_git_repos or _module_names) or not global_configmap:
print( print(
"global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!") "global_configmap, git_repos, module_names, client_name, ilens_version and release_version details not found!!!!!")
sys.exit() sys.exit()
_branch = f"{_client_name}_{_ilens_version}.{_release_version}" _branch = f"{_client_name}_{_ilens_version}.{_release_version}"
if not os.path.exists(HELM_TEMP_PATH): if not os.path.exists(helm_temp_path):
os.makedirs(HELM_TEMP_PATH) os.makedirs(helm_temp_path)
if not os.path.exists(OUTPUT_PATH): if not os.path.exists(output_path):
os.makedirs(OUTPUT_PATH) os.makedirs(output_path)
helm_path = os.path.join(HELM_TEMP_PATH, "helm-charts") helm_path = os.path.join(helm_temp_path, "helm-charts")
if not git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path, if not self.git_handler_obj.clone_repository(repo_link=helm_repo, module_output_path=helm_path,
clone_branch=_reference_branch or _client_name): clone_branch=_reference_branch or _client_name):
logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}") logging.error(f"Cannot clone helm repo with branch: {_reference_branch or _client_name}")
base_helm_directory_path = os.path.join(HELM_TEMP_PATH, "helm-charts", "ilens-core", "ilens-modules") base_helm_directory_path = os.path.join(helm_temp_path, "helm-charts", "ilens-core", "ilens-modules")
if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[0].lower() == "all": if os.path.exists(base_helm_directory_path) and len(_module_names) == 1 and _module_names[
0].lower() == "all":
updated_list = [] updated_list = []
files_info = os.listdir(base_helm_directory_path) files_info = os.listdir(base_helm_directory_path)
sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info)) sorted_files = list(filter(lambda f: f.endswith(".yml"), files_info))
_module_names = [_each.replace(".yml", "") for _each in sorted_files] _module_names = [_each.replace(".yml", "") for _each in sorted_files]
global_config_data = common_util.convert_yaml_to_define_obj( global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(base_helm_directory_path, global_configmap)) os.path.join(base_helm_directory_path, global_configmap))
# global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml") # global_config_data = common_util.convert_yaml_to_json("ilens-global-configmap.yml")
if repo_info: if repo_info:
global_config_data.update(repo_info) global_config_data.update(repo_info)
variables_file = "variables.yml" variables_file = "variables.yml"
template_path = os.path.join(GENERAL_TEMP_PATH, "../templates") template_path = os.path.join(general_temp_path, "../templates")
if not os.path.exists(template_path): if not os.path.exists(template_path):
os.makedirs(template_path) os.makedirs(template_path)
for _module in _module_names: for _module in _module_names:
module_path = os.path.join(GENERAL_TEMP_PATH) module_path = os.path.join(general_temp_path)
module_path = os.path.join(module_path, _module) module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path): if not os.path.exists(module_path):
os.makedirs(module_path) os.makedirs(module_path)
helm_out_file_path = os.path.join(OUTPUT_PATH, f'{_module}.yml') helm_out_file_path = os.path.join(output_path, f'{_module}.yml')
if os.path.exists(helm_out_file_path): if os.path.exists(helm_out_file_path):
logging.debug(f"Helm Deployment File found for selected the module {_module}") logging.debug(f"Helm Deployment File found for selected the module {_module}")
continue continue
variables_file_path = os.path.join(module_path, variables_file) variables_file_path = os.path.join(module_path, variables_file)
git_info = git_handler_obj.get_git_url_by_module_name(module_name=_module) git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module)
if not git_info: if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation") logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue continue
if not git_handler_obj.clone_repository_with_defined_file(repo_link=git_info, clone_branch=_branch_name, if not self.git_handler_obj.clone_repository_with_defined_file(repo_link=git_info,
clone_branch=_branch_name,
file_output_path=variables_file_path, file_output_path=variables_file_path,
clone_file_path=variables_file): clone_file_path=variables_file):
logging.debug("Failed to clone module!! Skipping Helm File Preparation") logging.debug("Failed to clone module!! Skipping Helm File Preparation")
continue continue
_module_data = common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if module_env_variables = {_v['name']: _v['value'] for _v in module_env_variables if
{'name', 'value'}.issubset(set(list(_v.keys())))} {'name', 'value'}.issubset(set(list(_v.keys())))}
template_file = os.path.join(template_path, f'{_module}.yml') template_file = os.path.join(template_path, f'{_module}.yml')
session_obj = get_db_for_func() session_obj = get_db_for_func()
module_info = db_handler.get_module_versions( module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version, input_data=GetRequest(module_name=_module, client='iLens', ilens_version=_ilens_version,
release_version=_release_version), db=session_obj) release_version=_release_version), db=session_obj)
session_obj.close() session_obj.close()
image_url = module_info.get("image_url", '') if module_info else '' image_url = module_info.get("image_url", '') if module_info else ''
existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml') existing_yml_path = os.path.join(base_helm_directory_path, f'{_module}.yml')
new_module = deepcopy(_module) new_module = deepcopy(_module)
worker_type = True if "worker" in _module or "celery" in _module else False worker_type = "worker" in _module or "celery" in _module
if not os.path.exists(existing_yml_path): if not os.path.exists(existing_yml_path):
existing_yml_path = os.path.join("../templates", "helm_deployment.yml") existing_yml_path = os.path.join("../templates", "helm_deployment.yml")
new_module = "helm_deployment" new_module = "helm_deployment"
helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path, image_tag=image_url, self.helm_handler.create_helm_deployment_file(template_yml_path=existing_yml_path,
image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
helm_out_file_path=helm_out_file_path, helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, module_name=new_module) global_config_data=global_config_data,
module_name=new_module)
else: else:
helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=existing_yml_path,
image_tag=image_url, image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
helm_out_file_path=helm_out_file_path, helm_out_file_path=helm_out_file_path,
global_config_data=global_config_data, global_config_data=global_config_data,
module_name=new_module, template_file=template_file, module_name=new_module,
template_file=template_file,
template_path=template_path) template_path=template_path)
if os.path.exists(base_helm_directory_path): if os.path.exists(base_helm_directory_path):
files_info = os.listdir(base_helm_directory_path) files_info = os.listdir(base_helm_directory_path)
...@@ -197,10 +140,10 @@ if __name__ == '__main__': ...@@ -197,10 +140,10 @@ if __name__ == '__main__':
updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x] updated_files_info = [x for x in sorted_files if f'{_module}.yml' != x]
for _file in updated_files_info: for _file in updated_files_info:
sub_file_path = os.path.join(base_helm_directory_path, _file) sub_file_path = os.path.join(base_helm_directory_path, _file)
helm_out_file_path = os.path.join(OUTPUT_PATH, _file) helm_out_file_path = os.path.join(output_path, _file)
template_file = os.path.join(template_path, _file) template_file = os.path.join(template_path, _file)
worker_type = True if "worker" in _module or "celery" in _module else False worker_type = "worker" in _module or "celery" in _module
helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path, self.helm_handler.create_existing_helm_deployment_file(template_yml_path=sub_file_path,
image_tag=image_url, image_tag=image_url,
module_env_variables=module_env_variables, module_env_variables=module_env_variables,
template_file=template_file, template_file=template_file,
...@@ -209,8 +152,10 @@ if __name__ == '__main__': ...@@ -209,8 +152,10 @@ if __name__ == '__main__':
global_config_data=global_config_data, global_config_data=global_config_data,
service_type=False, service_type=False,
module_name=_file.split(".yml")[0]) module_name=_file.split(".yml")[0])
# git_handler_obj.push_helm_deployments(helm_repo, git_access_token, _client_name, final_helm_path=OUTPUT_PATH, # self.git_handler_obj.push_deployments_to_git(repo_link=helm_repo,
# base_path=helm_path) # private_token=git_access_token,
# branch=_client_name,
# folder_path=output_path, base_path=helm_path)
except Exception as e: except Exception as e:
logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}") logging.exception(f"Exception Occurred while processing the Helm-Script Preparation {e.args}")
......
scripts/core/docker_handler.py
View file @ 29ed9a0f
...@@ -3,8 +3,9 @@ import os ...@@ -3,8 +3,9 @@ import os
from scripts.config import EnvironmentVariables from scripts.config import EnvironmentVariables
from scripts.core import ILensVersionHandler from scripts.core import ILensVersionHandler
from scripts.core.git_handler import GitHandler from scripts.core.git_handler import GitHandler
from scripts.db.psql.databases import get_db_for_func
from scripts.logging import logging from scripts.logging import logging
from scripts.schemas import DockerComposeSchema from scripts.schemas import DockerComposeSchema, GetRequest
from scripts.utils.common_utils import CommonUtils from scripts.utils.common_utils import CommonUtils
...@@ -39,13 +40,12 @@ class DockerHandler: ...@@ -39,13 +40,12 @@ class DockerHandler:
logging.debug("Failed to clone module!! Skipping Docker File Preparation") logging.debug("Failed to clone module!! Skipping Docker File Preparation")
continue continue
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
# session_obj = get_db_for_func() session_obj = get_db_for_func()
# module_info = self.db_handler.get_module_versions( module_info = self.db_handler.get_module_versions(
# input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version, input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version,
# release_version=release_version), db=session_obj) release_version=release_version), db=session_obj)
# session_obj.close() session_obj.close()
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_info = {}
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables} module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables}
image_url = module_info.get("image_url", '') if module_info else '' image_url = module_info.get("image_url", '') if module_info else ''
...@@ -105,11 +105,20 @@ class DockerHandler: ...@@ -105,11 +105,20 @@ class DockerHandler:
def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments, def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments,
tmp_path, output_path): tmp_path, output_path):
ilens_version = arguments.get("ilens_version")
branch_name = arguments.get("branch_name")
release_version = arguments.get("release_version")
compose_dict = {"services": {}} compose_dict = {"services": {}}
try: try:
for module in modules: for module in modules:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client='iLens', ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
compose_dict['services'].update({ compose_dict['services'].update({
module: DockerComposeSchema().dict() module: DockerComposeSchema(image=image_url).dict()
}) })
compose_out_file_path = os.path.join(output_path, "docker-compose.yml") compose_out_file_path = os.path.join(output_path, "docker-compose.yml")
jinja_template_file = 'docker_deployment.yml' jinja_template_file = 'docker_deployment.yml'
......
templates/docker_deployment.yml
View file @ 29ed9a0f
version: "2" {% for k,v in modules.items() %}
x-{{ k }}-image: &{{k}}-image {{v['image']}}
{% endfor %}
services: services:
{% for k,v in modules.items() %} {% for k,v in modules.items() %}
{{k}}: {{k}}:
image: {{v['image']}} image: *{{k}}-image
ports: ports:
- {{v['ports']}} - {{v['ports']}}
environment: environment:
...@@ -16,3 +18,8 @@ services: ...@@ -16,3 +18,8 @@ services:
- {{volume}} - {{volume}}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
networks:
default:
external:
name: "ilens"
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment