Skip to content

H
Helm-Automation-Script
Commit ba823dcc authored by harshavardhan.c's avatar harshavardhan.c
Browse files
Options

Dev: code optimization fixes

parent 29ed9a0f
Show whitespace changes
Inline Side-by-side
Showing with 120 additions and 85 deletions
docker_automation/docker_automate_script.py
View file @ ba823dcc
...@@ -31,6 +31,12 @@ class DockerVersionUpgrade: ...@@ -31,6 +31,12 @@ class DockerVersionUpgrade:
general_temp_path = f"{int(time.time())}_tmp" general_temp_path = f"{int(time.time())}_tmp"
output_path = f"{int(time.time())}_docker-compose" output_path = f"{int(time.time())}_docker-compose"
try: try:
self.process_docker_automation(docker_temp_path, output_path, general_temp_path)
except Exception as e:
logging.error(f'Exception occurred while preparing the docker compose {e.args}')
def process_docker_automation(self, docker_temp_path, output_path, general_temp_path):
_release_version = self.arguments["release_version"] _release_version = self.arguments["release_version"]
_client_name = self.arguments['client_name'] _client_name = self.arguments['client_name']
_git_repos = self.arguments["git_repos"] _git_repos = self.arguments["git_repos"]
...@@ -60,5 +66,3 @@ class DockerVersionUpgrade: ...@@ -60,5 +66,3 @@ class DockerVersionUpgrade:
tmp_path=general_temp_path, tmp_path=general_temp_path,
output_path=output_path, output_path=output_path,
docker_compose_path=docker_compose_path) docker_compose_path=docker_compose_path)
except Exception as e:
logging.error(f'Exception occurred while preparing the docker compose {e.args}')
docker_automation/docker_register.py
View file @ ba823dcc
import json import json
from scripts.config import EnvironmentVariables
if __name__ == "__main__": if __name__ == "__main__":
from dotenv import load_dotenv from dotenv import load_dotenv
...@@ -69,6 +71,12 @@ class DockerRegistration: ...@@ -69,6 +71,12 @@ class DockerRegistration:
arguments=self.arguments, compose_info=repo_info, arguments=self.arguments, compose_info=repo_info,
tmp_path=general_temp_path, tmp_path=general_temp_path,
output_path=output_path, modules=_module_names) output_path=output_path, modules=_module_names)
# self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo,
# private_token=EnvironmentVariables.git_access_token,
# branch=_branch,
# folder_path=output_path, helm_deployment=False,
# base_path=docker_compose_path)
# self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=_branch,
# destination_branch=_client_name)
except Exception as e: except Exception as e:
logging.exception(f"Exception Occurred while generating the docker compose file {e.args}") logging.exception(f"Exception Occurred while generating the docker compose file {e.args}")
scripts/core/docker_handler.py
View file @ ba823dcc
...@@ -16,20 +16,22 @@ class DockerHandler: ...@@ -16,20 +16,22 @@ class DockerHandler:
self.common_util = CommonUtils() self.common_util = CommonUtils()
self.db_handler = ILensVersionHandler() self.db_handler = ILensVersionHandler()
def process_module_request(self, compose_data: dict, variables_file, general_path, def process_module_request(self, compose_data: dict, variables_file, general_path, image_mapping_dict: dict,
**kwargs): **kwargs):
try: try:
ilens_version = kwargs.get("ilens_version") ilens_version = kwargs.get("ilens_version")
client_version = kwargs.get("client", "iLens")
branch_name = kwargs.get("branch_name") branch_name = kwargs.get("branch_name")
release_version = kwargs.get("release_version") release_version = kwargs.get("release_version")
for _module, module_dict in compose_data['services'].items(): for _module, module_dict in compose_data['services'].items():
variable_name = f'x-{_module}-image' updated_module_name = self.get_defined_module_name(module_name=_module,
module_mapping_dict=image_mapping_dict)
module_path = os.path.join(general_path) module_path = os.path.join(general_path)
module_path = os.path.join(module_path, _module) module_path = os.path.join(module_path, _module)
if not os.path.exists(module_path): if not os.path.exists(module_path):
os.makedirs(module_path) os.makedirs(module_path)
variables_file_path = os.path.join(module_path, variables_file) variables_file_path = os.path.join(module_path, variables_file)
git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=_module) git_info = self.git_handler_obj.get_git_url_by_module_name(module_name=updated_module_name)
if not git_info: if not git_info:
logging.debug("Failed to fetch module info!! Skipping Helm File Preparation") logging.debug("Failed to fetch module info!! Skipping Helm File Preparation")
continue continue
...@@ -40,18 +42,14 @@ class DockerHandler: ...@@ -40,18 +42,14 @@ class DockerHandler:
logging.debug("Failed to clone module!! Skipping Docker File Preparation") logging.debug("Failed to clone module!! Skipping Docker File Preparation")
continue continue
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path) _module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
session_obj = get_db_for_func() if updated_module_name in image_mapping_dict:
module_info = self.db_handler.get_module_versions( image_url = image_mapping_dict.get(updated_module_name, '')
input_data=GetRequest(module_name=_module, client='iLens', ilens_version=ilens_version, else:
release_version=release_version), db=session_obj) image_url = self.get_image_tag_from_db(module=_module, ilens_version=ilens_version,
session_obj.close() release_version=release_version, client=client_version)
_module_data = self.common_util.convert_yaml_to_define_obj(variables_file_path)
module_env_variables = _module_data.get('deployment', {}).get('environmentVar', []) module_env_variables = _module_data.get('deployment', {}).get('environmentVar', [])
module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables} module_env_variables = {_v['name']: _v.get('value') for _v in module_env_variables}
image_url = module_info.get("image_url", '') if module_info else '' image_mapping_dict[_module] = image_url
if image_url:
module_dict['image'] = image_url
compose_data[variable_name] = image_url
existing_env_variables = module_dict.get('environment', {}) existing_env_variables = module_dict.get('environment', {})
diff_keys = list( diff_keys = list(
set(existing_env_variables.keys()).symmetric_difference(set(module_env_variables.keys()))) set(existing_env_variables.keys()).symmetric_difference(set(module_env_variables.keys())))
...@@ -72,63 +70,88 @@ class DockerHandler: ...@@ -72,63 +70,88 @@ class DockerHandler:
tmp_path, output_path, source_branch, destination_branch): tmp_path, output_path, source_branch, destination_branch):
docker_repo = EnvironmentVariables.docker_repo docker_repo = EnvironmentVariables.docker_repo
global_configmap = EnvironmentVariables.global_configmap global_configmap = EnvironmentVariables.global_configmap
jinja_template_file = 'docker_deployment.yml'
template_path = "./templates"
try: try:
files_info = os.listdir(docker_compose_path) files_info = os.listdir(docker_compose_path)
sorted_files = list(filter(lambda f: f.endswith(".yml") and "docker" in f, files_info)) sorted_files = list(filter(lambda f: f.endswith(".yml") and "docker" in f, files_info))
global_config_data = self.common_util.convert_yaml_to_define_obj( global_config_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(docker_compose_path, global_configmap)) os.path.join(docker_compose_path, global_configmap))
image_mapping_dict = {}
for _file in sorted_files: for index, _file in enumerate(sorted_files):
docker_compose_data = self.common_util.convert_yaml_to_define_obj( docker_compose_data = self.common_util.convert_yaml_to_define_obj(
os.path.join(docker_compose_path, _file), os.path.join(docker_compose_path, _file))
load_type=None)
service_dict = docker_compose_data.get("services") service_dict = docker_compose_data.get("services")
if not service_dict: if not service_dict:
logging.debug(f'Services not found for current docker compose file - {_file}') logging.debug(f'Services not found for current docker compose file - {_file}')
if index + 1 == len(sorted_files):
modules = self.get_modules_dict_for_registration(modules=arguments.get('module_names', []),
existing_modules=list(
docker_compose_data['services'].keys()))
docker_compose_data['services'].update(modules)
if response_data := self.process_module_request(compose_data=docker_compose_data, if response_data := self.process_module_request(compose_data=docker_compose_data,
variables_file=variables_file, variables_file=variables_file,
image_mapping_dict=image_mapping_dict,
general_path=tmp_path, **arguments): general_path=tmp_path, **arguments):
compose_out_file_path = os.path.join(output_path, _file) compose_out_file_path = os.path.join(output_path, _file)
self.common_util.convert_json_to_yaml(json_data=response_data, # self.common_util.convert_json_to_yaml(json_data=response_data,
output_file_path=compose_out_file_path) # output_file_path=compose_out_file_path)
self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo, self.common_util.render_deployment_chart(data_dict={"modules": response_data.get('services', {})},
private_token=EnvironmentVariables.git_access_token, jinja_template_file=jinja_template_file,
branch=source_branch, template_path=template_path,
folder_path=output_path, helm_deployment=False, outfile_path=compose_out_file_path)
base_path=docker_compose_path) # self.git_handler_obj.push_deployments_to_git(repo_link=docker_repo,
self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=source_branch, # private_token=EnvironmentVariables.git_access_token,
destination_branch=destination_branch) # branch=source_branch,
# folder_path=output_path, helm_deployment=False,
# base_path=docker_compose_path)
# self.git_handler_obj.create_merge_request(repo_link=docker_repo, source_branch=source_branch,
# destination_branch=destination_branch)
except Exception as e: except Exception as e:
logging.exception(f'Exception Occurred while processing the compose data info {e.args}') logging.exception(f'Exception Occurred while processing the compose data info {e.args}')
def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments, def process_compose_data_by_template(self, modules, compose_info: dict, variables_file, arguments,
tmp_path, output_path): tmp_path, output_path):
ilens_version = arguments.get("ilens_version") compose_dict = {"services": self.get_modules_dict_for_registration(modules=modules)}
branch_name = arguments.get("branch_name")
release_version = arguments.get("release_version")
compose_dict = {"services": {}}
try: try:
for module in modules:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client='iLens', ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
compose_dict['services'].update({
module: DockerComposeSchema(image=image_url).dict()
})
compose_out_file_path = os.path.join(output_path, "docker-compose.yml") compose_out_file_path = os.path.join(output_path, "docker-compose.yml")
jinja_template_file = 'docker_deployment.yml' jinja_template_file = 'docker_deployment.yml'
template_path = "./templates" template_path = "./templates"
compose_data = self.process_module_request(compose_data=compose_dict, image_mapping_dict = {}
compose_data = self.process_module_request(compose_data=compose_dict, image_mapping_dict=image_mapping_dict,
variables_file=variables_file, variables_file=variables_file,
general_path=tmp_path, **arguments) general_path=tmp_path, **arguments)
self.common_util.render_helm_chart(data_dict={"modules": compose_data.get('services', {})}, self.common_util.render_deployment_chart(data_dict={"modules": compose_data.get('services', {})},
jinja_template_file=jinja_template_file, template_path=template_path, jinja_template_file=jinja_template_file,
template_path=template_path,
outfile_path=compose_out_file_path) outfile_path=compose_out_file_path)
except Exception as e: except Exception as e:
logging.exception(f'Exception Occurred while processing the compose data by template {e.args}') logging.exception(f'Exception Occurred while processing the compose data by template {e.args}')
@staticmethod
def get_modules_dict_for_registration(modules, existing_modules=None):
if existing_modules is None:
existing_modules = []
return {module: DockerComposeSchema().dict() for module in modules if module not in existing_modules}
@staticmethod
def get_defined_module_name(module_name, module_mapping_dict):
for key in module_mapping_dict:
if key.startswith(module_name):
module_name = key
break
return module_name
def get_image_tag_from_db(self, module, ilens_version, client, release_version):
image_url = ''
try:
session_obj = get_db_for_func()
module_info = self.db_handler.get_module_versions(
input_data=GetRequest(module_name=module, client=client, ilens_version=ilens_version,
release_version=release_version), db=session_obj)
session_obj.close()
image_url = module_info.get("image_url", '') if module_info else ''
except Exception as e:
logging.exception(f'Exception occurred while fetching the image details {e.args}')
return image_url
scripts/core/helm_handler.py
View file @ ba823dcc
...@@ -33,7 +33,7 @@ class HelmHandler: ...@@ -33,7 +33,7 @@ class HelmHandler:
existing_data['deployment']['imageName'] = image_tag or existing_data['deployment']['imageName'] existing_data['deployment']['imageName'] = image_tag or existing_data['deployment']['imageName']
self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file) self.common_utils.convert_json_to_yaml(json_data=existing_data, output_file_path=template_file)
self.common_utils.render_helm_chart(data_dict=global_config_vars, jinja_template_file=f'{module_name}.yml', self.common_utils.render_deployment_chart(data_dict=global_config_vars, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path=template_path, outfile_path=helm_out_file_path, template_path=template_path,
service_type=service_type) service_type=service_type)
return True return True
...@@ -61,7 +61,7 @@ class HelmHandler: ...@@ -61,7 +61,7 @@ class HelmHandler:
value_key = v.strip("<{ }>") value_key = v.strip("<{ }>")
global_config_vars.update({k: global_config_vars.get(value_key, v)}) global_config_vars.update({k: global_config_vars.get(value_key, v)})
self.common_utils.render_helm_chart(data_dict={}, jinja_template_file=f'{module_name}.yml', self.common_utils.render_deployment_chart(data_dict={}, jinja_template_file=f'{module_name}.yml',
outfile_path=helm_out_file_path, template_path="templates", outfile_path=helm_out_file_path, template_path="templates",
service_type=service_type, image_url=image_tag, service_type=service_type, image_url=image_tag,
variables=global_config_vars, variables=global_config_vars,
......
scripts/utils/common_utils.py
View file @ ba823dcc
...@@ -35,7 +35,7 @@ class CommonUtils: ...@@ -35,7 +35,7 @@ class CommonUtils:
return False return False
@staticmethod @staticmethod
def render_helm_chart(data_dict, jinja_template_file, template_path, outfile_path, **kwargs): def render_deployment_chart(data_dict, jinja_template_file, template_path, outfile_path, **kwargs):
try: try:
environment = jinja2.Environment( environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=template_path), loader=jinja2.FileSystemLoader(searchpath=template_path),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment