Commit 35838748 authored by Yori Fournier's avatar Yori Fournier
Browse files

finish the populate-template script for the manifest

parent 79c89650
project_name = "ml-solar-plates"
pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-features", "extract-features-properties"]
# RESOURCES: define STORAGE and CPU usage (need to cope with Quotas)
[resources]
[resources.archives_volume]
storage = "10Gi"
[resources.artifacts_volume]
storage = "10Gi"
# STEPS: define the configuration of the steps: repo, commit, volumes mount point...
[steps]
[steps.extract-features]
[steps.extract-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
commit = "1ccb3969"
# the archives to be mounted
archives = ["ml_solar_plates_mock"]
# The volume where output are stored
[steps.extract-features.artifact_volume]
mount_path = "/home/fournier/artifact"
# The list of archives and where they will be mounted in the container
[[steps.extract-features.archives_volume]]
name = "ml_solar_plates_mock"
mount_path = "/home/fournier/ml_solar_plates"
[steps.label-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
commit = "0f33b92"
# no archives to be mounted
archives = []
# The list of steps on which this one depends (dependancies)
dep = ['extract-features']
# The volume where output are stored
[steps.label-features.artifact_volume]
mount_path = "home/fournier/artifact"
# The list of dependancies and where they are mounted
[[steps.label-features.deps_volume]]
name = "extract-features"
mount_path = "home/fournier/extract-features"
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
commit = "1ccb3969"
archives = ["ml_solar_plates_mock"]
[steps.select-meaningful-plates-and-features]
[steps.label-features]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
commit = "81ca31c"
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
commit = "0f33b92"
# The archives to be mounted
archives = ["ml_solar_plates_mock"]
archives = []
# The list of dependancies
dep = ['extract-features', 'label-features']
dep = ['extract-features']
# The volume where output are stored
[steps.select-meaningful-plates-and-features.artifact_volume]
mount_path = "home/fournier/artifact"
[steps.select-meaningful-plates-and-features]
# The list of archives and where they will be mounted in the container
[[steps.select-meaningful-plates-and-features.archives_volume]]
name = "ml_solar_plates_mock"
mount_path = "home/fournier/ml_solar_plates"
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
commit = "81ca31c"
# The list of dependancies and where they are mounted
[[steps.select-meaningful-plates-and-features.deps_volume]]
name = "extract-features"
mount_path = "home/fournier/extract-features"
archives = ["ml_solar_plates_mock"]
[[steps.select-meaningful-plates-and-features.deps_volume]]
name = "label-features"
mount_path = "home/fournier/label-features"
dep = ['extract-features', 'label-features']
[steps.extract-features-properties]
[steps.extract-features-properties]
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features-properties.git"
commit = "e36591f9"
type = "git_repo"
repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features-properties.git"
commit = "e36591f9"
......@@ -8,7 +8,7 @@ spec:
- ReadWriteOnce
resources:
requests:
storage: {{ artifacts_volume.storage_size }}
storage: {{ resources.artifacts_volume.storage }}
---
apiVersion: v1
kind: PersistentVolumeClaim
......@@ -20,12 +20,12 @@ spec:
- ReadWriteOnce
resources:
requests:
storage: {{ archives_volume.storage_size }}
storage: {{ resources.archives_volume.storage }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ job.step_name }}
name: {{ step_name }}
spec:
template:
spec:
......@@ -37,22 +37,22 @@ spec:
persistentVolumeClaim:
claimName: {{ project_name }}-archives-pv-claim
containers:
- name: {{ job.container.name }}
image: {{ job.container.image }}
- name: {{ step_name }}
image: {{ step_name }}:{{ commit }}
imagePullPolicy: Never
volumeMounts:
- mountPath: {{ job.container.artifact_volume.mount_path }}
- mountPath: {{ artifact_volume.mount_path }}
name: artifacts
subPath: {{ job.container.artifact_volume.sub_path }}
{% for n in range(job.container.deps_volume | length) -%}
- mountPath: {{ job.container.deps_volume[n].mount_path }}
subPath: {{ artifact_volume.sub_path }}
{% for n in range(deps_volume | length) -%}
- mountPath: {{ deps_volume[n].mount_path }}
name: artifacts
subPath: {{ job.container.deps_volume[n].sub_path }}
subPath: {{ deps_volume[n].sub_path }}
{% endfor %}
{%- for n in range(job.container.archives_volume | length) -%}
- mountPath: {{ job.container.archives_volume[n].mount_path }}
{%- for n in range(archives_volume | length) -%}
- mountPath: {{ archives_volume[n].mount_path }}
name: archives
subPath: {{ job.container.archives_volume[n].sub_path }}
subPath: {{ archives_volume[n].sub_path }}
{%- endfor %}
restartPolicy: Never
backoffLimit: 1
\ No newline at end of file
from jinja2 import Template
import toml
# INPUT PARAM ARE:
# STEP_NAME, COMMTI, DEPS_DIR (alike for run_step.sh)
import argparse
parser = argparse.ArgumentParser(description='Populate a step manifest template with pipeline config')
parser.add_argument('step_name', metavar='STEP_NAME', type=str,
help='The name of the step to run')
parser.add_argument('commit', metavar='COMMIT', type=str,
help='The commit of the src repo (tag of the corresponding image)')
parser.add_argument('deps_dir', metavar='DEPS_DIR', type=str, nargs='+',
help='The list of dependances directory')
args = parser.parse_args()
with open('multi-manifest.yaml', 'r') as fd:
template = Template(fd.read())
with open('step_config.toml', 'r') as fd:
with open('../config.toml', 'r') as fd:
config = toml.load(fd)
#print(config)
# update config with
step_config = config['steps'][args.step_name]
# print(step_config)
step_config['step_name'] = args.step_name
step_config['project_name'] = config['project_name']
step_config['resources'] = config['resources']
for dep_dir in args.deps_dir:
for volume in step_config['deps_volume']:
if volume['name'] == dep_dir[0:-27]: # name of the step
volume['sub_path'] = dep_dir
for volume in step_config['archives_volume']:
volume['sub_path'] = volume['name']
print(step_config)
# WHERE steps.${step_name}.deps_volumes.name = DEPS_DIR[:: -27] (dep step_name)
# steps.${step_name}.deps_volumes.sub_path = DEPS_DIR
rendered = template.render(**config)
rendered = template.render(**step_config)
print(rendered)
......
......@@ -12,14 +12,6 @@ storage_size = "1Gi"
step_name = "select-features"
[job.artifacts_volume]
name = "artifacts"
[job.archives_volume]
name = "archives"
[job.container]
name = "select-features"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment