finish the populate-template script for the manifest

config.toml

+project_name = "ml-solar-plates"
+
 pipeline = ["extract-features", "label-features", "select-meaningful-plates-and-features", "extract-features-properties"]
 
+# RESOURCES: define STORAGE and CPU usage (need to cope with Quotas)
+
+[resources]
+
+    [resources.archives_volume]
+
+         storage = "10Gi"
+
+    [resources.artifacts_volume]
+
+         storage = "10Gi"
+
+
+# STEPS: define the configuration of the steps: repo, commit, volumes mount point...
+
 [steps]
 
-[steps.extract-features]
+    [steps.extract-features]
+
+        type = "git_repo"
+        repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
+        commit = "1ccb3969"
+
+	# the archives to be mounted
+        archives = ["ml_solar_plates_mock"] 
+
+	# The volume where output are stored
+        [steps.extract-features.artifact_volume]
+            mount_path = "/home/fournier/artifact"
+
+	# The list of archives and where they will be mounted in the container
+        [[steps.extract-features.archives_volume]]
+            name = "ml_solar_plates_mock"
+            mount_path = "/home/fournier/ml_solar_plates"
+
+
+    [steps.label-features]
+
+        type = "git_repo"
+        repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
+        commit = "0f33b92"
+
+	# no archives to be mounted
+        archives = []
+
+	# The list of steps on which this one depends (dependancies)
+        dep = ['extract-features']
+
+	# The volume where output are stored
+        [steps.label-features.artifact_volume]
+            mount_path = "home/fournier/artifact"
+
+	# The list of dependancies and where they are mounted
+        [[steps.label-features.deps_volume]]
+            name = "extract-features"
+            mount_path = "home/fournier/extract-features"
 
-type = "git_repo"
-repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features.git"
-commit = "1ccb3969"
 
-archives = ["ml_solar_plates_mock"]
+    [steps.select-meaningful-plates-and-features]
 
-[steps.label-features]
+        type = "git_repo"
+        repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
+        commit = "81ca31c"
 
-type = "git_repo"
-repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/label-features.git"
-commit = "0f33b92"
+	# The archives to be mounted
+        archives = ["ml_solar_plates_mock"]
 
-archives = []
+	# The list of dependancies 
+        dep = ['extract-features', 'label-features']
 
-dep = ['extract-features']
+	# The volume where output are stored
+        [steps.select-meaningful-plates-and-features.artifact_volume]
+            mount_path = "home/fournier/artifact"
 
-[steps.select-meaningful-plates-and-features]
+	# The list of archives and where they will be mounted in the container
+        [[steps.select-meaningful-plates-and-features.archives_volume]]
+            name = "ml_solar_plates_mock"
+            mount_path = "home/fournier/ml_solar_plates"
 
-type = "git_repo"
-repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/select-meaningful-plates-and-features.git"
-commit = "81ca31c"
+	# The list of dependancies and where they are mounted
+        [[steps.select-meaningful-plates-and-features.deps_volume]]
+            name = "extract-features"
+            mount_path = "home/fournier/extract-features"
 
-archives = ["ml_solar_plates_mock"]
+        [[steps.select-meaningful-plates-and-features.deps_volume]]
+            name = "label-features"
+            mount_path = "home/fournier/label-features"
 
-dep = ['extract-features', 'label-features']
 
-[steps.extract-features-properties]
+    [steps.extract-features-properties]
 
-type = "git_repo"
-repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features-properties.git"
-commit = "e36591f9"
+    type = "git_repo"
+    repo_url = "https://gitlab.aip.de/ml_solar_plates/modern_pipeline/extract-features-properties.git"
+    commit = "e36591f9"
 
 

manifests-template/multi-manifest.yaml

@@ -8,7 +8,7 @@ spec:
     - ReadWriteOnce
   resources:
     requests:
-      storage: {{ artifacts_volume.storage_size }}
+      storage: {{ resources.artifacts_volume.storage }}
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -20,12 +20,12 @@ spec:
     - ReadWriteOnce
   resources:
     requests:
-      storage: {{ archives_volume.storage_size }}
+      storage: {{ resources.archives_volume.storage }}
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ job.step_name }}
+  name: {{ step_name }}
 spec:
   template:
     spec:
@@ -37,22 +37,22 @@ spec:
           persistentVolumeClaim:
             claimName: {{ project_name }}-archives-pv-claim	    
       containers:
-        - name: {{ job.container.name }}
-          image: {{ job.container.image }}
+        - name: {{ step_name }}
+          image: {{ step_name }}:{{ commit }}
           imagePullPolicy: Never
           volumeMounts:
-            - mountPath: {{ job.container.artifact_volume.mount_path }}
+            - mountPath: {{ artifact_volume.mount_path }}
               name: artifacts
-              subPath: {{ job.container.artifact_volume.sub_path }}
-	    {% for n in range(job.container.deps_volume | length) -%}
-	    - mountPath: {{ job.container.deps_volume[n].mount_path }}
+              subPath: {{ artifact_volume.sub_path }}
+	    {% for n in range(deps_volume | length) -%}
+	    - mountPath: {{ deps_volume[n].mount_path }}
 	      name: artifacts
-	      subPath: {{ job.container.deps_volume[n].sub_path }}
+	      subPath: {{ deps_volume[n].sub_path }}
 	    {% endfor %}
-	    {%- for n in range(job.container.archives_volume | length) -%}
-	    - mountPath: {{ job.container.archives_volume[n].mount_path }}
+	    {%- for n in range(archives_volume | length) -%}
+	    - mountPath: {{ archives_volume[n].mount_path }}
 	      name: archives
-	      subPath: {{ job.container.archives_volume[n].sub_path }}
+	      subPath: {{ archives_volume[n].sub_path }}
 	    {%- endfor %}
       restartPolicy: Never
   backoffLimit: 1
\ No newline at end of file

manifests-template/populate_template.py

 from jinja2 import Template
 import toml
 
+# INPUT PARAM ARE:
+# STEP_NAME, COMMTI, DEPS_DIR (alike for run_step.sh)
+
+import argparse
+
+parser = argparse.ArgumentParser(description='Populate a step manifest template with pipeline config')
+parser.add_argument('step_name', metavar='STEP_NAME', type=str,
+                    help='The name of the step to run')
+parser.add_argument('commit', metavar='COMMIT', type=str,
+                    help='The commit of the src repo (tag of the corresponding image)')
+parser.add_argument('deps_dir', metavar='DEPS_DIR', type=str, nargs='+',
+                    help='The list of dependances directory')
+
+
+args = parser.parse_args()
+
 with open('multi-manifest.yaml', 'r') as fd:
     template = Template(fd.read())
 
-with open('step_config.toml', 'r') as fd:
+with open('../config.toml', 'r') as fd:
     config = toml.load(fd)
 
-#print(config)
+# update config with
+
+step_config = config['steps'][args.step_name]
+
+# print(step_config)
+
+step_config['step_name'] = args.step_name
+step_config['project_name'] = config['project_name']
+step_config['resources'] = config['resources']
+
+for dep_dir in args.deps_dir:
+
+    for volume in step_config['deps_volume']:
+        if volume['name'] == dep_dir[0:-27]: # name of the step
+            volume['sub_path'] = dep_dir
+
+for volume in step_config['archives_volume']:
+    volume['sub_path'] = volume['name']
+    
+
+print(step_config)
+
+# WHERE steps.${step_name}.deps_volumes.name = DEPS_DIR[:: -27] (dep step_name)
+# steps.${step_name}.deps_volumes.sub_path = DEPS_DIR
     
-rendered = template.render(**config)
+rendered = template.render(**step_config)
 
 print(rendered)
 

manifests-template/step_config.toml

@@ -12,14 +12,6 @@ storage_size = "1Gi"
 
 step_name = "select-features"
 
-[job.artifacts_volume]
-
-name = "artifacts"
-
-[job.archives_volume]
-
-name = "archives"
-
 [job.container]
 
 name = "select-features"